[bugfix] 2643 bug search for account url doesnt always work when redirected (#2673)

* update activity library so dereferencer returns full response and checks *final* link to allow for redirects

* temporarily add bodged fixed library

* remove unused code

* update getAccountFeatured() to use dereferenceCollectionPage()

* make sure to release map

* perform a 2nd decode to ensure reader is empty after primary decode

* add comment explaining choice of using Decode() instead of Unmarshal()

* update embedded activity library to latest matching https://github.com/superseriousbusiness/activity/pull/21

* add checks to look for changed URI and re-check database if redirected

* update max iteration count to 512, add checks during dereferenceAncestors() for indirect URLs

* remove doubled-up code

* fix use of status instead of current

* use URIs for checking equality for security

* use the latest known URI for boost_of_uri in case original was an indirect

* add dereferenceCollection() function for dereferenceAccountFeatured()

* pull in latest github.com/superseriousbusiness/activity version (and remove the bodge!!)

* fix typo in code comments

* update decodeType() to accept a readcloser and handle body closing

* switch to checking using BoostOfID and add note why not using BoostOfURI

* ensure InReplyTo gets unset when deleting status parent in case currently stubbed

* add tests for Collection and CollectionPage iterators

vendor/github.com/superseriousbusiness/activity/pub/util.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -71,6 +72,59 @@ const (
 	acceptHeader = "Accept"
 )
 
+// readActivityPubReq reads ActivityPub data from an incoming request, handling body close.
+func readActivityPubReq(req *http.Request) (map[string]interface{}, error) {
+	// Ensure closed when done.
+	defer req.Body.Close()
+
+	var m map[string]interface{}
+
+	// Wrap body in a JSON decoder.
+	dec := json.NewDecoder(req.Body)
+
+	// Decode JSON body as "raw" AP data map.
+	if err := dec.Decode(&m); err != nil {
+		return nil, err
+	}
+
+	// Perform a final second decode to ensure no trailing
+	// garbage data or second JSON value (indicates malformed).
+	if err := dec.Decode(&struct{}{}); err != io.EOF {
+		return nil, errors.New("trailing data after json")
+	}
+
+	return m, nil
+}
+
+// readActivityPubResp reads ActivityPub data from a dereference response, handling media type check and body close.
+func readActivityPubResp(resp *http.Response) (map[string]interface{}, error) {
+	// Ensure closed when done.
+	defer resp.Body.Close()
+
+	// Check the incoming response media type is the expected ActivityPub content-type.
+	if mediaType := resp.Header.Get("Content-Type"); !headerIsActivityPubMediaType(mediaType) {
+		return nil, fmt.Errorf("%s %s resp was not ActivityPub media type: %s", resp.Request.Method, resp.Request.URL, mediaType)
+	}
+
+	var m map[string]interface{}
+
+	// Wrap body in a JSON decoder.
+	dec := json.NewDecoder(resp.Body)
+
+	// Decode JSON body as "raw" AP data map.
+	if err := dec.Decode(&m); err != nil {
+		return nil, err
+	}
+
+	// Perform a final second decode to ensure no trailing
+	// garbage data or second JSON value (indicates malformed).
+	if err := dec.Decode(&struct{}{}); err != io.EOF {
+		return nil, errors.New("trailing data after json")
+	}
+
+	return m, nil
+}
+
 // isActivityPubPost returns true if the request is a POST request that has the
 // ActivityStreams content type header
 func isActivityPubPost(r *http.Request) bool {
@@ -774,12 +828,12 @@ func mustHaveActivityActorsMatchObjectActors(c context.Context,
 		if err != nil {
 			return err
 		}
-		b, err := tport.Dereference(c, iri)
+		resp, err := tport.Dereference(c, iri)
 		if err != nil {
 			return err
 		}
-		var m map[string]interface{}
-		if err = json.Unmarshal(b, &m); err != nil {
+		m, err := readActivityPubResp(resp)
+		if err != nil {
 			return err
 		}
 		t, err := streams.ToType(c, m)