Dereference remote replies (#132)

* decided where to put reply dereferencing

* fiddling with dereferencing threads

* further adventures

* tidy up some stuff

* move dereferencing functionality

* a bunch of refactoring

* go fmt

* more refactoring

* bleep bloop

* docs and linting

* start implementing replies collection on gts side

* fiddling around

* allow dereferencing our replies

* lint, fmt

internal/federation/dereferencing/thread.go

@@ -0,0 +1,250 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// DereferenceThread takes a statusable (something that has withReplies and withInReplyTo),
+// and dereferences statusables in the conversation.
+//
+// This process involves working up and down the chain of replies, and parsing through the collections of IDs
+// presented by remote instances as part of their replies collections, and will likely involve making several calls to
+// multiple different hosts.
+func (d *deref) DereferenceThread(username string, statusIRI *url.URL) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "DereferenceThread",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering DereferenceThread")
+
+	// if it's our status we already have everything stashed so we can bail early
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, bailing")
+		return nil
+	}
+
+	// first make sure we have this status in our db
+	_, statusable, _, err := d.GetRemoteStatus(username, statusIRI, true)
+	if err != nil {
+		return fmt.Errorf("DereferenceThread: error getting status with id %s: %s", statusIRI.String(), err)
+	}
+
+	// first iterate up through ancestors, dereferencing if necessary as we go
+	if err := d.iterateAncestors(username, *statusIRI); err != nil {
+		return fmt.Errorf("error iterating ancestors of status %s: %s", statusIRI.String(), err)
+	}
+
+	// now iterate down through descendants, again dereferencing as we go
+	if err := d.iterateDescendants(username, *statusIRI, statusable); err != nil {
+		return fmt.Errorf("error iterating descendants of status %s: %s", statusIRI.String(), err)
+	}
+
+	return nil
+}
+
+// iterateAncestors has the goal of reaching the oldest ancestor of a given status, and stashing all statuses along the way.
+func (d *deref) iterateAncestors(username string, statusIRI url.URL) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "iterateAncestors",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering iterateAncestors")
+
+	// if it's our status we don't need to dereference anything so we can immediately move up the chain
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, moving up to next ancestor")
+
+		// since this is our status, we know we can extract the id from the status path
+		_, id, err := util.ParseStatusesPath(&statusIRI)
+		if err != nil {
+			return err
+		}
+
+		status := &gtsmodel.Status{}
+		if err := d.db.GetByID(id, status); err != nil {
+			return err
+		}
+
+		if status.InReplyToURI == "" {
+			// status doesn't reply to anything
+			return nil
+		}
+		nextIRI, err := url.Parse(status.URI)
+		if err != nil {
+			return err
+		}
+		return d.iterateAncestors(username, *nextIRI)
+	}
+
+	// If we reach here, we're looking at a remote status -- make sure we have it in our db by calling GetRemoteStatus
+	// We call it with refresh to true because we want the statusable representation to parse inReplyTo from.
+	status, statusable, _, err := d.GetRemoteStatus(username, &statusIRI, true)
+	if err != nil {
+		l.Debugf("error getting remote status: %s", err)
+		return nil
+	}
+
+	inReplyTo := ap.ExtractInReplyToURI(statusable)
+	if inReplyTo == nil || inReplyTo.String() == "" {
+		// status doesn't reply to anything
+		return nil
+	}
+
+	// get the ancestor status into our database if we don't have it yet
+	if _, _, _, err := d.GetRemoteStatus(username, inReplyTo, false); err != nil {
+		l.Debugf("error getting remote status: %s", err)
+		return nil
+	}
+
+	// now enrich the current status, since we should have the ancestor in the db
+	if _, err := d.EnrichRemoteStatus(username, status); err != nil {
+		l.Debugf("error enriching remote status: %s", err)
+		return nil
+	}
+
+	// now move up to the next ancestor
+	return d.iterateAncestors(username, *inReplyTo)
+}
+
+func (d *deref) iterateDescendants(username string, statusIRI url.URL, statusable ap.Statusable) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "iterateDescendants",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering iterateDescendants")
+
+	// if it's our status we already have descendants stashed so we can bail early
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, bailing")
+		return nil
+	}
+
+	replies := statusable.GetActivityStreamsReplies()
+	if replies == nil || !replies.IsActivityStreamsCollection() {
+		l.Debug("no replies, bailing")
+		return nil
+	}
+
+	repliesCollection := replies.GetActivityStreamsCollection()
+	if repliesCollection == nil {
+		l.Debug("replies collection is nil, bailing")
+		return nil
+	}
+
+	first := repliesCollection.GetActivityStreamsFirst()
+	if first == nil {
+		l.Debug("replies collection has no first, bailing")
+		return nil
+	}
+
+	firstPage := first.GetActivityStreamsCollectionPage()
+	if firstPage == nil {
+		l.Debug("first has no collection page, bailing")
+		return nil
+	}
+
+	firstPageNext := firstPage.GetActivityStreamsNext()
+	if firstPageNext == nil || !firstPageNext.IsIRI() {
+		l.Debug("next is not an iri, bailing")
+		return nil
+	}
+
+	var foundReplies int
+	currentPageIRI := firstPageNext.GetIRI()
+
+pageLoop:
+	for {
+		l.Debugf("dereferencing page %s", currentPageIRI)
+		nextPage, err := d.DereferenceCollectionPage(username, currentPageIRI)
+		if err != nil {
+			return nil
+		}
+
+		// next items could be either a list of URLs or a list of statuses
+
+		nextItems := nextPage.GetActivityStreamsItems()
+		if nextItems.Len() == 0 {
+			// no items on this page, which means we're done
+			break pageLoop
+		}
+
+		// have a look through items and see what we can find
+		for iter := nextItems.Begin(); iter != nextItems.End(); iter = iter.Next() {
+			// We're looking for a url to feed to GetRemoteStatus.
+			// Items can be either an IRI, or a Note.
+			// If a note, we grab the ID from it and call it, rather than parsing the note.
+
+			var itemURI *url.URL
+			if iter.IsIRI() {
+				// iri, easy
+				itemURI = iter.GetIRI()
+			} else if iter.IsActivityStreamsNote() {
+				// note, get the id from it to use as iri
+				n := iter.GetActivityStreamsNote()
+				id := n.GetJSONLDId()
+				if id != nil && id.IsIRI() {
+					itemURI = id.GetIRI()
+				}
+			} else {
+				// if it's not an iri or a note, we don't know how to process it
+				continue
+			}
+
+			if itemURI.Host == d.config.Host {
+				// skip if the reply is from us -- we already have it then
+				continue
+			}
+
+			// we can confidently say now that we found something
+			foundReplies = foundReplies + 1
+
+			// get the remote statusable and put it in the db
+			_, statusable, new, err := d.GetRemoteStatus(username, itemURI, false)
+			if new && err == nil && statusable != nil {
+				// now iterate descendants of *that* status
+				if err := d.iterateDescendants(username, *itemURI, statusable); err != nil {
+					continue
+				}
+			}
+		}
+
+		next := nextPage.GetActivityStreamsNext()
+		if next != nil && next.IsIRI() {
+			l.Debug("setting next page")
+			currentPageIRI = next.GetIRI()
+		} else {
+			l.Debug("no next page, bailing")
+			break pageLoop
+		}
+	}
+
+	l.Debugf("foundReplies %d", foundReplies)
+	return nil
+}