Domain block (#76)

* start work on admin domain blocking

* move stuff around + further work on domain blocks

* move + restructure processor

* prep work for deleting account

* tidy

* go fmt

* formatting

* domain blocking more work

* check domain blocks way earlier on

* progress on delete account

* delete more stuff when an account is gone

* and more...

* domain blocky block block

* get individual domain block, delete a block

internal/federation/authenticate.go

@@ -25,7 +25,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"net/http"
 	"net/url"
 	"strings"
 
@@ -35,6 +34,7 @@ import (
 	"github.com/go-fed/httpsig"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 /*
@@ -115,34 +115,30 @@ func getPublicKeyFromResponse(c context.Context, b []byte, keyID *url.URL) (voca
 //
 // Also note that this function *does not* dereference the remote account that the signature key is associated with.
 // Other functions should use the returned URL to dereference the remote account, if required.
-func (f *federator) AuthenticateFederatedRequest(requestedUsername string, r *http.Request) (*url.URL, error) {
+func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedUsername string) (*url.URL, bool, error) {
+	l := f.log.WithField("func", "AuthenticateFederatedRequest")
 
 	var publicKey interface{}
 	var pkOwnerURI *url.URL
 	var err error
 
-	// set this extra field for signature validation
-	r.Header.Set("host", f.config.Host)
-
-	verifier, err := httpsig.NewVerifier(r)
-	if err != nil {
-		return nil, fmt.Errorf("could not create http sig verifier: %s", err)
+	// thanks to signaturecheck.go in the security package, we should already have a signature verifier set on the context
+	vi := ctx.Value(util.APRequestingPublicKeyVerifier)
+	if vi == nil {
+		l.Debug("request wasn't signed")
+		return nil, false, nil // request wasn't signed
+	}
+
+	verifier, ok := vi.(httpsig.Verifier)
+	if !ok {
+		l.Debug("couldn't extract sig verifier")
+		return nil, false, nil // couldn't extract the verifier
 	}
 
-	// The key ID should be given in the signature so that we know where to fetch it from the remote server.
-	// This will be something like https://example.org/users/whatever_requesting_user#main-key
 	requestingPublicKeyID, err := url.Parse(verifier.KeyId())
 	if err != nil {
-		return nil, fmt.Errorf("could not parse key id into a url: %s", err)
-	}
-
-	// if the domain is blocked we want to make as few calls towards it as possible, so already bail here if that's the case!
-	blockedDomain, err := f.blockedDomain(requestingPublicKeyID.Host)
-	if err != nil {
-		return nil, fmt.Errorf("could not tell if domain %s was blocked or not: %s", requestingPublicKeyID.Host, err)
-	}
-	if blockedDomain {
-		return nil, fmt.Errorf("host %s was domain blocked, aborting auth", requestingPublicKeyID.Host)
+		l.Debug("couldn't parse public key URL")
+		return nil, false, nil // couldn't parse the public key ID url
 	}
 
 	requestingRemoteAccount := &gtsmodel.Account{}
@@ -152,12 +148,12 @@ func (f *federator) AuthenticateFederatedRequest(requestedUsername string, r *ht
 		// LOCAL ACCOUNT REQUEST
 		// the request is coming from INSIDE THE HOUSE so skip the remote dereferencing
 		if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingLocalAccount); err != nil {
-			return nil, fmt.Errorf("couldn't get local account with public key uri %s from the database: %s", requestingPublicKeyID.String(), err)
+			return nil, false, fmt.Errorf("couldn't get local account with public key uri %s from the database: %s", requestingPublicKeyID.String(), err)
 		}
 		publicKey = requestingLocalAccount.PublicKey
 		pkOwnerURI, err = url.Parse(requestingLocalAccount.URI)
 		if err != nil {
-			return nil, fmt.Errorf("error parsing url %s: %s", requestingLocalAccount.URI, err)
+			return nil, false, fmt.Errorf("error parsing url %s: %s", requestingLocalAccount.URI, err)
 		}
 	} else if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingRemoteAccount); err == nil {
 		// REMOTE ACCOUNT REQUEST WITH KEY CACHED LOCALLY
@@ -165,7 +161,7 @@ func (f *federator) AuthenticateFederatedRequest(requestedUsername string, r *ht
 		publicKey = requestingRemoteAccount.PublicKey
 		pkOwnerURI, err = url.Parse(requestingRemoteAccount.URI)
 		if err != nil {
-			return nil, fmt.Errorf("error parsing url %s: %s", requestingRemoteAccount.URI, err)
+			return nil, false, fmt.Errorf("error parsing url %s: %s", requestingRemoteAccount.URI, err)
 		}
 	} else {
 		// REMOTE ACCOUNT REQUEST WITHOUT KEY CACHED LOCALLY
@@ -173,72 +169,55 @@ func (f *federator) AuthenticateFederatedRequest(requestedUsername string, r *ht
 		// so we need to authenticate the request properly by dereferencing the remote key
 		transport, err := f.GetTransportForUser(requestedUsername)
 		if err != nil {
-			return nil, fmt.Errorf("transport err: %s", err)
+			return nil, false, fmt.Errorf("transport err: %s", err)
 		}
 
 		// The actual http call to the remote server is made right here in the Dereference function.
 		b, err := transport.Dereference(context.Background(), requestingPublicKeyID)
 		if err != nil {
-			return nil, fmt.Errorf("error deferencing key %s: %s", requestingPublicKeyID.String(), err)
+			return nil, false, fmt.Errorf("error deferencing key %s: %s", requestingPublicKeyID.String(), err)
 		}
 
 		// if the key isn't in the response, we can't authenticate the request
 		requestingPublicKey, err := getPublicKeyFromResponse(context.Background(), b, requestingPublicKeyID)
 		if err != nil {
-			return nil, fmt.Errorf("error getting key %s from response %s: %s", requestingPublicKeyID.String(), string(b), err)
+			return nil, false, fmt.Errorf("error getting key %s from response %s: %s", requestingPublicKeyID.String(), string(b), err)
 		}
 
 		// we should be able to get the actual key embedded in the vocab.W3IDSecurityV1PublicKey
 		pkPemProp := requestingPublicKey.GetW3IDSecurityV1PublicKeyPem()
 		if pkPemProp == nil || !pkPemProp.IsXMLSchemaString() {
-			return nil, errors.New("publicKeyPem property is not provided or it is not embedded as a value")
+			return nil, false, errors.New("publicKeyPem property is not provided or it is not embedded as a value")
 		}
 
 		// and decode the PEM so that we can parse it as a golang public key
 		pubKeyPem := pkPemProp.Get()
 		block, _ := pem.Decode([]byte(pubKeyPem))
 		if block == nil || block.Type != "PUBLIC KEY" {
-			return nil, errors.New("could not decode publicKeyPem to PUBLIC KEY pem block type")
+			return nil, false, errors.New("could not decode publicKeyPem to PUBLIC KEY pem block type")
 		}
 
 		publicKey, err = x509.ParsePKIXPublicKey(block.Bytes)
 		if err != nil {
-			return nil, fmt.Errorf("could not parse public key from block bytes: %s", err)
+			return nil, false, fmt.Errorf("could not parse public key from block bytes: %s", err)
 		}
 
 		// all good! we just need the URI of the key owner to return
 		pkOwnerProp := requestingPublicKey.GetW3IDSecurityV1Owner()
 		if pkOwnerProp == nil || !pkOwnerProp.IsIRI() {
-			return nil, errors.New("publicKeyOwner property is not provided or it is not embedded as a value")
+			return nil, false, errors.New("publicKeyOwner property is not provided or it is not embedded as a value")
 		}
 		pkOwnerURI = pkOwnerProp.GetIRI()
 	}
 	if publicKey == nil {
-		return nil, errors.New("returned public key was empty")
+		return nil, false, errors.New("returned public key was empty")
 	}
 
 	// do the actual authentication here!
 	algo := httpsig.RSA_SHA256 // TODO: make this more robust
 	if err := verifier.Verify(publicKey, algo); err != nil {
-		return nil, fmt.Errorf("error verifying key %s: %s", requestingPublicKeyID.String(), err)
+		return nil, false, nil
 	}
 
-	return pkOwnerURI, nil
-}
-
-func (f *federator) blockedDomain(host string) (bool, error) {
-	b := &gtsmodel.DomainBlock{}
-	err := f.db.GetWhere([]db.Where{{Key: "domain", Value: host, CaseInsensitive: true}}, b)
-	if err == nil {
-		// block exists
-		return true, nil
-	}
-
-	if _, ok := err.(db.ErrNoEntries); ok {
-		// there are no entries so there's no block
-		return false, nil
-	}
-
-	// there's an actual error
-	return false, err
+	return pkOwnerURI, true, nil
 }

internal/federation/dereference.go

@@ -9,7 +9,11 @@ import (
 
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
@@ -17,6 +21,10 @@ func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *u
 	f.startHandshake(username, remoteAccountID)
 	defer f.stopHandshake(username, remoteAccountID)
 
+	if blocked, err := f.blockedDomain(remoteAccountID.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceRemoteAccount: domain %s is blocked", remoteAccountID.Host)
+	}
+
 	transport, err := f.GetTransportForUser(username)
 	if err != nil {
 		return nil, fmt.Errorf("transport err: %s", err)
@@ -62,6 +70,10 @@ func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *u
 }
 
 func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) {
+	if blocked, err := f.blockedDomain(remoteStatusID.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceRemoteStatus: domain %s is blocked", remoteStatusID.Host)
+	}
+
 	transport, err := f.GetTransportForUser(username)
 	if err != nil {
 		return nil, fmt.Errorf("transport err: %s", err)
@@ -144,6 +156,10 @@ func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url
 }
 
 func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
+	if blocked, err := f.blockedDomain(remoteInstanceURI.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceRemoteInstance: domain %s is blocked", remoteInstanceURI.Host)
+	}
+
 	transport, err := f.GetTransportForUser(username)
 	if err != nil {
 		return nil, fmt.Errorf("transport err: %s", err)
@@ -151,3 +167,358 @@ func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI
 
 	return transport.DereferenceInstance(context.Background(), remoteInstanceURI)
 }
+
+// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming
+// federated status, back in the federating db's Create function.
+//
+// When a status comes in from the federation API, there are certain fields that
+// haven't been dereferenced yet, because we needed to provide a snappy synchronous
+// response to the caller. By the time it reaches this function though, it's being
+// processed asynchronously, so we have all the time in the world to fetch the various
+// bits and bobs that are attached to the status, and properly flesh it out, before we
+// send the status to any timelines and notify people.
+//
+// Things to dereference and fetch here:
+//
+// 1. Media attachments.
+// 2. Hashtags.
+// 3. Emojis.
+// 4. Mentions.
+// 5. Posting account.
+// 6. Replied-to-status.
+//
+// SIDE EFFECTS:
+// This function will deference all of the above, insert them in the database as necessary,
+// and attach them to the status. The status itself will not be added to the database yet,
+// that's up the caller to do.
+func (f *federator) DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error {
+	l := f.log.WithFields(logrus.Fields{
+		"func":   "dereferenceStatusFields",
+		"status": fmt.Sprintf("%+v", status),
+	})
+	l.Debug("entering function")
+
+	statusURI, err := url.Parse(status.URI)
+	if err != nil {
+		return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err)
+	}
+	if blocked, err := f.blockedDomain(statusURI.Host); blocked || err != nil {
+		return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host)
+	}
+
+	t, err := f.GetTransportForUser(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("error creating transport: %s", err)
+	}
+
+	// the status should have an ID by now, but just in case it doesn't let's generate one here
+	// because we'll need it further down
+	if status.ID == "" {
+		newID, err := id.NewULIDFromTime(status.CreatedAt)
+		if err != nil {
+			return err
+		}
+		status.ID = newID
+	}
+
+	// 1. Media attachments.
+	//
+	// At this point we should know:
+	// * the media type of the file we're looking for (a.File.ContentType)
+	// * the blurhash (a.Blurhash)
+	// * the file type (a.Type)
+	// * the remote URL (a.RemoteURL)
+	// This should be enough to pass along to the media processor.
+	attachmentIDs := []string{}
+	for _, a := range status.GTSMediaAttachments {
+		l.Debugf("dereferencing attachment: %+v", a)
+
+		// it might have been processed elsewhere so check first if it's already in the database or not
+		maybeAttachment := &gtsmodel.MediaAttachment{}
+		err := f.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
+		if err == nil {
+			// we already have it in the db, dereferenced, no need to do it again
+			l.Debugf("attachment already exists with id %s", maybeAttachment.ID)
+			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
+			continue
+		}
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			// we have a real error
+			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		// it just doesn't exist yet so carry on
+		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
+		deferencedAttachment, err := f.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
+		if err != nil {
+			l.Errorf("error dereferencing status attachment: %s", err)
+			continue
+		}
+		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
+		deferencedAttachment.StatusID = status.ID
+		deferencedAttachment.Description = a.Description
+		if err := f.db.Put(deferencedAttachment); err != nil {
+			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
+	}
+	status.Attachments = attachmentIDs
+
+	// 2. Hashtags
+
+	// 3. Emojis
+
+	// 4. Mentions
+	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
+	//
+	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
+	mentions := []string{}
+	for _, m := range status.GTSMentions {
+		if m.ID == "" {
+			mID, err := id.NewRandomULID()
+			if err != nil {
+				return err
+			}
+			m.ID = mID
+		}
+
+		uri, err := url.Parse(m.MentionedAccountURI)
+		if err != nil {
+			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
+			continue
+		}
+
+		m.StatusID = status.ID
+		m.OriginAccountID = status.GTSAuthorAccount.ID
+		m.OriginAccountURI = status.GTSAuthorAccount.URI
+
+		targetAccount := &gtsmodel.Account{}
+		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil {
+			// proper error
+			if _, ok := err.(db.ErrNoEntries); !ok {
+				return fmt.Errorf("db error checking for account with uri %s", uri.String())
+			}
+
+			// we just don't have it yet, so we should go get it....
+			accountable, err := f.DereferenceRemoteAccount(requestingUsername, uri)
+			if err != nil {
+				// we can't dereference it so just skip it
+				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
+				continue
+			}
+
+			targetAccount, err = f.typeConverter.ASRepresentationToAccount(accountable, false)
+			if err != nil {
+				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
+				continue
+			}
+
+			targetAccountID, err := id.NewRandomULID()
+			if err != nil {
+				return err
+			}
+			targetAccount.ID = targetAccountID
+
+			if err := f.db.Put(targetAccount); err != nil {
+				return fmt.Errorf("db error inserting account with uri %s", uri.String())
+			}
+		}
+
+		// by this point, we know the targetAccount exists in our database with an ID :)
+		m.TargetAccountID = targetAccount.ID
+		if err := f.db.Put(m); err != nil {
+			return fmt.Errorf("error creating mention: %s", err)
+		}
+		mentions = append(mentions, m.ID)
+	}
+	status.Mentions = mentions
+
+	return nil
+}
+
+func (f *federator) DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
+	l := f.log.WithFields(logrus.Fields{
+		"func":               "dereferenceAccountFields",
+		"requestingUsername": requestingUsername,
+	})
+
+	accountURI, err := url.Parse(account.URI)
+	if err != nil {
+		return fmt.Errorf("DereferenceAccountFields: couldn't parse account URI %s: %s", account.URI, err)
+	}
+	if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil {
+		return fmt.Errorf("DereferenceAccountFields: domain %s is blocked", accountURI.Host)
+	}
+
+	t, err := f.GetTransportForUser(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("error getting transport for user: %s", err)
+	}
+
+	// fetch the header and avatar
+	if err := f.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
+		// if this doesn't work, just skip it -- we can do it later
+		l.Debugf("error fetching header/avi for account: %s", err)
+	}
+
+	if err := f.db.UpdateByID(account.ID, account); err != nil {
+		return fmt.Errorf("error updating account in database: %s", err)
+	}
+
+	return nil
+}
+
+func (f *federator) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
+	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
+		// we can't do anything unfortunately
+		return errors.New("DereferenceAnnounce: no URI to dereference")
+	}
+
+	boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI)
+	if err != nil {
+		return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+	if blocked, err := f.blockedDomain(boostedStatusURI.Host); blocked || err != nil {
+		return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host)
+	}
+
+	// check if we already have the boosted status in the database
+	boostedStatus := &gtsmodel.Status{}
+	err = f.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus)
+	if err == nil {
+		// nice, we already have it so we don't actually need to dereference it from remote
+		announce.Content = boostedStatus.Content
+		announce.ContentWarning = boostedStatus.ContentWarning
+		announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+		announce.Sensitive = boostedStatus.Sensitive
+		announce.Language = boostedStatus.Language
+		announce.Text = boostedStatus.Text
+		announce.BoostOfID = boostedStatus.ID
+		announce.Visibility = boostedStatus.Visibility
+		announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+		announce.GTSBoostedStatus = boostedStatus
+		return nil
+	}
+
+	// we don't have it so we need to dereference it
+	statusable, err := f.DereferenceRemoteStatus(requestingUsername, boostedStatusURI)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// make sure we have the author account in the db
+	attributedToProp := statusable.GetActivityStreamsAttributedTo()
+	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
+		accountURI := iter.GetIRI()
+		if accountURI == nil {
+			continue
+		}
+
+		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, &gtsmodel.Account{}); err == nil {
+			// we already have it, fine
+			continue
+		}
+
+		// we don't have the boosted status author account yet so dereference it
+		accountable, err := f.DereferenceRemoteAccount(requestingUsername, accountURI)
+		if err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err)
+		}
+		account, err := f.typeConverter.ASRepresentationToAccount(accountable, false)
+		if err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err)
+		}
+
+		accountID, err := id.NewRandomULID()
+		if err != nil {
+			return err
+		}
+		account.ID = accountID
+
+		if err := f.db.Put(account); err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err)
+		}
+
+		if err := f.DereferenceAccountFields(account, requestingUsername, false); err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err)
+		}
+	}
+
+	// now convert the statusable into something we can understand
+	boostedStatus, err = f.typeConverter.ASStatusToStatus(statusable)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt)
+	if err != nil {
+		return nil
+	}
+	boostedStatus.ID = boostedStatusID
+
+	if err := f.db.Put(boostedStatus); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// now dereference additional fields straight away (we're already async here so we have time)
+	if err := f.DereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// update with the newly dereferenced fields
+	if err := f.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err)
+	}
+
+	// we have everything we need!
+	announce.Content = boostedStatus.Content
+	announce.ContentWarning = boostedStatus.ContentWarning
+	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+	announce.Sensitive = boostedStatus.Sensitive
+	announce.Language = boostedStatus.Language
+	announce.Text = boostedStatus.Text
+	announce.BoostOfID = boostedStatus.ID
+	announce.Visibility = boostedStatus.Visibility
+	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+	announce.GTSBoostedStatus = boostedStatus
+	return nil
+}
+
+// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
+// on behalf of requestingUsername.
+//
+// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
+//
+// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated
+// to reflect the creation of these new attachments.
+func (f *federator) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
+	accountURI, err := url.Parse(targetAccount.URI)
+	if err != nil {
+		return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err)
+	}
+	if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil {
+		return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host)
+	}
+
+	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
+		a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.AvatarRemoteURL,
+			Avatar:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing avatar for user: %s", err)
+		}
+		targetAccount.AvatarMediaAttachmentID = a.ID
+	}
+
+	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
+		a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.HeaderRemoteURL,
+			Header:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing header for user: %s", err)
+		}
+		targetAccount.HeaderMediaAttachmentID = a.ID
+	}
+	return nil
+}

internal/federation/federatingprotocol.go

@@ -119,10 +119,15 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 		return nil, false, fmt.Errorf("could not fetch requested account with username %s: %s", username, err)
 	}
 
-	publicKeyOwnerURI, err := f.AuthenticateFederatedRequest(requestedAccount.Username, r)
+	publicKeyOwnerURI, authenticated, err := f.AuthenticateFederatedRequest(ctx, requestedAccount.Username)
 	if err != nil {
 		l.Debugf("request not authenticated: %s", err)
-		return ctx, false, fmt.Errorf("not authenticated: %s", err)
+		return ctx, false, err
+	}
+
+	if !authenticated {
+		w.WriteHeader(http.StatusForbidden)
+		return ctx, false, nil
 	}
 
 	// authentication has passed, so add an instance entry for this instance if it hasn't been done already
@@ -230,6 +235,14 @@ func (f *federator) Blocked(ctx context.Context, actorIRIs []*url.URL) (bool, er
 	}
 
 	for _, uri := range actorIRIs {
+		blockedDomain, err := f.blockedDomain(uri.Host)
+		if err != nil {
+			return false, fmt.Errorf("error checking domain block: %s", err)
+		}
+		if blockedDomain {
+			return true, nil
+		}
+
 		a := &gtsmodel.Account{}
 		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, a); err != nil {
 			_, ok := err.(db.ErrNoEntries)

internal/federation/federator.go

@@ -19,7 +19,7 @@
 package federation
 
 import (
-	"net/http"
+	"context"
 	"net/url"
 	"sync"
 
@@ -29,6 +29,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
@@ -41,7 +42,13 @@ type Federator interface {
 	FederatingDB() federatingdb.DB
 	// AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources.
 	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
-	AuthenticateFederatedRequest(username string, r *http.Request) (*url.URL, error)
+	//
+	// If the request is valid and passes authentication, the URL of the key owner ID will be returned, as well as true, and nil.
+	//
+	// If the request does not pass authentication, or there's a domain block, nil, false, nil will be returned.
+	//
+	// If something goes wrong during authentication, nil, false, and an error will be returned.
+	AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, bool, error)
 	// FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that
 	// account, or an error if it doesn't exist or can't be retrieved.
 	FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error)
@@ -54,6 +61,12 @@ type Federator interface {
 	// DereferenceRemoteInstance takes the URL of a remote instance, and a username (optional) to spin up a transport with. It then
 	// does its damnedest to get some kind of information back about the instance, trying /api/v1/instance, then /.well-known/nodeinfo
 	DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
+	// DereferenceStatusFields does further dereferencing on a status.
+	DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error
+	// DereferenceAccountFields does further dereferencing on an account.
+	DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error
+	// DereferenceAnnounce does further dereferencing on an announce.
+	DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error
 	// GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username.
 	// This can be used for making signed http requests.
 	//
@@ -72,6 +85,7 @@ type federator struct {
 	clock               pub.Clock
 	typeConverter       typeutils.TypeConverter
 	transportController transport.Controller
+	mediaHandler        media.Handler
 	actor               pub.FederatingActor
 	log                 *logrus.Logger
 	handshakes          map[string][]*url.URL
@@ -79,7 +93,7 @@ type federator struct {
 }
 
 // NewFederator returns a new federator
-func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter) Federator {
+func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator {
 
 	clock := &Clock{}
 	f := &federator{
@@ -89,6 +103,7 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr
 		clock:               &Clock{},
 		typeConverter:       typeConverter,
 		transportController: transportController,
+		mediaHandler:        mediaHandler,
 		log:                 log,
 		handshakeSync:       &sync.Mutex{},
 	}

internal/federation/federator_test.go

@@ -89,7 +89,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() {
 		return nil, nil
 	}))
 	// setup module being tested
-	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter)
+	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
 
 	// setup request
 	ctx := context.Background()
@@ -155,7 +155,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() {
 	}))
 
 	// now setup module being tested, with the mock transport controller
-	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter)
+	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
 
 	// setup request
 	ctx := context.Background()

internal/federation/finger.go

@@ -30,6 +30,9 @@ import (
 )
 
 func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) {
+	if blocked, err := f.blockedDomain(targetDomain); blocked || err != nil {
+		return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain)
+	}
 
 	t, err := f.GetTransportForUser(requestingUsername)
 	if err != nil {

internal/federation/util.go

@@ -0,0 +1,23 @@
+package federation
+
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (f *federator) blockedDomain(host string) (bool, error) {
+	b := &gtsmodel.DomainBlock{}
+	err := f.db.GetWhere([]db.Where{{Key: "domain", Value: host, CaseInsensitive: true}}, b)
+	if err == nil {
+		// block exists
+		return true, nil
+	}
+
+	if _, ok := err.(db.ErrNoEntries); ok {
+		// there are no entries so there's no block
+		return false, nil
+	}
+
+	// there's an actual error
+	return false, err
+}