move + restructure processor

internal/api/client/account/accountupdate_test.go

@@ -53,7 +53,7 @@ func (suite *AccountUpdateTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.accountModule = account.New(suite.config, suite.processor, suite.log).(*account.Module)
 	testrig.StandardDBSetup(suite.db)

internal/api/client/fileserver/servefile_test.go

@@ -78,7 +78,7 @@ func (suite *ServeFileTestSuite) SetupSuite() {
 	suite.db = testrig.NewTestDB()
 	suite.log = testrig.NewTestLog()
 	suite.storage = testrig.NewTestStorage()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)

internal/api/client/media/mediacreate_test.go

@@ -84,7 +84,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 
 	// setup module being tested

internal/api/client/status/statusboost_test.go

@@ -52,7 +52,7 @@ func (suite *StatusBoostTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
 	testrig.StandardDBSetup(suite.db)

internal/api/client/status/statuscreate_test.go

@@ -57,7 +57,7 @@ func (suite *StatusCreateTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
 	testrig.StandardDBSetup(suite.db)

internal/api/client/status/statusfave_test.go

@@ -55,7 +55,7 @@ func (suite *StatusFaveTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
 	testrig.StandardDBSetup(suite.db)

internal/api/client/status/statusfavedby_test.go

@@ -55,7 +55,7 @@ func (suite *StatusFavedByTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
 	testrig.StandardDBSetup(suite.db)

internal/api/client/status/statusget_test.go

@@ -45,7 +45,7 @@ func (suite *StatusGetTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
 	testrig.StandardDBSetup(suite.db)

internal/api/client/status/statusunfave_test.go

@@ -55,7 +55,7 @@ func (suite *StatusUnfaveTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
 	testrig.StandardDBSetup(suite.db)

internal/api/s2s/user/userget_test.go

@@ -42,7 +42,7 @@ func (suite *UserGetTestSuite) SetupTest() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)))
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module)
 	testrig.StandardDBSetup(suite.db)
@@ -98,7 +98,7 @@ func (suite *UserGetTestSuite) TestGetUser() {
 		}, nil
 	}))
 	// get this transport controller embedded right in the user module we're testing
-	federator := testrig.NewTestFederator(suite.db, tc)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator)
 	userModule := user.New(suite.config, processor, suite.log).(*user.Module)
 

internal/cliactions/server/server.go

@@ -112,7 +112,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log
 	mediaHandler := media.New(c, dbService, storageBackend, log)
 	oauthServer := oauth.New(dbService, log)
 	transportController := transport.NewController(c, &federation.Clock{}, http.DefaultClient, log)
-	federator := federation.NewFederator(dbService, federatingDB, transportController, c, log, typeConverter)
+	federator := federation.NewFederator(dbService, federatingDB, transportController, c, log, typeConverter, mediaHandler)
 	processor := processing.NewProcessor(c, typeConverter, federator, oauthServer, mediaHandler, storageBackend, timelineManager, dbService, log)
 	if err := processor.Start(); err != nil {
 		return fmt.Errorf("error starting processor: %s", err)

internal/cliactions/testrig/testrig.go

@@ -57,7 +57,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log
 			Body:       r,
 		}, nil
 	}))
-	federator := testrig.NewTestFederator(dbService, transportController)
+	federator := testrig.NewTestFederator(dbService, transportController, storageBackend)
 
 	processor := testrig.NewTestProcessor(dbService, storageBackend, federator)
 	if err := processor.Start(); err != nil {

internal/db/db.go

@@ -65,11 +65,6 @@ type DB interface {
 	// In case of no entries, a 'no entries' error will be returned
 	GetWhere(where []Where, i interface{}) error
 
-	// // GetWhereMany gets one entry where key = value for *ALL* parameters passed as "where".
-	// // That is, if you pass 2 'where' entries, with 1 being Key username and Value test, and the second
-	// // being Key domain and Value example.org, only entries will be returned where BOTH conditions are true.
-	// GetWhereMany(i interface{}, where ...model.Where) error
-
 	// GetAll will try to get all entries of type i.
 	// The given interface i will be set to the result of the query, whatever it is. Use a pointer or a slice.
 	// In case of no entries, a 'no entries' error will be returned
@@ -261,6 +256,10 @@ type DB interface {
 
 	// GetDomainCountForInstance returns the number of known instances known that the given domain federates with.
 	GetDomainCountForInstance(domain string) (int, error)
+
+	// GetAccountsForInstance returns a slice of accounts from the given instance, arranged by ID.
+	GetAccountsForInstance(domain string, maxID string, limit int) ([]*gtsmodel.Account, error)
+
 	/*
 		USEFUL CONVERSION FUNCTIONS
 	*/

internal/db/pg/instance.go

@@ -2,6 +2,7 @@ package pg
 
 import (
 	"github.com/go-pg/pg/v10"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
@@ -50,3 +51,31 @@ func (ps *postgresService) GetDomainCountForInstance(domain string) (int, error)
 
 	return q.Count()
 }
+
+func (ps *postgresService) GetAccountsForInstance(domain string, maxID string, limit int) ([]*gtsmodel.Account, error) {
+	accounts := []*gtsmodel.Account{}
+
+	q := ps.conn.Model(&accounts).Where("domain = ?", domain).Order("id DESC")
+
+	if maxID != "" {
+		q = q.Where("id < ?", maxID)
+	}
+
+	if limit > 0 {
+		q = q.Limit(limit)
+	}
+
+	err := q.Select()
+	if err != nil {
+		if err == pg.ErrNoRows {
+			return nil, db.ErrNoEntries{}
+		}
+		return nil, err
+	}
+
+	if len(accounts) == 0 {
+		return nil, db.ErrNoEntries{}
+	}
+
+	return accounts, nil
+}

internal/federation/dereference.go

@@ -9,7 +9,11 @@ import (
 
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
@@ -151,3 +155,331 @@ func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI
 
 	return transport.DereferenceInstance(context.Background(), remoteInstanceURI)
 }
+
+// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming
+// federated status, back in the federating db's Create function.
+//
+// When a status comes in from the federation API, there are certain fields that
+// haven't been dereferenced yet, because we needed to provide a snappy synchronous
+// response to the caller. By the time it reaches this function though, it's being
+// processed asynchronously, so we have all the time in the world to fetch the various
+// bits and bobs that are attached to the status, and properly flesh it out, before we
+// send the status to any timelines and notify people.
+//
+// Things to dereference and fetch here:
+//
+// 1. Media attachments.
+// 2. Hashtags.
+// 3. Emojis.
+// 4. Mentions.
+// 5. Posting account.
+// 6. Replied-to-status.
+//
+// SIDE EFFECTS:
+// This function will deference all of the above, insert them in the database as necessary,
+// and attach them to the status. The status itself will not be added to the database yet,
+// that's up the caller to do.
+func (f *federator) DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error {
+	l := f.log.WithFields(logrus.Fields{
+		"func":   "dereferenceStatusFields",
+		"status": fmt.Sprintf("%+v", status),
+	})
+	l.Debug("entering function")
+
+	t, err := f.GetTransportForUser(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("error creating transport: %s", err)
+	}
+
+	// the status should have an ID by now, but just in case it doesn't let's generate one here
+	// because we'll need it further down
+	if status.ID == "" {
+		newID, err := id.NewULIDFromTime(status.CreatedAt)
+		if err != nil {
+			return err
+		}
+		status.ID = newID
+	}
+
+	// 1. Media attachments.
+	//
+	// At this point we should know:
+	// * the media type of the file we're looking for (a.File.ContentType)
+	// * the blurhash (a.Blurhash)
+	// * the file type (a.Type)
+	// * the remote URL (a.RemoteURL)
+	// This should be enough to pass along to the media processor.
+	attachmentIDs := []string{}
+	for _, a := range status.GTSMediaAttachments {
+		l.Debugf("dereferencing attachment: %+v", a)
+
+		// it might have been processed elsewhere so check first if it's already in the database or not
+		maybeAttachment := &gtsmodel.MediaAttachment{}
+		err := f.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
+		if err == nil {
+			// we already have it in the db, dereferenced, no need to do it again
+			l.Debugf("attachment already exists with id %s", maybeAttachment.ID)
+			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
+			continue
+		}
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			// we have a real error
+			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		// it just doesn't exist yet so carry on
+		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
+		deferencedAttachment, err := f.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
+		if err != nil {
+			l.Errorf("error dereferencing status attachment: %s", err)
+			continue
+		}
+		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
+		deferencedAttachment.StatusID = status.ID
+		deferencedAttachment.Description = a.Description
+		if err := f.db.Put(deferencedAttachment); err != nil {
+			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
+	}
+	status.Attachments = attachmentIDs
+
+	// 2. Hashtags
+
+	// 3. Emojis
+
+	// 4. Mentions
+	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
+	//
+	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
+	mentions := []string{}
+	for _, m := range status.GTSMentions {
+		if m.ID == "" {
+			mID, err := id.NewRandomULID()
+			if err != nil {
+				return err
+			}
+			m.ID = mID
+		}
+
+		uri, err := url.Parse(m.MentionedAccountURI)
+		if err != nil {
+			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
+			continue
+		}
+
+		m.StatusID = status.ID
+		m.OriginAccountID = status.GTSAuthorAccount.ID
+		m.OriginAccountURI = status.GTSAuthorAccount.URI
+
+		targetAccount := &gtsmodel.Account{}
+		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil {
+			// proper error
+			if _, ok := err.(db.ErrNoEntries); !ok {
+				return fmt.Errorf("db error checking for account with uri %s", uri.String())
+			}
+
+			// we just don't have it yet, so we should go get it....
+			accountable, err := f.DereferenceRemoteAccount(requestingUsername, uri)
+			if err != nil {
+				// we can't dereference it so just skip it
+				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
+				continue
+			}
+
+			targetAccount, err = f.typeConverter.ASRepresentationToAccount(accountable, false)
+			if err != nil {
+				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
+				continue
+			}
+
+			targetAccountID, err := id.NewRandomULID()
+			if err != nil {
+				return err
+			}
+			targetAccount.ID = targetAccountID
+
+			if err := f.db.Put(targetAccount); err != nil {
+				return fmt.Errorf("db error inserting account with uri %s", uri.String())
+			}
+		}
+
+		// by this point, we know the targetAccount exists in our database with an ID :)
+		m.TargetAccountID = targetAccount.ID
+		if err := f.db.Put(m); err != nil {
+			return fmt.Errorf("error creating mention: %s", err)
+		}
+		mentions = append(mentions, m.ID)
+	}
+	status.Mentions = mentions
+
+	return nil
+}
+
+func (f *federator) DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
+	l := f.log.WithFields(logrus.Fields{
+		"func":               "dereferenceAccountFields",
+		"requestingUsername": requestingUsername,
+	})
+
+	t, err := f.GetTransportForUser(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("error getting transport for user: %s", err)
+	}
+
+	// fetch the header and avatar
+	if err := f.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
+		// if this doesn't work, just skip it -- we can do it later
+		l.Debugf("error fetching header/avi for account: %s", err)
+	}
+
+	if err := f.db.UpdateByID(account.ID, account); err != nil {
+		return fmt.Errorf("error updating account in database: %s", err)
+	}
+
+	return nil
+}
+
+func (f *federator) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
+	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
+		// we can't do anything unfortunately
+		return errors.New("dereferenceAnnounce: no URI to dereference")
+	}
+
+	// check if we already have the boosted status in the database
+	boostedStatus := &gtsmodel.Status{}
+	err := f.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus)
+	if err == nil {
+		// nice, we already have it so we don't actually need to dereference it from remote
+		announce.Content = boostedStatus.Content
+		announce.ContentWarning = boostedStatus.ContentWarning
+		announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+		announce.Sensitive = boostedStatus.Sensitive
+		announce.Language = boostedStatus.Language
+		announce.Text = boostedStatus.Text
+		announce.BoostOfID = boostedStatus.ID
+		announce.Visibility = boostedStatus.Visibility
+		announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+		announce.GTSBoostedStatus = boostedStatus
+		return nil
+	}
+
+	// we don't have it so we need to dereference it
+	remoteStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error parsing url %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	statusable, err := f.DereferenceRemoteStatus(requestingUsername, remoteStatusURI)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// make sure we have the author account in the db
+	attributedToProp := statusable.GetActivityStreamsAttributedTo()
+	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
+		accountURI := iter.GetIRI()
+		if accountURI == nil {
+			continue
+		}
+
+		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, &gtsmodel.Account{}); err == nil {
+			// we already have it, fine
+			continue
+		}
+
+		// we don't have the boosted status author account yet so dereference it
+		accountable, err := f.DereferenceRemoteAccount(requestingUsername, accountURI)
+		if err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err)
+		}
+		account, err := f.typeConverter.ASRepresentationToAccount(accountable, false)
+		if err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err)
+		}
+
+		accountID, err := id.NewRandomULID()
+		if err != nil {
+			return err
+		}
+		account.ID = accountID
+
+		if err := f.db.Put(account); err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err)
+		}
+
+		if err := f.DereferenceAccountFields(account, requestingUsername, false); err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err)
+		}
+	}
+
+	// now convert the statusable into something we can understand
+	boostedStatus, err = f.typeConverter.ASStatusToStatus(statusable)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt)
+	if err != nil {
+		return nil
+	}
+	boostedStatus.ID = boostedStatusID
+
+	if err := f.db.Put(boostedStatus); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// now dereference additional fields straight away (we're already async here so we have time)
+	if err := f.DereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// update with the newly dereferenced fields
+	if err := f.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err)
+	}
+
+	// we have everything we need!
+	announce.Content = boostedStatus.Content
+	announce.ContentWarning = boostedStatus.ContentWarning
+	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+	announce.Sensitive = boostedStatus.Sensitive
+	announce.Language = boostedStatus.Language
+	announce.Text = boostedStatus.Text
+	announce.BoostOfID = boostedStatus.ID
+	announce.Visibility = boostedStatus.Visibility
+	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+	announce.GTSBoostedStatus = boostedStatus
+	return nil
+}
+
+// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
+// on behalf of requestingUsername.
+//
+// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
+//
+// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated
+// to reflect the creation of these new attachments.
+func (f *federator) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
+	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
+		a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.AvatarRemoteURL,
+			Avatar:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing avatar for user: %s", err)
+		}
+		targetAccount.AvatarMediaAttachmentID = a.ID
+	}
+
+	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
+		a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.HeaderRemoteURL,
+			Header:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing header for user: %s", err)
+		}
+		targetAccount.HeaderMediaAttachmentID = a.ID
+	}
+	return nil
+}

internal/federation/federator.go

@@ -29,6 +29,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
@@ -54,6 +55,12 @@ type Federator interface {
 	// DereferenceRemoteInstance takes the URL of a remote instance, and a username (optional) to spin up a transport with. It then
 	// does its damnedest to get some kind of information back about the instance, trying /api/v1/instance, then /.well-known/nodeinfo
 	DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
+	// DereferenceStatusFields does further dereferencing on a status.
+	DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error
+	// DereferenceAccountFields does further dereferencing on an account.
+	DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error
+	// DereferenceAnnounce does further dereferencing on an announce.
+	DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error
 	// GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username.
 	// This can be used for making signed http requests.
 	//
@@ -72,6 +79,7 @@ type federator struct {
 	clock               pub.Clock
 	typeConverter       typeutils.TypeConverter
 	transportController transport.Controller
+	mediaHandler        media.Handler
 	actor               pub.FederatingActor
 	log                 *logrus.Logger
 	handshakes          map[string][]*url.URL
@@ -79,7 +87,7 @@ type federator struct {
 }
 
 // NewFederator returns a new federator
-func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter) Federator {
+func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator {
 
 	clock := &Clock{}
 	f := &federator{

internal/federation/federator_test.go

@@ -89,7 +89,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() {
 		return nil, nil
 	}))
 	// setup module being tested
-	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter)
+	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
 
 	// setup request
 	ctx := context.Background()
@@ -155,7 +155,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() {
 	}))
 
 	// now setup module being tested, with the mock transport controller
-	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter)
+	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
 
 	// setup request
 	ctx := context.Background()

internal/processing/account.go

@@ -19,512 +19,43 @@
 package processing
 
 import (
-	"errors"
-	"fmt"
-
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
-	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-// accountCreate does the dirty work of making an account and user in the database.
-// It then returns a token to the caller, for use with the new account, as per the
-// spec here: https://docs.joinmastodon.org/methods/accounts/
 func (p *processor) AccountCreate(authed *oauth.Auth, form *apimodel.AccountCreateRequest) (*apimodel.Token, error) {
-	l := p.log.WithField("func", "accountCreate")
+	return p.accountProcessor.Create(authed.Token, authed.Application, form)
-
-	if err := p.db.IsEmailAvailable(form.Email); err != nil {
-		return nil, err
-	}
-
-	if err := p.db.IsUsernameAvailable(form.Username); err != nil {
-		return nil, err
-	}
-
-	// don't store a reason if we don't require one
-	reason := form.Reason
-	if !p.config.AccountsConfig.ReasonRequired {
-		reason = ""
-	}
-
-	l.Trace("creating new username and account")
-	user, err := p.db.NewSignup(form.Username, reason, p.config.AccountsConfig.RequireApproval, form.Email, form.Password, form.IP, form.Locale, authed.Application.ID)
-	if err != nil {
-		return nil, fmt.Errorf("error creating new signup in the database: %s", err)
-	}
-
-	l.Tracef("generating a token for user %s with account %s and application %s", user.ID, user.AccountID, authed.Application.ID)
-	accessToken, err := p.oauthServer.GenerateUserAccessToken(authed.Token, authed.Application.ClientSecret, user.ID)
-	if err != nil {
-		return nil, fmt.Errorf("error creating new access token for user %s: %s", user.ID, err)
-	}
-
-	return &apimodel.Token{
-		AccessToken: accessToken.GetAccess(),
-		TokenType:   "Bearer",
-		Scope:       accessToken.GetScope(),
-		CreatedAt:   accessToken.GetAccessCreateAt().Unix(),
-	}, nil
 }
 
 func (p *processor) AccountGet(authed *oauth.Auth, targetAccountID string) (*apimodel.Account, error) {
-	targetAccount := &gtsmodel.Account{}
+	return p.accountProcessor.Get(authed.Account, targetAccountID)
-	if err := p.db.GetByID(targetAccountID, targetAccount); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
-			return nil, errors.New("account not found")
-		}
-		return nil, fmt.Errorf("db error: %s", err)
-	}
-
-	// lazily dereference things on the account if it hasn't been done yet
-	var requestingUsername string
-	if authed.Account != nil {
-		requestingUsername = authed.Account.Username
-	}
-	if err := p.dereferenceAccountFields(targetAccount, requestingUsername, false); err != nil {
-		p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err)
-	}
-
-	var mastoAccount *apimodel.Account
-	var err error
-	if authed.Account != nil && targetAccount.ID == authed.Account.ID {
-		mastoAccount, err = p.tc.AccountToMastoSensitive(targetAccount)
-	} else {
-		mastoAccount, err = p.tc.AccountToMastoPublic(targetAccount)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("error converting account: %s", err)
-	}
-	return mastoAccount, nil
 }
 
 func (p *processor) AccountUpdate(authed *oauth.Auth, form *apimodel.UpdateCredentialsRequest) (*apimodel.Account, error) {
-	l := p.log.WithField("func", "AccountUpdate")
+	return p.accountProcessor.Update(authed.Account, form)
-
-	if form.Discoverable != nil {
-		if err := p.db.UpdateOneByID(authed.Account.ID, "discoverable", *form.Discoverable, &gtsmodel.Account{}); err != nil {
-			return nil, fmt.Errorf("error updating discoverable: %s", err)
-		}
-	}
-
-	if form.Bot != nil {
-		if err := p.db.UpdateOneByID(authed.Account.ID, "bot", *form.Bot, &gtsmodel.Account{}); err != nil {
-			return nil, fmt.Errorf("error updating bot: %s", err)
-		}
-	}
-
-	if form.DisplayName != nil {
-		if err := util.ValidateDisplayName(*form.DisplayName); err != nil {
-			return nil, err
-		}
-		if err := p.db.UpdateOneByID(authed.Account.ID, "display_name", *form.DisplayName, &gtsmodel.Account{}); err != nil {
-			return nil, err
-		}
-	}
-
-	if form.Note != nil {
-		if err := util.ValidateNote(*form.Note); err != nil {
-			return nil, err
-		}
-		if err := p.db.UpdateOneByID(authed.Account.ID, "note", *form.Note, &gtsmodel.Account{}); err != nil {
-			return nil, err
-		}
-	}
-
-	if form.Avatar != nil && form.Avatar.Size != 0 {
-		avatarInfo, err := p.updateAccountAvatar(form.Avatar, authed.Account.ID)
-		if err != nil {
-			return nil, err
-		}
-		l.Tracef("new avatar info for account %s is %+v", authed.Account.ID, avatarInfo)
-	}
-
-	if form.Header != nil && form.Header.Size != 0 {
-		headerInfo, err := p.updateAccountHeader(form.Header, authed.Account.ID)
-		if err != nil {
-			return nil, err
-		}
-		l.Tracef("new header info for account %s is %+v", authed.Account.ID, headerInfo)
-	}
-
-	if form.Locked != nil {
-		if err := p.db.UpdateOneByID(authed.Account.ID, "locked", *form.Locked, &gtsmodel.Account{}); err != nil {
-			return nil, err
-		}
-	}
-
-	if form.Source != nil {
-		if form.Source.Language != nil {
-			if err := util.ValidateLanguage(*form.Source.Language); err != nil {
-				return nil, err
-			}
-			if err := p.db.UpdateOneByID(authed.Account.ID, "language", *form.Source.Language, &gtsmodel.Account{}); err != nil {
-				return nil, err
-			}
-		}
-
-		if form.Source.Sensitive != nil {
-			if err := p.db.UpdateOneByID(authed.Account.ID, "locked", *form.Locked, &gtsmodel.Account{}); err != nil {
-				return nil, err
-			}
-		}
-
-		if form.Source.Privacy != nil {
-			if err := util.ValidatePrivacy(*form.Source.Privacy); err != nil {
-				return nil, err
-			}
-			if err := p.db.UpdateOneByID(authed.Account.ID, "privacy", *form.Source.Privacy, &gtsmodel.Account{}); err != nil {
-				return nil, err
-			}
-		}
-	}
-
-	// fetch the account with all updated values set
-	updatedAccount := &gtsmodel.Account{}
-	if err := p.db.GetByID(authed.Account.ID, updatedAccount); err != nil {
-		return nil, fmt.Errorf("could not fetch updated account %s: %s", authed.Account.ID, err)
-	}
-
-	p.fromClientAPI <- gtsmodel.FromClientAPI{
-		APObjectType:   gtsmodel.ActivityStreamsProfile,
-		APActivityType: gtsmodel.ActivityStreamsUpdate,
-		GTSModel:       updatedAccount,
-		OriginAccount:  updatedAccount,
-	}
-
-	acctSensitive, err := p.tc.AccountToMastoSensitive(updatedAccount)
-	if err != nil {
-		return nil, fmt.Errorf("could not convert account into mastosensitive account: %s", err)
-	}
-	return acctSensitive, nil
 }
 
 func (p *processor) AccountStatusesGet(authed *oauth.Auth, targetAccountID string, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) ([]apimodel.Status, gtserror.WithCode) {
-	targetAccount := &gtsmodel.Account{}
+	return p.accountProcessor.StatusesGet(authed.Account, targetAccountID, limit, excludeReplies, maxID, pinned, mediaOnly)
-	if err := p.db.GetByID(targetAccountID, targetAccount); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("no entry found for account id %s", targetAccountID))
-		}
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
-	statuses := []gtsmodel.Status{}
-	apiStatuses := []apimodel.Status{}
-	if err := p.db.GetStatusesByTimeDescending(targetAccountID, &statuses, limit, excludeReplies, maxID, pinned, mediaOnly); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
-			return apiStatuses, nil
-		}
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
-	for _, s := range statuses {
-		visible, err := p.filter.StatusVisible(&s, authed.Account)
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error checking status visibility: %s", err))
-		}
-		if !visible {
-			continue
-		}
-
-		apiStatus, err := p.tc.StatusToMasto(&s, authed.Account)
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting status to masto: %s", err))
-		}
-
-		apiStatuses = append(apiStatuses, *apiStatus)
-	}
-
-	return apiStatuses, nil
 }
 
 func (p *processor) AccountFollowersGet(authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, gtserror.WithCode) {
-	blocked, err := p.db.Blocked(authed.Account.ID, targetAccountID)
+	return p.accountProcessor.FollowersGet(authed.Account, targetAccountID)
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
-	if blocked {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts"))
-	}
-
-	followers := []gtsmodel.Follow{}
-	accounts := []apimodel.Account{}
-	if err := p.db.GetFollowersByAccountID(targetAccountID, &followers, false); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
-			return accounts, nil
-		}
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
-	for _, f := range followers {
-		blocked, err := p.db.Blocked(authed.Account.ID, f.AccountID)
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(err)
-		}
-		if blocked {
-			continue
-		}
-
-		a := &gtsmodel.Account{}
-		if err := p.db.GetByID(f.AccountID, a); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
-				continue
-			}
-			return nil, gtserror.NewErrorInternalError(err)
-		}
-
-		// derefence account fields in case we haven't done it already
-		if err := p.dereferenceAccountFields(a, authed.Account.Username, false); err != nil {
-			// don't bail if we can't fetch them, we'll try another time
-			p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err)
-		}
-
-		account, err := p.tc.AccountToMastoPublic(a)
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(err)
-		}
-		accounts = append(accounts, *account)
-	}
-	return accounts, nil
 }
 
 func (p *processor) AccountFollowingGet(authed *oauth.Auth, targetAccountID string) ([]apimodel.Account, gtserror.WithCode) {
-	blocked, err := p.db.Blocked(authed.Account.ID, targetAccountID)
+	return p.accountProcessor.FollowingGet(authed.Account, targetAccountID)
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
-	if blocked {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts"))
-	}
-
-	following := []gtsmodel.Follow{}
-	accounts := []apimodel.Account{}
-	if err := p.db.GetFollowingByAccountID(targetAccountID, &following); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
-			return accounts, nil
-		}
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
-	for _, f := range following {
-		blocked, err := p.db.Blocked(authed.Account.ID, f.AccountID)
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(err)
-		}
-		if blocked {
-			continue
-		}
-
-		a := &gtsmodel.Account{}
-		if err := p.db.GetByID(f.TargetAccountID, a); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
-				continue
-			}
-			return nil, gtserror.NewErrorInternalError(err)
-		}
-
-		// derefence account fields in case we haven't done it already
-		if err := p.dereferenceAccountFields(a, authed.Account.Username, false); err != nil {
-			// don't bail if we can't fetch them, we'll try another time
-			p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err)
-		}
-
-		account, err := p.tc.AccountToMastoPublic(a)
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(err)
-		}
-		accounts = append(accounts, *account)
-	}
-	return accounts, nil
 }
 
 func (p *processor) AccountRelationshipGet(authed *oauth.Auth, targetAccountID string) (*apimodel.Relationship, gtserror.WithCode) {
-	if authed == nil || authed.Account == nil {
+	return p.accountProcessor.RelationshipGet(authed.Account, targetAccountID)
-		return nil, gtserror.NewErrorForbidden(errors.New("not authed"))
-	}
-
-	gtsR, err := p.db.GetRelationship(authed.Account.ID, targetAccountID)
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error getting relationship: %s", err))
-	}
-
-	r, err := p.tc.RelationshipToMasto(gtsR)
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting relationship: %s", err))
-	}
-
-	return r, nil
 }
 
 func (p *processor) AccountFollowCreate(authed *oauth.Auth, form *apimodel.AccountFollowRequest) (*apimodel.Relationship, gtserror.WithCode) {
-	// if there's a block between the accounts we shouldn't create the request ofc
+	return p.accountProcessor.FollowCreate(authed.Account, form)
-	blocked, err := p.db.Blocked(authed.Account.ID, form.TargetAccountID)
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-	if blocked {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("accountfollowcreate: block exists between accounts"))
-	}
-
-	// make sure the target account actually exists in our db
-	targetAcct := &gtsmodel.Account{}
-	if err := p.db.GetByID(form.TargetAccountID, targetAcct); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("accountfollowcreate: account %s not found in the db: %s", form.TargetAccountID, err))
-		}
-	}
-
-	// check if a follow exists already
-	follows, err := p.db.Follows(authed.Account, targetAcct)
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(fmt.Errorf("accountfollowcreate: error checking follow in db: %s", err))
-	}
-	if follows {
-		// already follows so just return the relationship
-		return p.AccountRelationshipGet(authed, form.TargetAccountID)
-	}
-
-	// check if a follow exists already
-	followRequested, err := p.db.FollowRequested(authed.Account, targetAcct)
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(fmt.Errorf("accountfollowcreate: error checking follow request in db: %s", err))
-	}
-	if followRequested {
-		// already follow requested so just return the relationship
-		return p.AccountRelationshipGet(authed, form.TargetAccountID)
-	}
-
-	// make the follow request
-	newFollowID, err := id.NewRandomULID()
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-
-	fr := &gtsmodel.FollowRequest{
-		ID:              newFollowID,
-		AccountID:       authed.Account.ID,
-		TargetAccountID: form.TargetAccountID,
-		ShowReblogs:     true,
-		URI:             util.GenerateURIForFollow(authed.Account.Username, p.config.Protocol, p.config.Host, newFollowID),
-		Notify:          false,
-	}
-	if form.Reblogs != nil {
-		fr.ShowReblogs = *form.Reblogs
-	}
-	if form.Notify != nil {
-		fr.Notify = *form.Notify
-	}
-
-	// whack it in the database
-	if err := p.db.Put(fr); err != nil {
-		return nil, gtserror.NewErrorInternalError(fmt.Errorf("accountfollowcreate: error creating follow request in db: %s", err))
-	}
-
-	// if it's a local account that's not locked we can just straight up accept the follow request
-	if !targetAcct.Locked && targetAcct.Domain == "" {
-		if _, err := p.db.AcceptFollowRequest(authed.Account.ID, form.TargetAccountID); err != nil {
-			return nil, gtserror.NewErrorInternalError(fmt.Errorf("accountfollowcreate: error accepting folow request for local unlocked account: %s", err))
-		}
-		// return the new relationship
-		return p.AccountRelationshipGet(authed, form.TargetAccountID)
-	}
-
-	// otherwise we leave the follow request as it is and we handle the rest of the process asynchronously
-	p.fromClientAPI <- gtsmodel.FromClientAPI{
-		APObjectType:   gtsmodel.ActivityStreamsFollow,
-		APActivityType: gtsmodel.ActivityStreamsCreate,
-		GTSModel:       fr,
-		OriginAccount:  authed.Account,
-		TargetAccount:  targetAcct,
-	}
-
-	// return whatever relationship results from this
-	return p.AccountRelationshipGet(authed, form.TargetAccountID)
 }
 
 func (p *processor) AccountFollowRemove(authed *oauth.Auth, targetAccountID string) (*apimodel.Relationship, gtserror.WithCode) {
-	// if there's a block between the accounts we shouldn't do anything
+	return p.accountProcessor.FollowRemove(authed.Account, targetAccountID)
-	blocked, err := p.db.Blocked(authed.Account.ID, targetAccountID)
-	if err != nil {
-		return nil, gtserror.NewErrorInternalError(err)
-	}
-	if blocked {
-		return nil, gtserror.NewErrorNotFound(fmt.Errorf("AccountFollowRemove: block exists between accounts"))
-	}
-
-	// make sure the target account actually exists in our db
-	targetAcct := &gtsmodel.Account{}
-	if err := p.db.GetByID(targetAccountID, targetAcct); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
-			return nil, gtserror.NewErrorNotFound(fmt.Errorf("AccountFollowRemove: account %s not found in the db: %s", targetAccountID, err))
-		}
-	}
-
-	// check if a follow request exists, and remove it if it does (storing the URI for later)
-	var frChanged bool
-	var frURI string
-	fr := &gtsmodel.FollowRequest{}
-	if err := p.db.GetWhere([]db.Where{
-		{Key: "account_id", Value: authed.Account.ID},
-		{Key: "target_account_id", Value: targetAccountID},
-	}, fr); err == nil {
-		frURI = fr.URI
-		if err := p.db.DeleteByID(fr.ID, fr); err != nil {
-			return nil, gtserror.NewErrorInternalError(fmt.Errorf("AccountFollowRemove: error removing follow request from db: %s", err))
-		}
-		frChanged = true
-	}
-
-	// now do the same thing for any existing follow
-	var fChanged bool
-	var fURI string
-	f := &gtsmodel.Follow{}
-	if err := p.db.GetWhere([]db.Where{
-		{Key: "account_id", Value: authed.Account.ID},
-		{Key: "target_account_id", Value: targetAccountID},
-	}, f); err == nil {
-		fURI = f.URI
-		if err := p.db.DeleteByID(f.ID, f); err != nil {
-			return nil, gtserror.NewErrorInternalError(fmt.Errorf("AccountFollowRemove: error removing follow from db: %s", err))
-		}
-		fChanged = true
-	}
-
-	// follow request status changed so send the UNDO activity to the channel for async processing
-	if frChanged {
-		p.fromClientAPI <- gtsmodel.FromClientAPI{
-			APObjectType:   gtsmodel.ActivityStreamsFollow,
-			APActivityType: gtsmodel.ActivityStreamsUndo,
-			GTSModel: &gtsmodel.Follow{
-				AccountID:       authed.Account.ID,
-				TargetAccountID: targetAccountID,
-				URI:             frURI,
-			},
-			OriginAccount: authed.Account,
-			TargetAccount: targetAcct,
-		}
-	}
-
-	// follow status changed so send the UNDO activity to the channel for async processing
-	if fChanged {
-		p.fromClientAPI <- gtsmodel.FromClientAPI{
-			APObjectType:   gtsmodel.ActivityStreamsFollow,
-			APActivityType: gtsmodel.ActivityStreamsUndo,
-			GTSModel: &gtsmodel.Follow{
-				AccountID:       authed.Account.ID,
-				TargetAccountID: targetAccountID,
-				URI:             fURI,
-			},
-			OriginAccount: authed.Account,
-			TargetAccount: targetAcct,
-		}
-	}
-
-	// return whatever relationship results from all this
-	return p.AccountRelationshipGet(authed, targetAccountID)
 }

internal/processing/account/account.go

@@ -0,0 +1,96 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"mime/multipart"
+
+	"github.com/sirupsen/logrus"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/internal/visibility"
+	"github.com/superseriousbusiness/oauth2/v4"
+)
+
+// Processor wraps a bunch of functions for processing account actions.
+type Processor interface {
+	// Create processes the given form for creating a new account, returning an oauth token for that account if successful.
+	Create(applicationToken oauth2.TokenInfo, application *gtsmodel.Application, form *apimodel.AccountCreateRequest) (*apimodel.Token, error)
+	// Delete deletes an account, and all of that account's statuses, media, follows, notifications, etc etc etc.
+	Delete(account *gtsmodel.Account) error
+	// Get processes the given request for account information.
+	Get(requestingAccount *gtsmodel.Account, targetAccountID string) (*apimodel.Account, error)
+	// Update processes the update of an account with the given form
+	Update(account *gtsmodel.Account, form *apimodel.UpdateCredentialsRequest) (*apimodel.Account, error)
+	// StatusesGet fetches a number of statuses (in time descending order) from the given account, filtered by visibility for
+	// the account given in authed.
+	StatusesGet(requestingAccount *gtsmodel.Account, targetAccountID string, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) ([]apimodel.Status, gtserror.WithCode)
+	// FollowersGet fetches a list of the target account's followers.
+	FollowersGet(requestingAccount *gtsmodel.Account, targetAccountID string) ([]apimodel.Account, gtserror.WithCode)
+	// FollowingGet fetches a list of the accounts that target account is following.
+	FollowingGet(requestingAccount *gtsmodel.Account, targetAccountID string) ([]apimodel.Account, gtserror.WithCode)
+	// RelationshipGet returns a relationship model describing the relationship of the targetAccount to the Authed account.
+	RelationshipGet(requestingAccount *gtsmodel.Account, targetAccountID string) (*apimodel.Relationship, gtserror.WithCode)
+	// FollowCreate handles a follow request to an account, either remote or local.
+	FollowCreate(requestingAccount *gtsmodel.Account, form *apimodel.AccountFollowRequest) (*apimodel.Relationship, gtserror.WithCode)
+	// FollowRemove handles the removal of a follow/follow request to an account, either remote or local.
+	FollowRemove(requestingAccount *gtsmodel.Account, targetAccountID string) (*apimodel.Relationship, gtserror.WithCode)
+	// UpdateHeader does the dirty work of checking the header part of an account update form,
+	// parsing and checking the image, and doing the necessary updates in the database for this to become
+	// the account's new header image.
+	UpdateAvatar(avatar *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error)
+	// UpdateAvatar does the dirty work of checking the avatar part of an account update form,
+	// parsing and checking the image, and doing the necessary updates in the database for this to become
+	// the account's new avatar image.
+	UpdateHeader(header *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error)
+}
+
+type processor struct {
+	tc            typeutils.TypeConverter
+	config        *config.Config
+	mediaHandler  media.Handler
+	fromClientAPI chan gtsmodel.FromClientAPI
+	oauthServer   oauth.Server
+	filter        visibility.Filter
+	db            db.DB
+	federator     federation.Federator
+	log           *logrus.Logger
+}
+
+// New returns a new account processor.
+func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, oauthServer oauth.Server, fromClientAPI chan gtsmodel.FromClientAPI, federator federation.Federator, config *config.Config, log *logrus.Logger) Processor {
+	return &processor{
+		tc:            tc,
+		config:        config,
+		mediaHandler:  mediaHandler,
+		fromClientAPI: fromClientAPI,
+		oauthServer:   oauthServer,
+		filter:        visibility.NewFilter(db, log),
+		db:            db,
+		federator:     federator,
+		log:           log,
+	}
+}

internal/processing/account/create.go

@@ -0,0 +1,64 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/oauth2/v4"
+)
+
+func (p *processor) Create(applicationToken oauth2.TokenInfo, application *gtsmodel.Application, form *apimodel.AccountCreateRequest) (*apimodel.Token, error) {
+	l := p.log.WithField("func", "accountCreate")
+
+	if err := p.db.IsEmailAvailable(form.Email); err != nil {
+		return nil, err
+	}
+
+	if err := p.db.IsUsernameAvailable(form.Username); err != nil {
+		return nil, err
+	}
+
+	// don't store a reason if we don't require one
+	reason := form.Reason
+	if !p.config.AccountsConfig.ReasonRequired {
+		reason = ""
+	}
+
+	l.Trace("creating new username and account")
+	user, err := p.db.NewSignup(form.Username, reason, p.config.AccountsConfig.RequireApproval, form.Email, form.Password, form.IP, form.Locale, application.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error creating new signup in the database: %s", err)
+	}
+
+	l.Tracef("generating a token for user %s with account %s and application %s", user.ID, user.AccountID, application.ID)
+	accessToken, err := p.oauthServer.GenerateUserAccessToken(applicationToken, application.ClientSecret, user.ID)
+	if err != nil {
+		return nil, fmt.Errorf("error creating new access token for user %s: %s", user.ID, err)
+	}
+
+	return &apimodel.Token{
+		AccessToken: accessToken.GetAccess(),
+		TokenType:   "Bearer",
+		Scope:       accessToken.GetScope(),
+		CreatedAt:   accessToken.GetAccessCreateAt().Unix(),
+	}, nil
+}

internal/processing/account/createfollow.go

@@ -0,0 +1,116 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (p *processor) FollowCreate(requestingAccount *gtsmodel.Account, form *apimodel.AccountFollowRequest) (*apimodel.Relationship, gtserror.WithCode) {
+	// if there's a block between the accounts we shouldn't create the request ofc
+	blocked, err := p.db.Blocked(requestingAccount.ID, form.TargetAccountID)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+	if blocked {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("accountfollowcreate: block exists between accounts"))
+	}
+
+	// make sure the target account actually exists in our db
+	targetAcct := &gtsmodel.Account{}
+	if err := p.db.GetByID(form.TargetAccountID, targetAcct); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("accountfollowcreate: account %s not found in the db: %s", form.TargetAccountID, err))
+		}
+	}
+
+	// check if a follow exists already
+	follows, err := p.db.Follows(requestingAccount, targetAcct)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("accountfollowcreate: error checking follow in db: %s", err))
+	}
+	if follows {
+		// already follows so just return the relationship
+		return p.RelationshipGet(requestingAccount, form.TargetAccountID)
+	}
+
+	// check if a follow exists already
+	followRequested, err := p.db.FollowRequested(requestingAccount, targetAcct)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("accountfollowcreate: error checking follow request in db: %s", err))
+	}
+	if followRequested {
+		// already follow requested so just return the relationship
+		return p.RelationshipGet(requestingAccount, form.TargetAccountID)
+	}
+
+	// make the follow request
+	newFollowID, err := id.NewRandomULID()
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	fr := &gtsmodel.FollowRequest{
+		ID:              newFollowID,
+		AccountID:       requestingAccount.ID,
+		TargetAccountID: form.TargetAccountID,
+		ShowReblogs:     true,
+		URI:             util.GenerateURIForFollow(requestingAccount.Username, p.config.Protocol, p.config.Host, newFollowID),
+		Notify:          false,
+	}
+	if form.Reblogs != nil {
+		fr.ShowReblogs = *form.Reblogs
+	}
+	if form.Notify != nil {
+		fr.Notify = *form.Notify
+	}
+
+	// whack it in the database
+	if err := p.db.Put(fr); err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("accountfollowcreate: error creating follow request in db: %s", err))
+	}
+
+	// if it's a local account that's not locked we can just straight up accept the follow request
+	if !targetAcct.Locked && targetAcct.Domain == "" {
+		if _, err := p.db.AcceptFollowRequest(requestingAccount.ID, form.TargetAccountID); err != nil {
+			return nil, gtserror.NewErrorInternalError(fmt.Errorf("accountfollowcreate: error accepting folow request for local unlocked account: %s", err))
+		}
+		// return the new relationship
+		return p.RelationshipGet(requestingAccount, form.TargetAccountID)
+	}
+
+	// otherwise we leave the follow request as it is and we handle the rest of the process asynchronously
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
+		APObjectType:   gtsmodel.ActivityStreamsFollow,
+		APActivityType: gtsmodel.ActivityStreamsCreate,
+		GTSModel:       fr,
+		OriginAccount:  requestingAccount,
+		TargetAccount:  targetAcct,
+	}
+
+	// return whatever relationship results from this
+	return p.RelationshipGet(requestingAccount, form.TargetAccountID)
+}

internal/processing/account/delete.go

@@ -0,0 +1,25 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+
+func (p *processor) Delete(account *gtsmodel.Account) error {
+   return nil
+}

internal/processing/account/get.go

@@ -0,0 +1,59 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"errors"
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) Get(requestingAccount *gtsmodel.Account, targetAccountID string) (*apimodel.Account, error) {
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetAccountID, targetAccount); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, errors.New("account not found")
+		}
+		return nil, fmt.Errorf("db error: %s", err)
+	}
+
+		// lazily dereference things on the account if it hasn't been done yet
+		var requestingUsername string
+		if requestingAccount != nil {
+			requestingUsername = requestingAccount.Username
+		}
+		if err := p.federator.DereferenceAccountFields(targetAccount, requestingUsername, false); err != nil {
+			p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err)
+		}
+
+	var mastoAccount *apimodel.Account
+	var err error
+	if requestingAccount != nil && targetAccount.ID == requestingAccount.ID {
+		mastoAccount, err = p.tc.AccountToMastoSensitive(targetAccount)
+	} else {
+		mastoAccount, err = p.tc.AccountToMastoPublic(targetAccount)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error converting account: %s", err)
+	}
+	return mastoAccount, nil
+}

internal/processing/account/getfollowers.go

@@ -0,0 +1,79 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) FollowersGet(requestingAccount *gtsmodel.Account, targetAccountID string) ([]apimodel.Account, gtserror.WithCode) {
+	blocked, err := p.db.Blocked(requestingAccount.ID, targetAccountID)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts"))
+	}
+
+	followers := []gtsmodel.Follow{}
+	accounts := []apimodel.Account{}
+	if err := p.db.GetFollowersByAccountID(targetAccountID, &followers, false); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return accounts, nil
+		}
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	for _, f := range followers {
+		blocked, err := p.db.Blocked(requestingAccount.ID, f.AccountID)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+		if blocked {
+			continue
+		}
+
+		a := &gtsmodel.Account{}
+		if err := p.db.GetByID(f.AccountID, a); err != nil {
+			if _, ok := err.(db.ErrNoEntries); ok {
+				continue
+			}
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+
+		// derefence account fields in case we haven't done it already
+		if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil {
+			// don't bail if we can't fetch them, we'll try another time
+			p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err)
+		}
+
+		account, err := p.tc.AccountToMastoPublic(a)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+		accounts = append(accounts, *account)
+	}
+	return accounts, nil
+}

internal/processing/account/getfollowing.go

@@ -0,0 +1,79 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) FollowingGet(requestingAccount *gtsmodel.Account, targetAccountID string) ([]apimodel.Account, gtserror.WithCode) {
+	blocked, err := p.db.Blocked(requestingAccount.ID, targetAccountID)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts"))
+	}
+
+	following := []gtsmodel.Follow{}
+	accounts := []apimodel.Account{}
+	if err := p.db.GetFollowingByAccountID(targetAccountID, &following); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return accounts, nil
+		}
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	for _, f := range following {
+		blocked, err := p.db.Blocked(requestingAccount.ID, f.AccountID)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+		if blocked {
+			continue
+		}
+
+		a := &gtsmodel.Account{}
+		if err := p.db.GetByID(f.TargetAccountID, a); err != nil {
+			if _, ok := err.(db.ErrNoEntries); ok {
+				continue
+			}
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+
+		// derefence account fields in case we haven't done it already
+		if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil {
+			// don't bail if we can't fetch them, we'll try another time
+			p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err)
+		}
+
+		account, err := p.tc.AccountToMastoPublic(a)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+		accounts = append(accounts, *account)
+	}
+	return accounts, nil
+}

internal/processing/account/getrelationship.go

@@ -0,0 +1,46 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"errors"
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) RelationshipGet(requestingAccount *gtsmodel.Account, targetAccountID string) (*apimodel.Relationship, gtserror.WithCode) {
+	if requestingAccount == nil {
+		return nil, gtserror.NewErrorForbidden(errors.New("not authed"))
+	}
+
+	gtsR, err := p.db.GetRelationship(requestingAccount.ID, targetAccountID)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error getting relationship: %s", err))
+	}
+
+	r, err := p.tc.RelationshipToMasto(gtsR)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting relationship: %s", err))
+	}
+
+	return r, nil
+}

internal/processing/account/getstatuses.go

@@ -0,0 +1,66 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) StatusesGet(requestingAccount *gtsmodel.Account, targetAccountID string, limit int, excludeReplies bool, maxID string, pinned bool, mediaOnly bool) ([]apimodel.Status, gtserror.WithCode) {
+	targetAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetAccountID, targetAccount); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("no entry found for account id %s", targetAccountID))
+		}
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	statuses := []gtsmodel.Status{}
+	apiStatuses := []apimodel.Status{}
+	if err := p.db.GetStatusesByTimeDescending(targetAccountID, &statuses, limit, excludeReplies, maxID, pinned, mediaOnly); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return apiStatuses, nil
+		}
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	for _, s := range statuses {
+		visible, err := p.filter.StatusVisible(&s, requestingAccount)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error checking status visibility: %s", err))
+		}
+		if !visible {
+			continue
+		}
+
+		apiStatus, err := p.tc.StatusToMasto(&s, requestingAccount)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(fmt.Errorf("error converting status to masto: %s", err))
+		}
+
+		apiStatuses = append(apiStatuses, *apiStatus)
+	}
+
+	return apiStatuses, nil
+}

internal/processing/account/removefollow.go

@@ -0,0 +1,110 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (p *processor) FollowRemove(requestingAccount *gtsmodel.Account, targetAccountID string) (*apimodel.Relationship, gtserror.WithCode) {
+	// if there's a block between the accounts we shouldn't do anything
+	blocked, err := p.db.Blocked(requestingAccount.ID, targetAccountID)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+	if blocked {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("AccountFollowRemove: block exists between accounts"))
+	}
+
+	// make sure the target account actually exists in our db
+	targetAcct := &gtsmodel.Account{}
+	if err := p.db.GetByID(targetAccountID, targetAcct); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil, gtserror.NewErrorNotFound(fmt.Errorf("AccountFollowRemove: account %s not found in the db: %s", targetAccountID, err))
+		}
+	}
+
+	// check if a follow request exists, and remove it if it does (storing the URI for later)
+	var frChanged bool
+	var frURI string
+	fr := &gtsmodel.FollowRequest{}
+	if err := p.db.GetWhere([]db.Where{
+		{Key: "account_id", Value: requestingAccount.ID},
+		{Key: "target_account_id", Value: targetAccountID},
+	}, fr); err == nil {
+		frURI = fr.URI
+		if err := p.db.DeleteByID(fr.ID, fr); err != nil {
+			return nil, gtserror.NewErrorInternalError(fmt.Errorf("AccountFollowRemove: error removing follow request from db: %s", err))
+		}
+		frChanged = true
+	}
+
+	// now do the same thing for any existing follow
+	var fChanged bool
+	var fURI string
+	f := &gtsmodel.Follow{}
+	if err := p.db.GetWhere([]db.Where{
+		{Key: "account_id", Value: requestingAccount.ID},
+		{Key: "target_account_id", Value: targetAccountID},
+	}, f); err == nil {
+		fURI = f.URI
+		if err := p.db.DeleteByID(f.ID, f); err != nil {
+			return nil, gtserror.NewErrorInternalError(fmt.Errorf("AccountFollowRemove: error removing follow from db: %s", err))
+		}
+		fChanged = true
+	}
+
+	// follow request status changed so send the UNDO activity to the channel for async processing
+	if frChanged {
+		p.fromClientAPI <- gtsmodel.FromClientAPI{
+			APObjectType:   gtsmodel.ActivityStreamsFollow,
+			APActivityType: gtsmodel.ActivityStreamsUndo,
+			GTSModel: &gtsmodel.Follow{
+				AccountID:       requestingAccount.ID,
+				TargetAccountID: targetAccountID,
+				URI:             frURI,
+			},
+			OriginAccount: requestingAccount,
+			TargetAccount: targetAcct,
+		}
+	}
+
+	// follow status changed so send the UNDO activity to the channel for async processing
+	if fChanged {
+		p.fromClientAPI <- gtsmodel.FromClientAPI{
+			APObjectType:   gtsmodel.ActivityStreamsFollow,
+			APActivityType: gtsmodel.ActivityStreamsUndo,
+			GTSModel: &gtsmodel.Follow{
+				AccountID:       requestingAccount.ID,
+				TargetAccountID: targetAccountID,
+				URI:             fURI,
+			},
+			OriginAccount: requestingAccount,
+			TargetAccount: targetAcct,
+		}
+	}
+
+	// return whatever relationship results from all this
+	return p.RelationshipGet(requestingAccount, targetAccountID)
+}

internal/processing/account/update.go

@@ -0,0 +1,199 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package account
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (p *processor) Update(account *gtsmodel.Account, form *apimodel.UpdateCredentialsRequest) (*apimodel.Account, error) {
+	l := p.log.WithField("func", "AccountUpdate")
+
+	if form.Discoverable != nil {
+		if err := p.db.UpdateOneByID(account.ID, "discoverable", *form.Discoverable, &gtsmodel.Account{}); err != nil {
+			return nil, fmt.Errorf("error updating discoverable: %s", err)
+		}
+	}
+
+	if form.Bot != nil {
+		if err := p.db.UpdateOneByID(account.ID, "bot", *form.Bot, &gtsmodel.Account{}); err != nil {
+			return nil, fmt.Errorf("error updating bot: %s", err)
+		}
+	}
+
+	if form.DisplayName != nil {
+		if err := util.ValidateDisplayName(*form.DisplayName); err != nil {
+			return nil, err
+		}
+		if err := p.db.UpdateOneByID(account.ID, "display_name", *form.DisplayName, &gtsmodel.Account{}); err != nil {
+			return nil, err
+		}
+	}
+
+	if form.Note != nil {
+		if err := util.ValidateNote(*form.Note); err != nil {
+			return nil, err
+		}
+		if err := p.db.UpdateOneByID(account.ID, "note", *form.Note, &gtsmodel.Account{}); err != nil {
+			return nil, err
+		}
+	}
+
+	if form.Avatar != nil && form.Avatar.Size != 0 {
+		avatarInfo, err := p.UpdateAvatar(form.Avatar, account.ID)
+		if err != nil {
+			return nil, err
+		}
+		l.Tracef("new avatar info for account %s is %+v", account.ID, avatarInfo)
+	}
+
+	if form.Header != nil && form.Header.Size != 0 {
+		headerInfo, err := p.UpdateHeader(form.Header, account.ID)
+		if err != nil {
+			return nil, err
+		}
+		l.Tracef("new header info for account %s is %+v", account.ID, headerInfo)
+	}
+
+	if form.Locked != nil {
+		if err := p.db.UpdateOneByID(account.ID, "locked", *form.Locked, &gtsmodel.Account{}); err != nil {
+			return nil, err
+		}
+	}
+
+	if form.Source != nil {
+		if form.Source.Language != nil {
+			if err := util.ValidateLanguage(*form.Source.Language); err != nil {
+				return nil, err
+			}
+			if err := p.db.UpdateOneByID(account.ID, "language", *form.Source.Language, &gtsmodel.Account{}); err != nil {
+				return nil, err
+			}
+		}
+
+		if form.Source.Sensitive != nil {
+			if err := p.db.UpdateOneByID(account.ID, "locked", *form.Locked, &gtsmodel.Account{}); err != nil {
+				return nil, err
+			}
+		}
+
+		if form.Source.Privacy != nil {
+			if err := util.ValidatePrivacy(*form.Source.Privacy); err != nil {
+				return nil, err
+			}
+			if err := p.db.UpdateOneByID(account.ID, "privacy", *form.Source.Privacy, &gtsmodel.Account{}); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// fetch the account with all updated values set
+	updatedAccount := &gtsmodel.Account{}
+	if err := p.db.GetByID(account.ID, updatedAccount); err != nil {
+		return nil, fmt.Errorf("could not fetch updated account %s: %s", account.ID, err)
+	}
+
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
+		APObjectType:   gtsmodel.ActivityStreamsProfile,
+		APActivityType: gtsmodel.ActivityStreamsUpdate,
+		GTSModel:       updatedAccount,
+		OriginAccount:  updatedAccount,
+	}
+
+	acctSensitive, err := p.tc.AccountToMastoSensitive(updatedAccount)
+	if err != nil {
+		return nil, fmt.Errorf("could not convert account into mastosensitive account: %s", err)
+	}
+	return acctSensitive, nil
+}
+
+// UpdateAvatar does the dirty work of checking the avatar part of an account update form,
+// parsing and checking the image, and doing the necessary updates in the database for this to become
+// the account's new avatar image.
+func (p *processor) UpdateAvatar(avatar *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) {
+	var err error
+	if int(avatar.Size) > p.config.MediaConfig.MaxImageSize {
+		err = fmt.Errorf("avatar with size %d exceeded max image size of %d bytes", avatar.Size, p.config.MediaConfig.MaxImageSize)
+		return nil, err
+	}
+	f, err := avatar.Open()
+	if err != nil {
+		return nil, fmt.Errorf("could not read provided avatar: %s", err)
+	}
+
+	// extract the bytes
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		return nil, fmt.Errorf("could not read provided avatar: %s", err)
+	}
+	if size == 0 {
+		return nil, errors.New("could not read provided avatar: size 0 bytes")
+	}
+
+	// do the setting
+	avatarInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(buf.Bytes(), accountID, media.Avatar, "")
+	if err != nil {
+		return nil, fmt.Errorf("error processing avatar: %s", err)
+	}
+
+	return avatarInfo, f.Close()
+}
+
+// UpdateHeader does the dirty work of checking the header part of an account update form,
+// parsing and checking the image, and doing the necessary updates in the database for this to become
+// the account's new header image.
+func (p *processor) UpdateHeader(header *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) {
+	var err error
+	if int(header.Size) > p.config.MediaConfig.MaxImageSize {
+		err = fmt.Errorf("header with size %d exceeded max image size of %d bytes", header.Size, p.config.MediaConfig.MaxImageSize)
+		return nil, err
+	}
+	f, err := header.Open()
+	if err != nil {
+		return nil, fmt.Errorf("could not read provided header: %s", err)
+	}
+
+	// extract the bytes
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		return nil, fmt.Errorf("could not read provided header: %s", err)
+	}
+	if size == 0 {
+		return nil, errors.New("could not read provided header: size 0 bytes")
+	}
+
+	// do the setting
+	headerInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(buf.Bytes(), accountID, media.Header, "")
+	if err != nil {
+		return nil, fmt.Errorf("error processing header: %s", err)
+	}
+
+	return headerInfo, f.Close()
+}

internal/processing/admin/admin.go

@@ -39,17 +39,19 @@ type processor struct {
 	tc            typeutils.TypeConverter
 	config        *config.Config
 	mediaHandler  media.Handler
+	fromClientAPI chan gtsmodel.FromClientAPI
 	db            db.DB
 	log           *logrus.Logger
 }
 
 // New returns a new admin processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, config *config.Config, log *logrus.Logger) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaHandler media.Handler, fromClientAPI chan gtsmodel.FromClientAPI, config *config.Config, log *logrus.Logger) Processor {
 	return &processor{
-		tc:            tc,
+		tc:           tc,
-		config:        config,
+		config:       config,
-		mediaHandler:  mediaHandler,
+		mediaHandler: mediaHandler,
-		db:            db,
+		fromClientAPI: fromClientAPI,
-		log:           log,
+		db:           db,
+		log:          log,
 	}
 }

internal/processing/admin/domainblock.go

@@ -109,6 +109,6 @@ func (p *processor) domainBlockProcessSideEffects(block *gtsmodel.DomainBlock) {
 		l.Errorf("domainBlockProcessSideEffects: db error removing instance account: %s", err)
 	}
 
-	aaaaaaaaa
 	// TODO: delete accounts through the normal account deletion system (which should also delete media + posts + remove posts from timelines)
+	
 }

internal/processing/fromfederator.go

@@ -21,7 +21,6 @@ package processing
 import (
 	"errors"
 	"fmt"
-	"net/url"
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -49,7 +48,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming status")
-			if err := p.dereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil {
+			if err := p.federator.DereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil {
 				return fmt.Errorf("error dereferencing status from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil {
@@ -72,7 +71,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming account")
-			if err := p.dereferenceAccountFields(incomingAccount, "", false); err != nil {
+			if err := p.federator.DereferenceAccountFields(incomingAccount, "", false); err != nil {
 				return fmt.Errorf("error dereferencing account from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
@@ -105,7 +104,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 				return errors.New("announce was not parseable as *gtsmodel.Status")
 			}
 
-			if err := p.dereferenceAnnounce(incomingAnnounce, federatorMsg.ReceivingAccount.Username); err != nil {
+			if err := p.federator.DereferenceAnnounce(incomingAnnounce, federatorMsg.ReceivingAccount.Username); err != nil {
 				return fmt.Errorf("error dereferencing announce from federator: %s", err)
 			}
 
@@ -140,7 +139,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming account")
-			if err := p.dereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil {
+			if err := p.federator.DereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil {
 				return fmt.Errorf("error dereferencing account from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
@@ -183,299 +182,3 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 
 	return nil
 }
-
-// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming
-// federated status, back in the federating db's Create function.
-//
-// When a status comes in from the federation API, there are certain fields that
-// haven't been dereferenced yet, because we needed to provide a snappy synchronous
-// response to the caller. By the time it reaches this function though, it's being
-// processed asynchronously, so we have all the time in the world to fetch the various
-// bits and bobs that are attached to the status, and properly flesh it out, before we
-// send the status to any timelines and notify people.
-//
-// Things to dereference and fetch here:
-//
-// 1. Media attachments.
-// 2. Hashtags.
-// 3. Emojis.
-// 4. Mentions.
-// 5. Posting account.
-// 6. Replied-to-status.
-//
-// SIDE EFFECTS:
-// This function will deference all of the above, insert them in the database as necessary,
-// and attach them to the status. The status itself will not be added to the database yet,
-// that's up the caller to do.
-func (p *processor) dereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error {
-	l := p.log.WithFields(logrus.Fields{
-		"func":   "dereferenceStatusFields",
-		"status": fmt.Sprintf("%+v", status),
-	})
-	l.Debug("entering function")
-
-	t, err := p.federator.GetTransportForUser(requestingUsername)
-	if err != nil {
-		return fmt.Errorf("error creating transport: %s", err)
-	}
-
-	// the status should have an ID by now, but just in case it doesn't let's generate one here
-	// because we'll need it further down
-	if status.ID == "" {
-		newID, err := id.NewULIDFromTime(status.CreatedAt)
-		if err != nil {
-			return err
-		}
-		status.ID = newID
-	}
-
-	// 1. Media attachments.
-	//
-	// At this point we should know:
-	// * the media type of the file we're looking for (a.File.ContentType)
-	// * the blurhash (a.Blurhash)
-	// * the file type (a.Type)
-	// * the remote URL (a.RemoteURL)
-	// This should be enough to pass along to the media processor.
-	attachmentIDs := []string{}
-	for _, a := range status.GTSMediaAttachments {
-		l.Debugf("dereferencing attachment: %+v", a)
-
-		// it might have been processed elsewhere so check first if it's already in the database or not
-		maybeAttachment := &gtsmodel.MediaAttachment{}
-		err := p.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
-		if err == nil {
-			// we already have it in the db, dereferenced, no need to do it again
-			l.Debugf("attachment already exists with id %s", maybeAttachment.ID)
-			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
-			continue
-		}
-		if _, ok := err.(db.ErrNoEntries); !ok {
-			// we have a real error
-			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		// it just doesn't exist yet so carry on
-		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
-		deferencedAttachment, err := p.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
-		if err != nil {
-			p.log.Errorf("error dereferencing status attachment: %s", err)
-			continue
-		}
-		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
-		deferencedAttachment.StatusID = status.ID
-		deferencedAttachment.Description = a.Description
-		if err := p.db.Put(deferencedAttachment); err != nil {
-			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
-	}
-	status.Attachments = attachmentIDs
-
-	// 2. Hashtags
-
-	// 3. Emojis
-
-	// 4. Mentions
-	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
-	//
-	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
-	mentions := []string{}
-	for _, m := range status.GTSMentions {
-		if m.ID == "" {
-			mID, err := id.NewRandomULID()
-			if err != nil {
-				return err
-			}
-			m.ID = mID
-		}
-
-		uri, err := url.Parse(m.MentionedAccountURI)
-		if err != nil {
-			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
-			continue
-		}
-
-		m.StatusID = status.ID
-		m.OriginAccountID = status.GTSAuthorAccount.ID
-		m.OriginAccountURI = status.GTSAuthorAccount.URI
-
-		targetAccount := &gtsmodel.Account{}
-		if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil {
-			// proper error
-			if _, ok := err.(db.ErrNoEntries); !ok {
-				return fmt.Errorf("db error checking for account with uri %s", uri.String())
-			}
-
-			// we just don't have it yet, so we should go get it....
-			accountable, err := p.federator.DereferenceRemoteAccount(requestingUsername, uri)
-			if err != nil {
-				// we can't dereference it so just skip it
-				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
-				continue
-			}
-
-			targetAccount, err = p.tc.ASRepresentationToAccount(accountable, false)
-			if err != nil {
-				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
-				continue
-			}
-
-			targetAccountID, err := id.NewRandomULID()
-			if err != nil {
-				return err
-			}
-			targetAccount.ID = targetAccountID
-
-			if err := p.db.Put(targetAccount); err != nil {
-				return fmt.Errorf("db error inserting account with uri %s", uri.String())
-			}
-		}
-
-		// by this point, we know the targetAccount exists in our database with an ID :)
-		m.TargetAccountID = targetAccount.ID
-		if err := p.db.Put(m); err != nil {
-			return fmt.Errorf("error creating mention: %s", err)
-		}
-		mentions = append(mentions, m.ID)
-	}
-	status.Mentions = mentions
-
-	return nil
-}
-
-func (p *processor) dereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
-	l := p.log.WithFields(logrus.Fields{
-		"func":               "dereferenceAccountFields",
-		"requestingUsername": requestingUsername,
-	})
-
-	t, err := p.federator.GetTransportForUser(requestingUsername)
-	if err != nil {
-		return fmt.Errorf("error getting transport for user: %s", err)
-	}
-
-	// fetch the header and avatar
-	if err := p.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
-		// if this doesn't work, just skip it -- we can do it later
-		l.Debugf("error fetching header/avi for account: %s", err)
-	}
-
-	if err := p.db.UpdateByID(account.ID, account); err != nil {
-		return fmt.Errorf("error updating account in database: %s", err)
-	}
-
-	return nil
-}
-
-func (p *processor) dereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
-	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
-		// we can't do anything unfortunately
-		return errors.New("dereferenceAnnounce: no URI to dereference")
-	}
-
-	// check if we already have the boosted status in the database
-	boostedStatus := &gtsmodel.Status{}
-	err := p.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus)
-	if err == nil {
-		// nice, we already have it so we don't actually need to dereference it from remote
-		announce.Content = boostedStatus.Content
-		announce.ContentWarning = boostedStatus.ContentWarning
-		announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
-		announce.Sensitive = boostedStatus.Sensitive
-		announce.Language = boostedStatus.Language
-		announce.Text = boostedStatus.Text
-		announce.BoostOfID = boostedStatus.ID
-		announce.Visibility = boostedStatus.Visibility
-		announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
-		announce.GTSBoostedStatus = boostedStatus
-		return nil
-	}
-
-	// we don't have it so we need to dereference it
-	remoteStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error parsing url %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	statusable, err := p.federator.DereferenceRemoteStatus(requestingUsername, remoteStatusURI)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// make sure we have the author account in the db
-	attributedToProp := statusable.GetActivityStreamsAttributedTo()
-	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
-		accountURI := iter.GetIRI()
-		if accountURI == nil {
-			continue
-		}
-
-		if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, &gtsmodel.Account{}); err == nil {
-			// we already have it, fine
-			continue
-		}
-
-		// we don't have the boosted status author account yet so dereference it
-		accountable, err := p.federator.DereferenceRemoteAccount(requestingUsername, accountURI)
-		if err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err)
-		}
-		account, err := p.tc.ASRepresentationToAccount(accountable, false)
-		if err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err)
-		}
-
-		accountID, err := id.NewRandomULID()
-		if err != nil {
-			return err
-		}
-		account.ID = accountID
-
-		if err := p.db.Put(account); err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err)
-		}
-
-		if err := p.dereferenceAccountFields(account, requestingUsername, false); err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err)
-		}
-	}
-
-	// now convert the statusable into something we can understand
-	boostedStatus, err = p.tc.ASStatusToStatus(statusable)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt)
-	if err != nil {
-		return nil
-	}
-	boostedStatus.ID = boostedStatusID
-
-	if err := p.db.Put(boostedStatus); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// now dereference additional fields straight away (we're already async here so we have time)
-	if err := p.dereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// update with the newly dereferenced fields
-	if err := p.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err)
-	}
-
-	// we have everything we need!
-	announce.Content = boostedStatus.Content
-	announce.ContentWarning = boostedStatus.ContentWarning
-	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
-	announce.Sensitive = boostedStatus.Sensitive
-	announce.Language = boostedStatus.Language
-	announce.Text = boostedStatus.Text
-	announce.BoostOfID = boostedStatus.ID
-	announce.Visibility = boostedStatus.Visibility
-	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
-	announce.GTSBoostedStatus = boostedStatus
-	return nil
-}

internal/processing/instance.go

@@ -131,7 +131,7 @@ func (p *processor) InstancePatch(form *apimodel.InstanceSettingsUpdateRequest)
 
 	// process avatar if provided
 	if form.Avatar != nil && form.Avatar.Size != 0 {
-		_, err := p.updateAccountAvatar(form.Avatar, ia.ID)
+		_, err := p.accountProcessor.UpdateAvatar(form.Avatar, ia.ID)
 		if err != nil {
 			return nil, gtserror.NewErrorBadRequest(err, "error processing avatar")
 		}
@@ -139,7 +139,7 @@ func (p *processor) InstancePatch(form *apimodel.InstanceSettingsUpdateRequest)
 
 	// process header if provided
 	if form.Header != nil && form.Header.Size != 0 {
-		_, err := p.updateAccountHeader(form.Header, ia.ID)
+		_, err := p.accountProcessor.UpdateHeader(form.Header, ia.ID)
 		if err != nil {
 			return nil, gtserror.NewErrorBadRequest(err, "error processing header")
 		}

internal/processing/processor.go

@@ -32,6 +32,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/processing/account"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/status"
 	"github.com/superseriousbusiness/gotosocial/internal/processing/streaming"
@@ -213,6 +214,7 @@ type processor struct {
 		SUB-PROCESSORS
 	*/
 
+	accountProcessor   account.Processor
 	adminProcessor     admin.Processor
 	statusProcessor    status.Processor
 	streamingProcessor streaming.Processor
@@ -226,7 +228,8 @@ func NewProcessor(config *config.Config, tc typeutils.TypeConverter, federator f
 
 	statusProcessor := status.New(db, tc, config, fromClientAPI, log)
 	streamingProcessor := streaming.New(db, tc, oauthServer, config, log)
-	adminProcessor := admin.New(db, tc, mediaHandler, config, log)
+	accountProcessor := account.New(db, tc, mediaHandler, oauthServer, fromClientAPI, federator, config, log)
+	adminProcessor := admin.New(db, tc, mediaHandler, fromClientAPI, config, log)
 
 	return &processor{
 		fromClientAPI:   fromClientAPI,
@@ -243,6 +246,7 @@ func NewProcessor(config *config.Config, tc typeutils.TypeConverter, federator f
 		db:              db,
 		filter:          visibility.NewFilter(db, log),
 
+		accountProcessor:   accountProcessor,
 		adminProcessor:     adminProcessor,
 		statusProcessor:    statusProcessor,
 		streamingProcessor: streamingProcessor,

internal/processing/search.go

@@ -176,7 +176,7 @@ func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve
 			}
 
 			// properly dereference everything in the status (media attachments etc)
-			if err := p.dereferenceStatusFields(status, authed.Account.Username); err != nil {
+			if err := p.federator.DereferenceStatusFields(status, authed.Account.Username); err != nil {
 				return nil, fmt.Errorf("error dereferencing status fields: %s", err)
 			}
 
@@ -223,7 +223,7 @@ func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve
 			return nil, fmt.Errorf("searchAccountByURI: error inserting account with uri %s: %s", uri.String(), err)
 		}
 
-		if err := p.dereferenceAccountFields(account, authed.Account.Username, false); err != nil {
+		if err := p.federator.DereferenceAccountFields(account, authed.Account.Username, false); err != nil {
 			return nil, fmt.Errorf("searchAccountByURI: error further dereferencing account with uri %s: %s", uri.String(), err)
 		}
 
@@ -301,7 +301,7 @@ func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, r
 		}
 
 		// properly dereference all the fields on the account immediately
-		if err := p.dereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil {
+		if err := p.federator.DereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil {
 			return nil, fmt.Errorf("searchAccountByMention: error dereferencing fields on account with uri %s: %s", acctURI.String(), err)
 		}
 	}

internal/processing/util.go

@@ -1,135 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package processing
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"mime/multipart"
-
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/media"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
-)
-
-/*
-	HELPER FUNCTIONS
-*/
-
-// TODO: try to combine the below two functions because this is a lot of code repetition.
-
-// updateAccountAvatar does the dirty work of checking the avatar part of an account update form,
-// parsing and checking the image, and doing the necessary updates in the database for this to become
-// the account's new avatar image.
-func (p *processor) updateAccountAvatar(avatar *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) {
-	var err error
-	if int(avatar.Size) > p.config.MediaConfig.MaxImageSize {
-		err = fmt.Errorf("avatar with size %d exceeded max image size of %d bytes", avatar.Size, p.config.MediaConfig.MaxImageSize)
-		return nil, err
-	}
-	f, err := avatar.Open()
-	if err != nil {
-		return nil, fmt.Errorf("could not read provided avatar: %s", err)
-	}
-
-	// extract the bytes
-	buf := new(bytes.Buffer)
-	size, err := io.Copy(buf, f)
-	if err != nil {
-		return nil, fmt.Errorf("could not read provided avatar: %s", err)
-	}
-	if size == 0 {
-		return nil, errors.New("could not read provided avatar: size 0 bytes")
-	}
-
-	// do the setting
-	avatarInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(buf.Bytes(), accountID, media.Avatar, "")
-	if err != nil {
-		return nil, fmt.Errorf("error processing avatar: %s", err)
-	}
-
-	return avatarInfo, f.Close()
-}
-
-// updateAccountHeader does the dirty work of checking the header part of an account update form,
-// parsing and checking the image, and doing the necessary updates in the database for this to become
-// the account's new header image.
-func (p *processor) updateAccountHeader(header *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) {
-	var err error
-	if int(header.Size) > p.config.MediaConfig.MaxImageSize {
-		err = fmt.Errorf("header with size %d exceeded max image size of %d bytes", header.Size, p.config.MediaConfig.MaxImageSize)
-		return nil, err
-	}
-	f, err := header.Open()
-	if err != nil {
-		return nil, fmt.Errorf("could not read provided header: %s", err)
-	}
-
-	// extract the bytes
-	buf := new(bytes.Buffer)
-	size, err := io.Copy(buf, f)
-	if err != nil {
-		return nil, fmt.Errorf("could not read provided header: %s", err)
-	}
-	if size == 0 {
-		return nil, errors.New("could not read provided header: size 0 bytes")
-	}
-
-	// do the setting
-	headerInfo, err := p.mediaHandler.ProcessHeaderOrAvatar(buf.Bytes(), accountID, media.Header, "")
-	if err != nil {
-		return nil, fmt.Errorf("error processing header: %s", err)
-	}
-
-	return headerInfo, f.Close()
-}
-
-// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
-// on behalf of requestingUsername.
-//
-// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
-//
-// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated
-// to reflect the creation of these new attachments.
-func (p *processor) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
-	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
-		a, err := p.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
-			RemoteURL: targetAccount.AvatarRemoteURL,
-			Avatar:    true,
-		}, targetAccount.ID)
-		if err != nil {
-			return fmt.Errorf("error processing avatar for user: %s", err)
-		}
-		targetAccount.AvatarMediaAttachmentID = a.ID
-	}
-
-	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
-		a, err := p.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
-			RemoteURL: targetAccount.HeaderRemoteURL,
-			Header:    true,
-		}, targetAccount.ID)
-		if err != nil {
-			return fmt.Errorf("error processing header for user: %s", err)
-		}
-		targetAccount.HeaderMediaAttachmentID = a.ID
-	}
-	return nil
-}

testrig/federator.go

@@ -19,12 +19,13 @@
 package testrig
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
 
 // NewTestFederator returns a federator with the given database and (mock!!) transport controller.
-func NewTestFederator(db db.DB, tc transport.Controller) federation.Federator {
+func NewTestFederator(db db.DB, tc transport.Controller, storage blob.Storage) federation.Federator {
-	return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestConfig(), NewTestLog(), NewTestTypeConverter(db))
+	return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestConfig(), NewTestLog(), NewTestTypeConverter(db), NewTestMediaHandler(db, storage))
 }