[feature] Show + federate emojis in accounts (#837)

* Start adding account emoji

* get emojis serialized + deserialized nicely

* update tests

* set / retrieve emojis on accounts

* show account emojis in web view

* fetch emojis from db based on ids

* fix typo in test

* lint

* fix pg migration

* update tests

* update emoji checking logic

* update comment

* clarify comments + add some spacing

* tidy up loops a lil (thanks kim)

internal/federation/dereferencing/account.go

@@ -76,6 +76,11 @@ type GetRemoteAccountParams struct {
 	// quickly fetch a remote account from the database or fail, and don't want to cause
 	// http requests to go flying around.
 	SkipResolve bool
+	// PartialAccount can be used if the GetRemoteAccount call results from a federated/ap
+	// account update. In this case, we will already have a partial representation of the account,
+	// derived from converting the AP representation to a gtsmodel representation. If this field
+	// is provided, then GetRemoteAccount will use this as a basis for building the full account.
+	PartialAccount *gtsmodel.Account
 }
 
 // GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account,
@@ -107,8 +112,16 @@ func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountPar
 	skipResolve := params.SkipResolve
 
 	// this first step checks if we have the
-	// account in the database somewhere already
+	// account in the database somewhere already,
+	// or if we've been provided it as a partial
 	switch {
+	case params.PartialAccount != nil:
+		foundAccount = params.PartialAccount
+		if foundAccount.Domain == "" || foundAccount.Domain == config.GetHost() || foundAccount.Domain == config.GetAccountDomain() {
+			// this is actually a local account,
+			// make sure we don't try to resolve
+			skipResolve = true
+		}
 	case params.RemoteAccountID != nil:
 		uri := params.RemoteAccountID
 		host := uri.Host
@@ -163,7 +176,7 @@ func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountPar
 		params.RemoteAccountHost = params.RemoteAccountID.Host
 		// ... but we still need the username so we can do a finger for the accountDomain
 
-		// check if we had the account stored already and got it earlier
+		// check if we got the account earlier
 		if foundAccount != nil {
 			params.RemoteAccountUsername = foundAccount.Username
 		} else {
@@ -201,9 +214,10 @@ func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountPar
 	// to save on remote calls, only webfinger if:
 	// - we don't know the remote account ActivityPub ID yet OR
 	// - we haven't found the account yet in some other way OR
+	// - we were passed a partial account in params OR
 	// - we haven't webfingered the account for two days AND the account isn't an instance account
 	var fingered time.Time
-	if params.RemoteAccountID == nil || foundAccount == nil || (foundAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(foundAccount)) {
+	if params.RemoteAccountID == nil || foundAccount == nil || params.PartialAccount != nil || (foundAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(foundAccount)) {
 		accountDomain, params.RemoteAccountID, err = d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost)
 		if err != nil {
 			err = fmt.Errorf("GetRemoteAccount: error while fingering: %s", err)
@@ -263,7 +277,7 @@ func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountPar
 		foundAccount.LastWebfingeredAt = fingered
 		foundAccount.UpdatedAt = time.Now()
 
-		err = d.db.Put(ctx, foundAccount)
+		foundAccount, err = d.db.PutAccount(ctx, foundAccount)
 		if err != nil {
 			err = fmt.Errorf("GetRemoteAccount: error putting new account: %s", err)
 			return
@@ -273,13 +287,10 @@ func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountPar
 	}
 
 	// we had the account already, but now we know the account domain, so update it if it's different
+	var accountDomainChanged bool
 	if !strings.EqualFold(foundAccount.Domain, accountDomain) {
+		accountDomainChanged = true
 		foundAccount.Domain = accountDomain
-		foundAccount, err = d.db.UpdateAccount(ctx, foundAccount)
-		if err != nil {
-			err = fmt.Errorf("GetRemoteAccount: error updating account: %s", err)
-			return
-		}
 	}
 
 	// if SharedInboxURI is nil, that means we don't know yet if this account has
@@ -327,8 +338,7 @@ func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountPar
 		foundAccount.LastWebfingeredAt = fingered
 	}
 
-	if fieldsChanged || fingeredChanged || sharedInboxChanged {
-		foundAccount.UpdatedAt = time.Now()
+	if accountDomainChanged || sharedInboxChanged || fieldsChanged || fingeredChanged {
 		foundAccount, err = d.db.UpdateAccount(ctx, foundAccount)
 		if err != nil {
 			return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err)
@@ -423,15 +433,20 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc
 		return false, fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host)
 	}
 
-	t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
-	if err != nil {
-		return false, fmt.Errorf("populateAccountFields: error getting transport for user: %s", err)
-	}
+	var changed bool
 
 	// fetch the header and avatar
-	changed, err := d.fetchRemoteAccountMedia(ctx, account, t, blocking)
-	if err != nil {
+	if mediaChanged, err := d.fetchRemoteAccountMedia(ctx, account, requestingUsername, blocking); err != nil {
 		return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err)
+	} else if mediaChanged {
+		changed = mediaChanged
+	}
+
+	// fetch any emojis used in note, fields, display name, etc
+	if emojisChanged, err := d.fetchRemoteAccountEmojis(ctx, account, requestingUsername); err != nil {
+		return false, fmt.Errorf("populateAccountFields: error fetching emojis for account: %s", err)
+	} else if emojisChanged {
+		changed = emojisChanged
 	}
 
 	return changed, nil
@@ -449,17 +464,11 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc
 //
 // If blocking is true, then the calls to the media manager made by this function will be blocking:
 // in other words, the function won't return until the header and the avatar have been fully processed.
-func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool) (bool, error) {
-	changed := false
-
-	accountURI, err := url.Parse(targetAccount.URI)
-	if err != nil {
-		return changed, fmt.Errorf("fetchRemoteAccountMedia: couldn't parse account URI %s: %s", targetAccount.URI, err)
-	}
-
-	if blocked, err := d.db.IsDomainBlocked(ctx, accountURI.Host); blocked || err != nil {
-		return changed, fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host)
-	}
+func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) {
+	var (
+		changed bool
+		t       transport.Transport
+	)
 
 	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") {
 		var processingMedia *media.ProcessingMedia
@@ -479,6 +488,14 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 				return changed, err
 			}
 
+			if t == nil {
+				var err error
+				t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername)
+				if err != nil {
+					return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
+				}
+			}
+
 			data := func(innerCtx context.Context) (io.Reader, int, error) {
 				return t.DereferenceMedia(innerCtx, avatarIRI)
 			}
@@ -537,6 +554,14 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 				return changed, err
 			}
 
+			if t == nil {
+				var err error
+				t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername)
+				if err != nil {
+					return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
+				}
+			}
+
 			data := func(innerCtx context.Context) (io.Reader, int, error) {
 				return t.DereferenceMedia(innerCtx, headerIRI)
 			}
@@ -580,6 +605,118 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
 	return changed, nil
 }
 
+func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) {
+	maybeEmojis := targetAccount.Emojis
+	maybeEmojiIDs := targetAccount.EmojiIDs
+
+	// It's possible that the account had emoji IDs set on it, but not Emojis
+	// themselves, depending on how it was fetched before being passed to us.
+	//
+	// If we only have IDs, fetch the emojis from the db. We know they're in
+	// there or else they wouldn't have IDs.
+	if len(maybeEmojiIDs) > len(maybeEmojis) {
+		maybeEmojis = []*gtsmodel.Emoji{}
+		for _, emojiID := range maybeEmojiIDs {
+			maybeEmoji, err := d.db.GetEmojiByID(ctx, emojiID)
+			if err != nil {
+				return false, err
+			}
+			maybeEmojis = append(maybeEmojis, maybeEmoji)
+		}
+	}
+
+	// For all the maybe emojis we have, we either fetch them from the database
+	// (if we haven't already), or dereference them from the remote instance.
+	gotEmojis, err := d.populateEmojis(ctx, maybeEmojis, requestingUsername)
+	if err != nil {
+		return false, err
+	}
+
+	// Extract the ID of each fetched or dereferenced emoji, so we can attach
+	// this to the account if necessary.
+	gotEmojiIDs := make([]string, 0, len(gotEmojis))
+	for _, e := range gotEmojis {
+		gotEmojiIDs = append(gotEmojiIDs, e.ID)
+	}
+
+	var (
+		changed  = false // have the emojis for this account changed?
+		maybeLen = len(maybeEmojis)
+		gotLen   = len(gotEmojis)
+	)
+
+	// if the length of everything is zero, this is simple:
+	// nothing has changed and there's nothing to do
+	if maybeLen == 0 && gotLen == 0 {
+		return changed, nil
+	}
+
+	// if the *amount* of emojis on the account has changed, then the got emojis
+	// are definitely different from the previous ones (if there were any) --
+	// the account has either more or fewer emojis set on it now, so take the
+	// discovered emojis as the new correct ones.
+	if maybeLen != gotLen {
+		changed = true
+		targetAccount.Emojis = gotEmojis
+		targetAccount.EmojiIDs = gotEmojiIDs
+		return changed, nil
+	}
+
+	// if the lengths are the same but not all of the slices are
+	// zero, something *might* have changed, so we have to check
+
+	// 1. did we have emojis before that we don't have now?
+	for _, maybeEmoji := range maybeEmojis {
+		var stillPresent bool
+
+		for _, gotEmoji := range gotEmojis {
+			if maybeEmoji.URI == gotEmoji.URI {
+				// the emoji we maybe had is still present now,
+				// so we can stop checking gotEmojis
+				stillPresent = true
+				break
+			}
+		}
+
+		if !stillPresent {
+			// at least one maybeEmoji is no longer present in
+			// the got emojis, so we can stop checking now
+			changed = true
+			targetAccount.Emojis = gotEmojis
+			targetAccount.EmojiIDs = gotEmojiIDs
+			return changed, nil
+		}
+	}
+
+	// 2. do we have emojis now that we didn't have before?
+	for _, gotEmoji := range gotEmojis {
+		var wasPresent bool
+
+		for _, maybeEmoji := range maybeEmojis {
+			// check emoji IDs here as well, because unreferenced
+			// maybe emojis we didn't already have would not have
+			// had IDs set on them yet
+			if gotEmoji.URI == maybeEmoji.URI && gotEmoji.ID == maybeEmoji.ID {
+				// this got emoji was present already in the maybeEmoji,
+				// so we can stop checking through maybeEmojis
+				wasPresent = true
+				break
+			}
+		}
+
+		if !wasPresent {
+			// at least one gotEmojis was not present in
+			// the maybeEmojis, so we can stop checking now
+			changed = true
+			targetAccount.Emojis = gotEmojis
+			targetAccount.EmojiIDs = gotEmojiIDs
+			return changed, nil
+		}
+	}
+
+	return changed, nil
+}
+
 func lockAndLoad(ctx context.Context, lock *sync.Mutex, processing *media.ProcessingMedia, processingMap map[string]*media.ProcessingMedia, accountID string) error {
 	// whatever happens, remove the in-process media from the map
 	defer func() {

internal/federation/dereferencing/account_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -195,6 +196,205 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() {
 	suite.Nil(fetchedAccount)
 }
 
+func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial() {
+	fetchingAccount := suite.testAccounts["local_account_1"]
+
+	remoteAccount := suite.testAccounts["remote_account_1"]
+	remoteAccountPartial := &gtsmodel.Account{
+		ID:                    remoteAccount.ID,
+		ActorType:             remoteAccount.ActorType,
+		Language:              remoteAccount.Language,
+		CreatedAt:             remoteAccount.CreatedAt,
+		UpdatedAt:             remoteAccount.UpdatedAt,
+		Username:              remoteAccount.Username,
+		Domain:                remoteAccount.Domain,
+		DisplayName:           remoteAccount.DisplayName,
+		URI:                   remoteAccount.URI,
+		InboxURI:              remoteAccount.URI,
+		SharedInboxURI:        remoteAccount.SharedInboxURI,
+		PublicKeyURI:          remoteAccount.PublicKeyURI,
+		URL:                   remoteAccount.URL,
+		FollowingURI:          remoteAccount.FollowingURI,
+		FollowersURI:          remoteAccount.FollowersURI,
+		OutboxURI:             remoteAccount.OutboxURI,
+		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
+		Emojis: []*gtsmodel.Emoji{
+			// dereference an emoji we don't have stored yet
+			{
+				URI:             "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",
+				Shortcode:       "kip_van_den_bos",
+				UpdatedAt:       testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
+				ImageRemoteURL:  "http://fossbros-anonymous.io/emoji/kip.gif",
+				Disabled:        testrig.FalseBool(),
+				VisibleInPicker: testrig.FalseBool(),
+				Domain:          "fossbros-anonymous.io",
+			},
+		},
+	}
+
+	fetchedAccount, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{
+		RequestingUsername:    fetchingAccount.Username,
+		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
+		RemoteAccountHost:     remoteAccount.Domain,
+		RemoteAccountUsername: remoteAccount.Username,
+		PartialAccount:        remoteAccountPartial,
+		Blocking:              true,
+	})
+	suite.NoError(err)
+	suite.NotNil(fetchedAccount)
+	suite.NotNil(fetchedAccount.EmojiIDs)
+	suite.NotNil(fetchedAccount.Emojis)
+}
+
+func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial2() {
+	fetchingAccount := suite.testAccounts["local_account_1"]
+
+	knownEmoji := suite.testEmojis["yell"]
+
+	remoteAccount := suite.testAccounts["remote_account_1"]
+	remoteAccountPartial := &gtsmodel.Account{
+		ID:                    remoteAccount.ID,
+		ActorType:             remoteAccount.ActorType,
+		Language:              remoteAccount.Language,
+		CreatedAt:             remoteAccount.CreatedAt,
+		UpdatedAt:             remoteAccount.UpdatedAt,
+		Username:              remoteAccount.Username,
+		Domain:                remoteAccount.Domain,
+		DisplayName:           remoteAccount.DisplayName,
+		URI:                   remoteAccount.URI,
+		InboxURI:              remoteAccount.URI,
+		SharedInboxURI:        remoteAccount.SharedInboxURI,
+		PublicKeyURI:          remoteAccount.PublicKeyURI,
+		URL:                   remoteAccount.URL,
+		FollowingURI:          remoteAccount.FollowingURI,
+		FollowersURI:          remoteAccount.FollowersURI,
+		OutboxURI:             remoteAccount.OutboxURI,
+		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
+		Emojis: []*gtsmodel.Emoji{
+			// an emoji we already have
+			{
+				URI:             knownEmoji.URI,
+				Shortcode:       knownEmoji.Shortcode,
+				UpdatedAt:       knownEmoji.CreatedAt,
+				ImageRemoteURL:  knownEmoji.ImageRemoteURL,
+				Disabled:        knownEmoji.Disabled,
+				VisibleInPicker: knownEmoji.VisibleInPicker,
+			},
+		},
+	}
+
+	fetchedAccount, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{
+		RequestingUsername:    fetchingAccount.Username,
+		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
+		RemoteAccountHost:     remoteAccount.Domain,
+		RemoteAccountUsername: remoteAccount.Username,
+		PartialAccount:        remoteAccountPartial,
+		Blocking:              true,
+	})
+	suite.NoError(err)
+	suite.NotNil(fetchedAccount)
+	suite.NotNil(fetchedAccount.EmojiIDs)
+	suite.NotNil(fetchedAccount.Emojis)
+}
+
+func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() {
+	fetchingAccount := suite.testAccounts["local_account_1"]
+
+	knownEmoji := suite.testEmojis["yell"]
+
+	remoteAccount := suite.testAccounts["remote_account_1"]
+	remoteAccountPartial := &gtsmodel.Account{
+		ID:                    remoteAccount.ID,
+		ActorType:             remoteAccount.ActorType,
+		Language:              remoteAccount.Language,
+		CreatedAt:             remoteAccount.CreatedAt,
+		UpdatedAt:             remoteAccount.UpdatedAt,
+		Username:              remoteAccount.Username,
+		Domain:                remoteAccount.Domain,
+		DisplayName:           remoteAccount.DisplayName,
+		URI:                   remoteAccount.URI,
+		InboxURI:              remoteAccount.URI,
+		SharedInboxURI:        remoteAccount.SharedInboxURI,
+		PublicKeyURI:          remoteAccount.PublicKeyURI,
+		URL:                   remoteAccount.URL,
+		FollowingURI:          remoteAccount.FollowingURI,
+		FollowersURI:          remoteAccount.FollowersURI,
+		OutboxURI:             remoteAccount.OutboxURI,
+		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
+		Emojis: []*gtsmodel.Emoji{
+			// an emoji we already have
+			{
+				URI:             knownEmoji.URI,
+				Shortcode:       knownEmoji.Shortcode,
+				UpdatedAt:       knownEmoji.CreatedAt,
+				ImageRemoteURL:  knownEmoji.ImageRemoteURL,
+				Disabled:        knownEmoji.Disabled,
+				VisibleInPicker: knownEmoji.VisibleInPicker,
+			},
+		},
+	}
+
+	fetchedAccount, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{
+		RequestingUsername:    fetchingAccount.Username,
+		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
+		RemoteAccountHost:     remoteAccount.Domain,
+		RemoteAccountUsername: remoteAccount.Username,
+		PartialAccount:        remoteAccountPartial,
+		Blocking:              true,
+	})
+	suite.NoError(err)
+	suite.NotNil(fetchedAccount)
+	suite.NotNil(fetchedAccount.EmojiIDs)
+	suite.NotNil(fetchedAccount.Emojis)
+	suite.Equal(knownEmoji.URI, fetchedAccount.Emojis[0].URI)
+
+	remoteAccountPartial2 := &gtsmodel.Account{
+		ID:                    remoteAccount.ID,
+		ActorType:             remoteAccount.ActorType,
+		Language:              remoteAccount.Language,
+		CreatedAt:             remoteAccount.CreatedAt,
+		UpdatedAt:             remoteAccount.UpdatedAt,
+		Username:              remoteAccount.Username,
+		Domain:                remoteAccount.Domain,
+		DisplayName:           remoteAccount.DisplayName,
+		URI:                   remoteAccount.URI,
+		InboxURI:              remoteAccount.URI,
+		SharedInboxURI:        remoteAccount.SharedInboxURI,
+		PublicKeyURI:          remoteAccount.PublicKeyURI,
+		URL:                   remoteAccount.URL,
+		FollowingURI:          remoteAccount.FollowingURI,
+		FollowersURI:          remoteAccount.FollowersURI,
+		OutboxURI:             remoteAccount.OutboxURI,
+		FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
+		Emojis: []*gtsmodel.Emoji{
+			// dereference an emoji we don't have stored yet
+			{
+				URI:             "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",
+				Shortcode:       "kip_van_den_bos",
+				UpdatedAt:       testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
+				ImageRemoteURL:  "http://fossbros-anonymous.io/emoji/kip.gif",
+				Disabled:        testrig.FalseBool(),
+				VisibleInPicker: testrig.FalseBool(),
+				Domain:          "fossbros-anonymous.io",
+			},
+		},
+	}
+
+	fetchedAccount2, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{
+		RequestingUsername:    fetchingAccount.Username,
+		RemoteAccountID:       testrig.URLMustParse(remoteAccount.URI),
+		RemoteAccountHost:     remoteAccount.Domain,
+		RemoteAccountUsername: remoteAccount.Username,
+		PartialAccount:        remoteAccountPartial2,
+		Blocking:              true,
+	})
+	suite.NoError(err)
+	suite.NotNil(fetchedAccount2)
+	suite.NotNil(fetchedAccount2.EmojiIDs)
+	suite.NotNil(fetchedAccount2.Emojis)
+	suite.Equal("http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", fetchedAccount2.Emojis[0].URI)
+}
+
 func TestAccountTestSuite(t *testing.T) {
 	suite.Run(t, new(AccountTestSuite))
 }

internal/federation/dereferencing/dereferencer_test.go

@@ -41,6 +41,7 @@ type DereferencerStandardTestSuite struct {
 	testRemoteServices    map[string]vocab.ActivityStreamsService
 	testRemoteAttachments map[string]testrig.RemoteAttachmentFile
 	testAccounts          map[string]*gtsmodel.Account
+	testEmojis            map[string]*gtsmodel.Emoji
 
 	dereferencer dereferencing.Dereferencer
 }
@@ -55,6 +56,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
 	suite.testRemoteGroups = testrig.NewTestFediGroups()
 	suite.testRemoteServices = testrig.NewTestFediServices()
 	suite.testRemoteAttachments = testrig.NewTestFediAttachments("../../../testrig/media")
+	suite.testEmojis = testrig.NewTestEmojis()
 
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewInMemoryStorage()

internal/federation/dereferencing/emoji.go

@@ -24,6 +24,10 @@ import (
 	"io"
 	"net/url"
 
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 )
 
@@ -49,3 +53,57 @@ func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, r
 
 	return processingMedia, nil
 }
+
+func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji, requestingUsername string) ([]*gtsmodel.Emoji, error) {
+	// At this point we should know:
+	// * the AP uri of the emoji
+	// * the domain of the emoji
+	// * the shortcode of the emoji
+	// * the remote URL of the image
+	// This should be enough to dereference the emoji
+
+	gotEmojis := make([]*gtsmodel.Emoji, 0, len(rawEmojis))
+
+	for _, e := range rawEmojis {
+		var gotEmoji *gtsmodel.Emoji
+		var err error
+
+		// check if we've already got this emoji in the db
+		if gotEmoji, err = d.db.GetEmojiByURI(ctx, e.URI); err != nil && err != db.ErrNoEntries {
+			log.Errorf("populateEmojis: error checking database for emoji %s: %s", e.URI, err)
+			continue
+		}
+
+		if gotEmoji == nil {
+			// it's new! go get it!
+			newEmojiID, err := id.NewRandomULID()
+			if err != nil {
+				log.Errorf("populateEmojis: error generating id for remote emoji %s: %s", e.URI, err)
+				continue
+			}
+
+			processingEmoji, err := d.GetRemoteEmoji(ctx, requestingUsername, e.ImageRemoteURL, e.Shortcode, newEmojiID, e.URI, &media.AdditionalEmojiInfo{
+				Domain:               &e.Domain,
+				ImageRemoteURL:       &e.ImageRemoteURL,
+				ImageStaticRemoteURL: &e.ImageRemoteURL,
+				Disabled:             e.Disabled,
+				VisibleInPicker:      e.VisibleInPicker,
+			})
+
+			if err != nil {
+				log.Errorf("populateEmojis: couldn't get remote emoji %s: %s", e.URI, err)
+				continue
+			}
+
+			if gotEmoji, err = processingEmoji.LoadEmoji(ctx); err != nil {
+				log.Errorf("populateEmojis: couldn't load remote emoji %s: %s", e.URI, err)
+				continue
+			}
+		}
+
+		// if we get here, we either had the emoji already or we successfully fetched it
+		gotEmojis = append(gotEmojis, gotEmoji)
+	}
+
+	return gotEmojis, nil
+}

internal/federation/dereferencing/status.go

@@ -406,58 +406,17 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.
 }
 
 func (d *deref) populateStatusEmojis(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
-	// At this point we should know:
-	// * the AP uri of the emoji
-	// * the domain of the emoji
-	// * the shortcode of the emoji
-	// * the remote URL of the image
-	// This should be enough to dereference the emoji
-
-	gotEmojis := make([]*gtsmodel.Emoji, 0, len(status.Emojis))
-	emojiIDs := make([]string, 0, len(status.Emojis))
-
-	for _, e := range status.Emojis {
-		var gotEmoji *gtsmodel.Emoji
-		var err error
-
-		// check if we've already got this emoji in the db
-		if gotEmoji, err = d.db.GetEmojiByURI(ctx, e.URI); err != nil && err != db.ErrNoEntries {
-			log.Errorf("populateStatusEmojis: error checking database for emoji %s: %s", e.URI, err)
-			continue
-		}
-
-		if gotEmoji == nil {
-			// it's new! go get it!
-			newEmojiID, err := id.NewRandomULID()
-			if err != nil {
-				log.Errorf("populateStatusEmojis: error generating id for remote emoji %s: %s", e.URI, err)
-				continue
-			}
-
-			processingEmoji, err := d.GetRemoteEmoji(ctx, requestingUsername, e.ImageRemoteURL, e.Shortcode, newEmojiID, e.URI, &media.AdditionalEmojiInfo{
-				Domain:               &e.Domain,
-				ImageRemoteURL:       &e.ImageRemoteURL,
-				ImageStaticRemoteURL: &e.ImageRemoteURL,
-				Disabled:             e.Disabled,
-				VisibleInPicker:      e.VisibleInPicker,
-			})
-			if err != nil {
-				log.Errorf("populateStatusEmojis: couldn't get remote emoji %s: %s", e.URI, err)
-				continue
-			}
-
-			if gotEmoji, err = processingEmoji.LoadEmoji(ctx); err != nil {
-				log.Errorf("populateStatusEmojis: couldn't load remote emoji %s: %s", e.URI, err)
-				continue
-			}
-		}
-
-		// if we get here, we either had the emoji already or we successfully fetched it
-		gotEmojis = append(gotEmojis, gotEmoji)
-		emojiIDs = append(emojiIDs, gotEmoji.ID)
+	emojis, err := d.populateEmojis(ctx, status.Emojis, requestingUsername)
+	if err != nil {
+		return err
 	}
 
-	status.Emojis = gotEmojis
+	emojiIDs := make([]string, 0, len(emojis))
+	for _, e := range emojis {
+		emojiIDs = append(emojiIDs, e.ID)
+	}
+
+	status.Emojis = emojis
 	status.EmojiIDs = emojiIDs
 	return nil
 }