[feature] Refetch emojis when they change on remote instances (#905)

* select emoji using image_static_url

* use updated on AP emojis

* allow refetch of updated emojis

* cheeky workaround for test

* clean up old files for refreshed emoji

* check error for originalPostData

* shorten GetEmojiByStaticImageURL

* delete kirby (sorry nintendo)

internal/db/bundb/emoji.go

@@ -21,6 +21,7 @@ package bundb
 import (
 	"context"
 	"strings"
+	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/cache"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -50,6 +51,23 @@ func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error
 	return nil
 }
 
+func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, db.Error) {
+	// Update the emoji's last-updated
+	emoji.UpdatedAt = time.Now()
+
+	if _, err := e.conn.
+		NewUpdate().
+		Model(emoji).
+		Where("? = ?", bun.Ident("emoji.id"), emoji.ID).
+		Column(columns...).
+		Exec(ctx); err != nil {
+		return nil, e.conn.ProcessError(err)
+	}
+
+	e.cache.Invalidate(emoji.ID)
+	return emoji, nil
+}
+
 func (e *emojiDB) GetEmojis(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, db.Error) {
 	emojiIDs := []string{}
 
@@ -232,6 +250,21 @@ func (e *emojiDB) GetEmojiByShortcodeDomain(ctx context.Context, shortcode strin
 	)
 }
 
+func (e *emojiDB) GetEmojiByStaticURL(ctx context.Context, imageStaticURL string) (*gtsmodel.Emoji, db.Error) {
+	return e.getEmoji(
+		ctx,
+		func() (*gtsmodel.Emoji, bool) {
+			return e.cache.GetByImageStaticURL(imageStaticURL)
+		},
+		func(emoji *gtsmodel.Emoji) error {
+			return e.
+				newEmojiQ(emoji).
+				Where("? = ?", bun.Ident("emoji.image_static_url"), imageStaticURL).
+				Scan(ctx)
+		},
+	)
+}
+
 func (e *emojiDB) getEmoji(ctx context.Context, cacheGet func() (*gtsmodel.Emoji, bool), dbQuery func(*gtsmodel.Emoji) error) (*gtsmodel.Emoji, db.Error) {
 	// Attempt to fetch cached emoji
 	emoji, cached := cacheGet()

internal/db/bundb/emoji_test.go

@@ -38,6 +38,13 @@ func (suite *EmojiTestSuite) TestGetUseableEmojis() {
 	suite.Equal("rainbow", emojis[0].Shortcode)
 }
 
+func (suite *EmojiTestSuite) TestGetEmojiByStaticURL() {
+	emoji, err := suite.db.GetEmojiByStaticURL(context.Background(), "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png")
+	suite.NoError(err)
+	suite.NotNil(emoji)
+	suite.Equal("rainbow", emoji.Shortcode)
+}
+
 func (suite *EmojiTestSuite) TestGetAllEmojis() {
 	emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 0)
 

internal/db/bundb/migrations/20221011125732_refetch_updated_emojis.go

@@ -0,0 +1,48 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package migrations
+
+import (
+	"context"
+
+	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/uptrace/bun"
+)
+
+func init() {
+	up := func(ctx context.Context, db *bun.DB) error {
+		_, err := db.
+			NewCreateIndex().
+			Model(&gtsmodel.Emoji{}).
+			Index("emojis_image_static_url_idx").
+			Column("image_static_url").
+			Exec(ctx)
+		return err
+	}
+
+	down := func(ctx context.Context, db *bun.DB) error {
+		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+			return nil
+		})
+	}
+
+	if err := Migrations.Register(up, down); err != nil {
+		panic(err)
+	}
+}

internal/db/emoji.go

@@ -32,6 +32,9 @@ const EmojiAllDomains string = "all"
 type Emoji interface {
 	// PutEmoji puts one emoji in the database.
 	PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) Error
+	// UpdateEmoji updates the given columns of one emoji.
+	// If no columns are specified, every column is updated.
+	UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, Error)
 	// GetUseableEmojis gets all emojis which are useable by accounts on this instance.
 	GetUseableEmojis(ctx context.Context) ([]*gtsmodel.Emoji, Error)
 	// GetEmojis gets emojis based on given parameters. Useful for admin actions.
@@ -43,4 +46,6 @@ type Emoji interface {
 	GetEmojiByShortcodeDomain(ctx context.Context, shortcode string, domain string) (*gtsmodel.Emoji, Error)
 	// GetEmojiByURI returns one emoji based on its ActivityPub URI.
 	GetEmojiByURI(ctx context.Context, uri string) (*gtsmodel.Emoji, Error)
+	// GetEmojiByStaticURL gets an emoji using the URL of the static version of the emoji image.
+	GetEmojiByStaticURL(ctx context.Context, imageStaticURL string) (*gtsmodel.Emoji, Error)
 }