[feature] Fetch + display custom emoji in statuses from remote instances (#807)

* start implementing remote emoji fetcher

* update status where pk

* aaa

* tidy up a little

* check size limits for emojis

* thank you linter, i love you <3

* update swagger docs

* add emoji dereference test

* make emoji max sizes configurable

* normalize db.ErrAlreadyExists

internal/config/config.go

@@ -79,6 +79,8 @@ type Configuration struct {
 	MediaDescriptionMinChars int `name:"media-description-min-chars" usage:"Min required chars for an image description"`
 	MediaDescriptionMaxChars int `name:"media-description-max-chars" usage:"Max permitted chars for an image description"`
 	MediaRemoteCacheDays     int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
+	MediaEmojiLocalMaxSize   int `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
+	MediaEmojiRemoteMaxSize  int `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
 
 	StorageBackend       string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
 	StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`

internal/config/defaults.go

@@ -58,6 +58,8 @@ var Defaults = Configuration{
 	MediaDescriptionMinChars: 0,
 	MediaDescriptionMaxChars: 500,
 	MediaRemoteCacheDays:     30,
+	MediaEmojiLocalMaxSize:   51200,  // 50kb
+	MediaEmojiRemoteMaxSize:  102400, // 100kb
 
 	StorageBackend:       "local",
 	StorageLocalBasePath: "/gotosocial/storage",

internal/config/flags.go

@@ -75,6 +75,8 @@ func AddServerFlags(cmd *cobra.Command) {
 		cmd.Flags().Int(MediaDescriptionMinCharsFlag(), cfg.MediaDescriptionMinChars, fieldtag("MediaDescriptionMinChars", "usage"))
 		cmd.Flags().Int(MediaDescriptionMaxCharsFlag(), cfg.MediaDescriptionMaxChars, fieldtag("MediaDescriptionMaxChars", "usage"))
 		cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage"))
+		cmd.Flags().Int(MediaEmojiLocalMaxSizeFlag(), cfg.MediaEmojiLocalMaxSize, fieldtag("MediaEmojiLocalMaxSize", "usage"))
+		cmd.Flags().Int(MediaEmojiRemoteMaxSizeFlag(), cfg.MediaEmojiRemoteMaxSize, fieldtag("MediaEmojiRemoteMaxSize", "usage"))
 
 		// Storage
 		cmd.Flags().String(StorageBackendFlag(), cfg.StorageBackend, fieldtag("StorageBackend", "usage"))

internal/config/helpers.gen.go

@@ -793,6 +793,56 @@ func GetMediaRemoteCacheDays() int { return global.GetMediaRemoteCacheDays() }
 // SetMediaRemoteCacheDays safely sets the value for global configuration 'MediaRemoteCacheDays' field
 func SetMediaRemoteCacheDays(v int) { global.SetMediaRemoteCacheDays(v) }
 
+// GetMediaEmojiLocalMaxSize safely fetches the Configuration value for state's 'MediaEmojiLocalMaxSize' field
+func (st *ConfigState) GetMediaEmojiLocalMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.MediaEmojiLocalMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetMediaEmojiLocalMaxSize safely sets the Configuration value for state's 'MediaEmojiLocalMaxSize' field
+func (st *ConfigState) SetMediaEmojiLocalMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaEmojiLocalMaxSize = v
+	st.reloadToViper()
+}
+
+// MediaEmojiLocalMaxSizeFlag returns the flag name for the 'MediaEmojiLocalMaxSize' field
+func MediaEmojiLocalMaxSizeFlag() string { return "media-emoji-local-max-size" }
+
+// GetMediaEmojiLocalMaxSize safely fetches the value for global configuration 'MediaEmojiLocalMaxSize' field
+func GetMediaEmojiLocalMaxSize() int { return global.GetMediaEmojiLocalMaxSize() }
+
+// SetMediaEmojiLocalMaxSize safely sets the value for global configuration 'MediaEmojiLocalMaxSize' field
+func SetMediaEmojiLocalMaxSize(v int) { global.SetMediaEmojiLocalMaxSize(v) }
+
+// GetMediaEmojiRemoteMaxSize safely fetches the Configuration value for state's 'MediaEmojiRemoteMaxSize' field
+func (st *ConfigState) GetMediaEmojiRemoteMaxSize() (v int) {
+	st.mutex.Lock()
+	v = st.config.MediaEmojiRemoteMaxSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetMediaEmojiRemoteMaxSize safely sets the Configuration value for state's 'MediaEmojiRemoteMaxSize' field
+func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.MediaEmojiRemoteMaxSize = v
+	st.reloadToViper()
+}
+
+// MediaEmojiRemoteMaxSizeFlag returns the flag name for the 'MediaEmojiRemoteMaxSize' field
+func MediaEmojiRemoteMaxSizeFlag() string { return "media-emoji-remote-max-size" }
+
+// GetMediaEmojiRemoteMaxSize safely fetches the value for global configuration 'MediaEmojiRemoteMaxSize' field
+func GetMediaEmojiRemoteMaxSize() int { return global.GetMediaEmojiRemoteMaxSize() }
+
+// SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field
+func SetMediaEmojiRemoteMaxSize(v int) { global.SetMediaEmojiRemoteMaxSize(v) }
+
 // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field
 func (st *ConfigState) GetStorageBackend() (v string) {
 	st.mutex.Lock()

internal/config/validate.go

@@ -67,6 +67,14 @@ func Validate() error {
 		errs = append(errs, fmt.Errorf("%s must be set", WebAssetBaseDirFlag()))
 	}
 
+	if m := GetMediaEmojiLocalMaxSize(); m < 0 {
+		errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiLocalMaxSizeFlag()))
+	}
+
+	if m := GetMediaEmojiRemoteMaxSize(); m < 0 {
+		errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiRemoteMaxSizeFlag()))
+	}
+
 	if len(errs) > 0 {
 		errStrings := []string{}
 		for _, err := range errs {

internal/config/validate_test.go

@@ -141,6 +141,16 @@ func (suite *ConfigValidateTestSuite) TestValidateConfigBadProtocolNoHost() {
 	suite.EqualError(err, "host must be set; protocol must be set to either http or https, provided value was foo")
 }
 
+func (suite *ConfigValidateTestSuite) TestValidateConfigBadEmojiSizes() {
+	testrig.InitTestConfig()
+
+	config.SetMediaEmojiLocalMaxSize(-10)
+	config.SetMediaEmojiRemoteMaxSize(-50)
+
+	err := config.Validate()
+	suite.EqualError(err, "media-emoji-local-max-size must not be less than 0; media-emoji-remote-max-size must not be less than 0")
+}
+
 func TestConfigValidateTestSuite(t *testing.T) {
 	suite.Run(t, &ConfigValidateTestSuite{})
 }