mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 02:12:25 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] support processing of (many) more media types (#3090)

Browse source 
* initial work replacing our media decoding / encoding pipeline with ffprobe + ffmpeg

* specify the video codec to use when generating static image from emoji

* update go-storage library (fixes incompatibility after updating go-iotools)

* maintain image aspect ratio when generating a thumbnail for it

* update readme to show go-ffmpreg

* fix a bunch of media tests, move filesize checking to callers of media manager for more flexibility

* remove extra debug from error message

* fix up incorrect function signatures

* update PutFile to just use regular file copy, as changes are file is on separate partition

* fix remaining tests, remove some unneeded tests now we're working with ffmpeg/ffprobe

* update more tests, add more code comments

* add utilities to generate processed emoji / media outputs

* fix remaining tests

* add test for opus media file, add license header to utility cmds

* limit the number of concurrently available ffmpeg / ffprobe instances

* reduce number of instances

* further reduce number of instances

* fix envparsing test with configuration variables

* update docs and configuration with new media-{local,remote}-max-size variables
This commit is contained in:
kim 2024-07-12 09:39:47 +00:00 • committed by GitHub
commit cde2fb6244
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
376 changed files with 8026 additions and 54091 deletions
Download patch file Download diff file Expand all files Collapse all files

4
internal/config/config.go
View file

@ -92,13 +92,13 @@ type Configuration struct {
AccountsAllowCustomCSS bool `name:"accounts-allow-custom-css" usage:"Allow accounts to enable custom CSS for their profile pages and statuses."`
AccountsCustomCSSLength int `name:"accounts-custom-css-length" usage:"Maximum permitted length (characters) of custom CSS for accounts."`
MediaImageMaxSize bytesize.Size `name:"media-image-max-size" usage:"Max size of accepted images in bytes"`
MediaVideoMaxSize bytesize.Size `name:"media-video-max-size" usage:"Max size of accepted videos in bytes"`
MediaDescriptionMinChars int `name:"media-description-min-chars" usage:"Min required chars for an image description"`
MediaDescriptionMaxChars int `name:"media-description-max-chars" usage:"Max permitted chars for an image description"`
MediaRemoteCacheDays int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."`
MediaEmojiLocalMaxSize bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."`
MediaEmojiRemoteMaxSize bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."`
MediaLocalMaxSize bytesize.Size `name:"media-local-max-size" usage:"Max size in bytes of media uploaded to this instance via API"`
MediaRemoteMaxSize bytesize.Size `name:"media-remote-max-size" usage:"Max size in bytes of media to download from other instances"`
MediaCleanupFrom string `name:"media-cleanup-from" usage:"Time of day from which to start running media cleanup/prune jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."`
MediaCleanupEvery time.Duration `name:"media-cleanup-every" usage:"Period to elapse between cleanups, starting from media-cleanup-at."`

4
internal/config/defaults.go
View file

@ -71,11 +71,11 @@ var Defaults = Configuration{
AccountsAllowCustomCSS: false,
AccountsCustomCSSLength: 10000,
MediaImageMaxSize: 10 * bytesize.MiB,
MediaVideoMaxSize: 40 * bytesize.MiB,
MediaDescriptionMinChars: 0,
MediaDescriptionMaxChars: 1500,
MediaRemoteCacheDays: 7,
MediaLocalMaxSize: 40 * bytesize.MiB,
MediaRemoteMaxSize: 40 * bytesize.MiB,
MediaEmojiLocalMaxSize: 50 * bytesize.KiB,
MediaEmojiRemoteMaxSize: 100 * bytesize.KiB,
MediaCleanupFrom: "00:00", // Midnight.

4
internal/config/flags.go
View file

@ -97,11 +97,11 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) {
cmd.Flags().Bool(AccountsAllowCustomCSSFlag(), cfg.AccountsAllowCustomCSS, fieldtag("AccountsAllowCustomCSS", "usage"))
// Media
cmd.Flags().Uint64(MediaImageMaxSizeFlag(), uint64(cfg.MediaImageMaxSize), fieldtag("MediaImageMaxSize", "usage"))
cmd.Flags().Uint64(MediaVideoMaxSizeFlag(), uint64(cfg.MediaVideoMaxSize), fieldtag("MediaVideoMaxSize", "usage"))
cmd.Flags().Int(MediaDescriptionMinCharsFlag(), cfg.MediaDescriptionMinChars, fieldtag("MediaDescriptionMinChars", "usage"))
cmd.Flags().Int(MediaDescriptionMaxCharsFlag(), cfg.MediaDescriptionMaxChars, fieldtag("MediaDescriptionMaxChars", "usage"))
cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage"))
cmd.Flags().Uint64(MediaLocalMaxSizeFlag(), uint64(cfg.MediaLocalMaxSize), fieldtag("MediaLocalMaxSize", "usage"))
cmd.Flags().Uint64(MediaRemoteMaxSizeFlag(), uint64(cfg.MediaRemoteMaxSize), fieldtag("MediaRemoteMaxSize", "usage"))
cmd.Flags().Uint64(MediaEmojiLocalMaxSizeFlag(), uint64(cfg.MediaEmojiLocalMaxSize), fieldtag("MediaEmojiLocalMaxSize", "usage"))
cmd.Flags().Uint64(MediaEmojiRemoteMaxSizeFlag(), uint64(cfg.MediaEmojiRemoteMaxSize), fieldtag("MediaEmojiRemoteMaxSize", "usage"))
cmd.Flags().String(MediaCleanupFromFlag(), cfg.MediaCleanupFrom, fieldtag("MediaCleanupFrom", "usage"))

100
internal/config/helpers.gen.go
View file

@ -1075,56 +1075,6 @@ func GetAccountsCustomCSSLength() int { return global.GetAccountsCustomCSSLength
// SetAccountsCustomCSSLength safely sets the value for global configuration 'AccountsCustomCSSLength' field
func SetAccountsCustomCSSLength(v int) { global.SetAccountsCustomCSSLength(v) }
// GetMediaImageMaxSize safely fetches the Configuration value for state's 'MediaImageMaxSize' field
func (st *ConfigState) GetMediaImageMaxSize() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaImageMaxSize
st.mutex.RUnlock()
return
}
// SetMediaImageMaxSize safely sets the Configuration value for state's 'MediaImageMaxSize' field
func (st *ConfigState) SetMediaImageMaxSize(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaImageMaxSize = v
st.reloadToViper()
}
// MediaImageMaxSizeFlag returns the flag name for the 'MediaImageMaxSize' field
func MediaImageMaxSizeFlag() string { return "media-image-max-size" }
// GetMediaImageMaxSize safely fetches the value for global configuration 'MediaImageMaxSize' field
func GetMediaImageMaxSize() bytesize.Size { return global.GetMediaImageMaxSize() }
// SetMediaImageMaxSize safely sets the value for global configuration 'MediaImageMaxSize' field
func SetMediaImageMaxSize(v bytesize.Size) { global.SetMediaImageMaxSize(v) }
// GetMediaVideoMaxSize safely fetches the Configuration value for state's 'MediaVideoMaxSize' field
func (st *ConfigState) GetMediaVideoMaxSize() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaVideoMaxSize
st.mutex.RUnlock()
return
}
// SetMediaVideoMaxSize safely sets the Configuration value for state's 'MediaVideoMaxSize' field
func (st *ConfigState) SetMediaVideoMaxSize(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaVideoMaxSize = v
st.reloadToViper()
}
// MediaVideoMaxSizeFlag returns the flag name for the 'MediaVideoMaxSize' field
func MediaVideoMaxSizeFlag() string { return "media-video-max-size" }
// GetMediaVideoMaxSize safely fetches the value for global configuration 'MediaVideoMaxSize' field
func GetMediaVideoMaxSize() bytesize.Size { return global.GetMediaVideoMaxSize() }
// SetMediaVideoMaxSize safely sets the value for global configuration 'MediaVideoMaxSize' field
func SetMediaVideoMaxSize(v bytesize.Size) { global.SetMediaVideoMaxSize(v) }
// GetMediaDescriptionMinChars safely fetches the Configuration value for state's 'MediaDescriptionMinChars' field
func (st *ConfigState) GetMediaDescriptionMinChars() (v int) {
st.mutex.RLock()
@ -1250,6 +1200,56 @@ func GetMediaEmojiRemoteMaxSize() bytesize.Size { return global.GetMediaEmojiRem
// SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field
func SetMediaEmojiRemoteMaxSize(v bytesize.Size) { global.SetMediaEmojiRemoteMaxSize(v) }
// GetMediaLocalMaxSize safely fetches the Configuration value for state's 'MediaLocalMaxSize' field
func (st *ConfigState) GetMediaLocalMaxSize() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaLocalMaxSize
st.mutex.RUnlock()
return
}
// SetMediaLocalMaxSize safely sets the Configuration value for state's 'MediaLocalMaxSize' field
func (st *ConfigState) SetMediaLocalMaxSize(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaLocalMaxSize = v
st.reloadToViper()
}
// MediaLocalMaxSizeFlag returns the flag name for the 'MediaLocalMaxSize' field
func MediaLocalMaxSizeFlag() string { return "media-local-max-size" }
// GetMediaLocalMaxSize safely fetches the value for global configuration 'MediaLocalMaxSize' field
func GetMediaLocalMaxSize() bytesize.Size { return global.GetMediaLocalMaxSize() }
// SetMediaLocalMaxSize safely sets the value for global configuration 'MediaLocalMaxSize' field
func SetMediaLocalMaxSize(v bytesize.Size) { global.SetMediaLocalMaxSize(v) }
// GetMediaRemoteMaxSize safely fetches the Configuration value for state's 'MediaRemoteMaxSize' field
func (st *ConfigState) GetMediaRemoteMaxSize() (v bytesize.Size) {
st.mutex.RLock()
v = st.config.MediaRemoteMaxSize
st.mutex.RUnlock()
return
}
// SetMediaRemoteMaxSize safely sets the Configuration value for state's 'MediaRemoteMaxSize' field
func (st *ConfigState) SetMediaRemoteMaxSize(v bytesize.Size) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.MediaRemoteMaxSize = v
st.reloadToViper()
}
// MediaRemoteMaxSizeFlag returns the flag name for the 'MediaRemoteMaxSize' field
func MediaRemoteMaxSizeFlag() string { return "media-remote-max-size" }
// GetMediaRemoteMaxSize safely fetches the value for global configuration 'MediaRemoteMaxSize' field
func GetMediaRemoteMaxSize() bytesize.Size { return global.GetMediaRemoteMaxSize() }
// SetMediaRemoteMaxSize safely sets the value for global configuration 'MediaRemoteMaxSize' field
func SetMediaRemoteMaxSize(v bytesize.Size) { global.SetMediaRemoteMaxSize(v) }
// GetMediaCleanupFrom safely fetches the Configuration value for state's 'MediaCleanupFrom' field
func (st *ConfigState) GetMediaCleanupFrom() (v string) {
st.mutex.RLock()