[feature] support processing of (many) more media types (#3090)

* initial work replacing our media decoding / encoding pipeline with ffprobe + ffmpeg

* specify the video codec to use when generating static image from emoji

* update go-storage library (fixes incompatibility after updating go-iotools)

* maintain image aspect ratio when generating a thumbnail for it

* update readme to show go-ffmpreg

* fix a bunch of media tests, move filesize checking to callers of media manager for more flexibility

* remove extra debug from error message

* fix up incorrect function signatures

* update PutFile to just use regular file copy, as changes are file is on separate partition

* fix remaining tests, remove some unneeded tests now we're working with ffmpeg/ffprobe

* update more tests, add more code comments

* add utilities to generate processed emoji / media outputs

* fix remaining tests

* add test for opus media file, add license header to utility cmds

* limit the number of concurrently available ffmpeg / ffprobe instances

* reduce number of instances

* further reduce number of instances

* fix envparsing test with configuration variables

* update docs and configuration with new media-{local,remote}-max-size variables
This commit is contained in:
kim 2024-07-12 09:39:47 +00:00 committed by GitHub
commit cde2fb6244
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
376 changed files with 8026 additions and 54091 deletions

76
vendor/modules.txt vendored
View file

@ -30,8 +30,17 @@ codeberg.org/gruf/go-fastcopy
# codeberg.org/gruf/go-fastpath/v2 v2.0.0
## explicit; go 1.14
codeberg.org/gruf/go-fastpath/v2
# codeberg.org/gruf/go-iotools v0.0.0-20230811115124-5d4223615a7f
## explicit; go 1.19
# codeberg.org/gruf/go-ffmpreg v0.2.2
## explicit; go 1.22.0
codeberg.org/gruf/go-ffmpreg/embed/ffmpeg
codeberg.org/gruf/go-ffmpreg/embed/ffprobe
codeberg.org/gruf/go-ffmpreg/ffmpeg
codeberg.org/gruf/go-ffmpreg/ffprobe
codeberg.org/gruf/go-ffmpreg/internal
codeberg.org/gruf/go-ffmpreg/util
codeberg.org/gruf/go-ffmpreg/wasm
# codeberg.org/gruf/go-iotools v0.0.0-20240710125620-934ae9c654cf
## explicit; go 1.21
codeberg.org/gruf/go-iotools
# codeberg.org/gruf/go-kv v1.6.4
## explicit; go 1.19
@ -52,6 +61,9 @@ codeberg.org/gruf/go-maps
# codeberg.org/gruf/go-mempool v0.0.0-20240507125005-cef10d64a760
## explicit; go 1.22.2
codeberg.org/gruf/go-mempool
# codeberg.org/gruf/go-mimetypes v1.2.0
## explicit; go 1.17
codeberg.org/gruf/go-mimetypes
# codeberg.org/gruf/go-mutexes v1.5.1
## explicit; go 1.22.2
codeberg.org/gruf/go-mutexes
@ -61,7 +73,7 @@ codeberg.org/gruf/go-runners
# codeberg.org/gruf/go-sched v1.2.3
## explicit; go 1.19
codeberg.org/gruf/go-sched
# codeberg.org/gruf/go-storage v0.1.1
# codeberg.org/gruf/go-storage v0.1.2
## explicit; go 1.22
codeberg.org/gruf/go-storage
codeberg.org/gruf/go-storage/disk
@ -71,9 +83,6 @@ codeberg.org/gruf/go-storage/s3
# codeberg.org/gruf/go-structr v0.8.7
## explicit; go 1.21
codeberg.org/gruf/go-structr
# codeberg.org/superseriousbusiness/exif-terminator v0.7.0
## explicit; go 1.21
codeberg.org/superseriousbusiness/exif-terminator
# github.com/DmitriyVTitov/size v1.5.0
## explicit; go 1.14
github.com/DmitriyVTitov/size
@ -89,11 +98,6 @@ github.com/Masterminds/semver/v3
# github.com/Masterminds/sprig/v3 v3.2.3
## explicit; go 1.13
github.com/Masterminds/sprig/v3
# github.com/abema/go-mp4 v1.2.0
## explicit; go 1.14
github.com/abema/go-mp4
github.com/abema/go-mp4/internal/bitio
github.com/abema/go-mp4/internal/util
# github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
## explicit; go 1.13
github.com/asaskevich/govalidator
@ -179,24 +183,6 @@ github.com/disintegration/imaging
# github.com/docker/go-units v0.5.0
## explicit
github.com/docker/go-units
# github.com/dsoprea/go-exif/v3 v3.0.0-20210625224831-a6301f85c82b
## explicit; go 1.12
github.com/dsoprea/go-exif/v3
github.com/dsoprea/go-exif/v3/common
github.com/dsoprea/go-exif/v3/undefined
# github.com/dsoprea/go-iptc v0.0.0-20200610044640-bc9ca208b413
## explicit; go 1.13
github.com/dsoprea/go-iptc
# github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd
## explicit; go 1.13
github.com/dsoprea/go-logging
# github.com/dsoprea/go-photoshop-info-format v0.0.0-20200610045659-121dd752914d
## explicit; go 1.13
github.com/dsoprea/go-photoshop-info-format
# github.com/dsoprea/go-utility/v2 v2.0.0-20200717064901-2fccff4aa15e
## explicit; go 1.12
github.com/dsoprea/go-utility/v2/filesystem
github.com/dsoprea/go-utility/v2/image
# github.com/dustin/go-humanize v1.0.1
## explicit; go 1.16
github.com/dustin/go-humanize
@ -232,9 +218,6 @@ github.com/gin-gonic/gin/binding
github.com/gin-gonic/gin/internal/bytesconv
github.com/gin-gonic/gin/internal/json
github.com/gin-gonic/gin/render
# github.com/go-errors/errors v1.4.1
## explicit; go 1.14
github.com/go-errors/errors
# github.com/go-fed/httpsig v1.1.0
## explicit; go 1.13
github.com/go-fed/httpsig
@ -322,9 +305,6 @@ github.com/go-swagger/go-swagger/cmd/swagger/commands/generate
github.com/go-swagger/go-swagger/cmd/swagger/commands/initcmd
github.com/go-swagger/go-swagger/codescan
github.com/go-swagger/go-swagger/generator
# github.com/go-xmlfmt/xmlfmt v0.0.0-20211206191508-7fd73a941850
## explicit
github.com/go-xmlfmt/xmlfmt
# github.com/goccy/go-json v0.10.3
## explicit; go 1.19
github.com/goccy/go-json
@ -342,13 +322,6 @@ github.com/godbus/dbus/v5
# github.com/golang-jwt/jwt v3.2.2+incompatible
## explicit
github.com/golang-jwt/jwt
# github.com/golang/geo v0.0.0-20210211234256-740aa86cb551
## explicit; go 1.12
github.com/golang/geo/r1
github.com/golang/geo/r2
github.com/golang/geo/r3
github.com/golang/geo/s1
github.com/golang/geo/s2
# github.com/google/go-cmp v0.6.0
## explicit; go 1.13
github.com/google/go-cmp/cmp
@ -385,12 +358,6 @@ github.com/gorilla/websocket
github.com/grpc-ecosystem/grpc-gateway/v2/internal/httprule
github.com/grpc-ecosystem/grpc-gateway/v2/runtime
github.com/grpc-ecosystem/grpc-gateway/v2/utilities
# github.com/h2non/filetype v1.1.3
## explicit; go 1.13
github.com/h2non/filetype
github.com/h2non/filetype/matchers
github.com/h2non/filetype/matchers/isobmff
github.com/h2non/filetype/types
# github.com/hashicorp/golang-lru/v2 v2.0.7
## explicit; go 1.18
github.com/hashicorp/golang-lru/v2
@ -815,12 +782,6 @@ github.com/superseriousbusiness/activity/streams/values/rfc2045
github.com/superseriousbusiness/activity/streams/values/rfc5988
github.com/superseriousbusiness/activity/streams/values/string
github.com/superseriousbusiness/activity/streams/vocab
# github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe
## explicit; go 1.17
github.com/superseriousbusiness/go-jpeg-image-structure/v2
# github.com/superseriousbusiness/go-png-image-structure/v2 v2.0.1-SSB
## explicit; go 1.12
github.com/superseriousbusiness/go-png-image-structure/v2
# github.com/superseriousbusiness/httpsig v1.2.0-SSB
## explicit; go 1.21
github.com/superseriousbusiness/httpsig
@ -851,6 +812,7 @@ github.com/tetratelabs/wazero
github.com/tetratelabs/wazero/api
github.com/tetratelabs/wazero/experimental
github.com/tetratelabs/wazero/experimental/sys
github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1
github.com/tetratelabs/wazero/internal/descriptor
github.com/tetratelabs/wazero/internal/engine/interpreter
github.com/tetratelabs/wazero/internal/engine/wazevo
@ -875,6 +837,7 @@ github.com/tetratelabs/wazero/internal/sysfs
github.com/tetratelabs/wazero/internal/u32
github.com/tetratelabs/wazero/internal/u64
github.com/tetratelabs/wazero/internal/version
github.com/tetratelabs/wazero/internal/wasip1
github.com/tetratelabs/wazero/internal/wasm
github.com/tetratelabs/wazero/internal/wasm/binary
github.com/tetratelabs/wazero/internal/wasmdebug
@ -1086,12 +1049,8 @@ golang.org/x/exp/slog/internal/buffer
## explicit; go 1.18
golang.org/x/image/bmp
golang.org/x/image/ccitt
golang.org/x/image/riff
golang.org/x/image/tiff
golang.org/x/image/tiff/lzw
golang.org/x/image/vp8
golang.org/x/image/vp8l
golang.org/x/image/webp
# golang.org/x/mod v0.18.0
## explicit; go 1.18
golang.org/x/mod/internal/lazyregexp
@ -1100,7 +1059,6 @@ golang.org/x/mod/semver
# golang.org/x/net v0.27.0
## explicit; go 1.18
golang.org/x/net/bpf
golang.org/x/net/context
golang.org/x/net/html
golang.org/x/net/html/atom
golang.org/x/net/http/httpguts