[feature] use webp for thumbnails (#3116)

* update to use webp for thumbnails

* bump webp quality up to 40% from 12% (it's a bit different to jpeg quality setting)

* update to use yuva colorspace, and use thumbnail=n=10 to select frame

* fix missing comma in ffmpeg args

* add links to appropriate ffmpeg docs

* update tests

* add file size tests for thumbnails

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>

internal/media/ffmpeg.go

@@ -47,10 +47,21 @@ func ffmpegClearMetadata(ctx context.Context, filepath string, ext string) error
 	// Clear metadata with ffmpeg.
 	if err := ffmpeg(ctx, dirpath,
 		"-loglevel", "error",
+
+		// Input file.
 		"-i", filepath,
+
+		// Drop all metadata.
 		"-map_metadata", "-1",
+
+		// Copy input codecs,
+		// i.e. no transcode.
 		"-codec", "copy",
+
+		// Overwrite.
 		"-y",
+
+		// Output.
 		outpath,
 	); err != nil {
 		return err
@@ -64,23 +75,54 @@ func ffmpegClearMetadata(ctx context.Context, filepath string, ext string) error
 	return nil
 }
 
-// ffmpegGenerateThumb generates a thumbnail jpeg from input media of any type, useful for any media.
+// ffmpegGenerateThumb generates a thumbnail webp from input media of any type, useful for any media.
 func ffmpegGenerateThumb(ctx context.Context, filepath string, width, height int) (string, error) {
+
 	// Get directory from filepath.
 	dirpath := path.Dir(filepath)
 
 	// Generate output frame file path.
-	outpath := filepath + "_thumb.jpg"
+	outpath := filepath + "_thumb.webp"
+
+	// Thumbnail size scaling argument.
+	scale := strconv.Itoa(width) + ":" +
+		strconv.Itoa(height)
 
 	// Generate thumb with ffmpeg.
 	if err := ffmpeg(ctx, dirpath,
 		"-loglevel", "error",
+
+		// Input file.
 		"-i", filepath,
-		"-filter:v", "thumbnail=n=10",
-		"-filter:v", "scale="+strconv.Itoa(width)+":"+strconv.Itoa(height),
-		"-qscale:v", "12", // ~ 70% quality
+
+		// Encode using libwebp.
+		// (NOT as libwebp_anim).
+		"-codec:v", "libwebp",
+
+		// Select thumb from first 10 frames
+		// (thumb filter: https://ffmpeg.org/ffmpeg-filters.html#thumbnail)
+		"-filter:v", "thumbnail=n=10,"+
+
+			// scale to dimensions
+			// (scale filter: https://ffmpeg.org/ffmpeg-filters.html#scale)
+			"scale="+scale+","+
+
+			// YUVA 4:2:0 pixel format
+			// (format filter: https://ffmpeg.org/ffmpeg-filters.html#format)
+			"format=pix_fmts=yuva420p",
+
+		// Only one frame
 		"-frames:v", "1",
+
+		// ~40% webp quality
+		// (codec options: https://ffmpeg.org/ffmpeg-codecs.html#toc-Codec-Options)
+		// (libwebp codec: https://ffmpeg.org/ffmpeg-codecs.html#Options-36)
+		"-qscale:v", "40",
+
+		// Overwrite.
 		"-y",
+
+		// Output.
 		outpath,
 	); err != nil {
 		return "", err
@@ -100,10 +142,21 @@ func ffmpegGenerateStatic(ctx context.Context, filepath string) (string, error)
 	// Generate static with ffmpeg.
 	if err := ffmpeg(ctx, dirpath,
 		"-loglevel", "error",
+
+		// Input file.
 		"-i", filepath,
-		"-codec:v", "png", // specifically NOT 'apng'
-		"-frames:v", "1", // in case animated, only take 1 frame
+
+		// Only first frame.
+		"-frames:v", "1",
+
+		// Encode using png.
+		// (NOT as apng).
+		"-codec:v", "png",
+
+		// Overwrite.
 		"-y",
+
+		// Output.
 		outpath,
 	); err != nil {
 		return "", err

internal/media/manager_test.go

@@ -273,9 +273,10 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcess() {
 		Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,
 	}, attachment.FileMeta.Small)
 	suite.Equal("image/jpeg", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
 	suite.Equal(269739, attachment.File.FileSize)
-	suite.Equal("LjCGfG#6RkRn_NvzRjWF?urqV@a$", attachment.Blurhash)
+	suite.Equal(8536, attachment.Thumbnail.FileSize)
+	suite.Equal("LcBzLU#6RkRn~qvzRjWF?urqV@jc", attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -284,7 +285,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcess() {
 
 	// ensure the files contain the expected data.
 	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-jpeg-processed.jpg")
-	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.jpg")
+	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.webp")
 }
 
 func (suite *ManagerTestSuite) TestSimpleJpegProcessTooLarge() {
@@ -425,9 +426,10 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() {
 		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
 	}, attachment.FileMeta.Small)
 	suite.Equal("video/mp4", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
 	suite.Equal(312453, attachment.File.FileSize)
-	suite.Equal("LrJuJat6NZkBt7ayW.j[_4WBsWoL", attachment.Blurhash)
+	suite.Equal(3746, attachment.Thumbnail.FileSize)
+	suite.Equal("LhIrNMt6Nsj[t7aybFj[_4WBspoe", attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -436,7 +438,7 @@ func (suite *ManagerTestSuite) TestSlothVineProcess() {
 
 	// ensure the files contain the expected data.
 	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-mp4-processed.mp4")
-	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-mp4-thumbnail.jpg")
+	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-mp4-thumbnail.webp")
 }
 
 func (suite *ManagerTestSuite) TestLongerMp4Process() {
@@ -484,9 +486,10 @@ func (suite *ManagerTestSuite) TestLongerMp4Process() {
 		Width: 512, Height: 281, Size: 143872, Aspect: 1.822064,
 	}, attachment.FileMeta.Small)
 	suite.Equal("video/mp4", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
 	suite.Equal(109569, attachment.File.FileSize)
-	suite.Equal("LASY{q~qD%_3~qD%ofRjM{ofofRj", attachment.Blurhash)
+	suite.Equal(2128, attachment.Thumbnail.FileSize)
+	suite.Equal("L8Q0aP~qnM_3~qD%ozRjRiofWXRj", attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -495,7 +498,7 @@ func (suite *ManagerTestSuite) TestLongerMp4Process() {
 
 	// ensure the files contain the expected data.
 	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/longer-mp4-processed.mp4")
-	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/longer-mp4-thumbnail.jpg")
+	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/longer-mp4-thumbnail.webp")
 }
 
 func (suite *ManagerTestSuite) TestBirdnestMp4Process() {
@@ -543,9 +546,10 @@ func (suite *ManagerTestSuite) TestBirdnestMp4Process() {
 		Width: 287, Height: 512, Size: 146944, Aspect: 0.5605469,
 	}, attachment.FileMeta.Small)
 	suite.Equal("video/mp4", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
 	suite.Equal(1409625, attachment.File.FileSize)
-	suite.Equal("LOGb||RjRO.99DRORPaetkV?afMw", attachment.Blurhash)
+	suite.Equal(9446, attachment.Thumbnail.FileSize)
+	suite.Equal("LKF~w1RjRO.99DRORPaetkV?WCMw", attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -554,7 +558,7 @@ func (suite *ManagerTestSuite) TestBirdnestMp4Process() {
 
 	// ensure the files contain the expected data.
 	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/birdnest-processed.mp4")
-	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/birdnest-thumbnail.jpg")
+	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/birdnest-thumbnail.webp")
 }
 
 func (suite *ManagerTestSuite) TestOpusProcess() {
@@ -650,9 +654,10 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcess() {
 		Width: 186, Height: 187, Size: 34782, Aspect: 0.9946524064171123,
 	}, attachment.FileMeta.Small)
 	suite.Equal("image/png", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
 	suite.Equal(17471, attachment.File.FileSize)
-	suite.Equal("LDQJl?%i-?WG%go#RURP~of3~UxV", attachment.Blurhash)
+	suite.Equal(2630, attachment.Thumbnail.FileSize)
+	suite.Equal("LBOW$@%i-=aj%go#RSRP_1av~Tt2", attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -661,7 +666,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcess() {
 
 	// ensure the files contain the expected data.
 	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-png-noalphachannel-processed.png")
-	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-png-noalphachannel-thumbnail.jpg")
+	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-png-noalphachannel-thumbnail.webp")
 }
 
 func (suite *ManagerTestSuite) TestPngAlphaChannelProcess() {
@@ -705,9 +710,10 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcess() {
 		Width: 186, Height: 187, Size: 34782, Aspect: 0.9946524064171123,
 	}, attachment.FileMeta.Small)
 	suite.Equal("image/png", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
 	suite.Equal(18904, attachment.File.FileSize)
-	suite.Equal("LDQJl?%i-?WG%go#RURP~of3~UxV", attachment.Blurhash)
+	suite.Equal(2630, attachment.Thumbnail.FileSize)
+	suite.Equal("LBOW$@%i-=aj%go#RSRP_1av~Tt2", attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -716,7 +722,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcess() {
 
 	// ensure the files contain the expected data.
 	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-png-alphachannel-processed.png")
-	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-png-alphachannel-thumbnail.jpg")
+	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-png-alphachannel-thumbnail.webp")
 }
 
 func (suite *ManagerTestSuite) TestSimpleJpegProcessWithCallback() {
@@ -760,9 +766,10 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessWithCallback() {
 		Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,
 	}, attachment.FileMeta.Small)
 	suite.Equal("image/jpeg", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
 	suite.Equal(269739, attachment.File.FileSize)
-	suite.Equal("LjCGfG#6RkRn_NvzRjWF?urqV@a$", attachment.Blurhash)
+	suite.Equal(8536, attachment.Thumbnail.FileSize)
+	suite.Equal("LcBzLU#6RkRn~qvzRjWF?urqV@jc", attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -771,7 +778,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessWithCallback() {
 
 	// ensure the files contain the expected data.
 	equalFiles(suite.T(), suite.state.Storage, dbAttachment.File.Path, "./test/test-jpeg-processed.jpg")
-	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.jpg")
+	equalFiles(suite.T(), suite.state.Storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.webp")
 }
 
 func (suite *ManagerTestSuite) TestSimpleJpegProcessWithDiskStorage() {
@@ -837,9 +844,10 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessWithDiskStorage() {
 		Width: 512, Height: 288, Size: 147456, Aspect: 1.7777777777777777,
 	}, attachment.FileMeta.Small)
 	suite.Equal("image/jpeg", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal("image/webp", attachment.Thumbnail.ContentType)
 	suite.Equal(269739, attachment.File.FileSize)
-	suite.Equal("LjCGfG#6RkRn_NvzRjWF?urqV@a$", attachment.Blurhash)
+	suite.Equal(8536, attachment.Thumbnail.FileSize)
+	suite.Equal("LcBzLU#6RkRn~qvzRjWF?urqV@jc", attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -848,7 +856,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessWithDiskStorage() {
 
 	// ensure the files contain the expected data.
 	equalFiles(suite.T(), storage, dbAttachment.File.Path, "./test/test-jpeg-processed.jpg")
-	equalFiles(suite.T(), storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.jpg")
+	equalFiles(suite.T(), storage, dbAttachment.Thumbnail.Path, "./test/test-jpeg-thumbnail.webp")
 }
 
 func (suite *ManagerTestSuite) TestSmallSizedMediaTypeDetection_issue2263() {

internal/media/processingmedia.go

@@ -255,7 +255,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 			string(TypeAttachment),
 			string(SizeSmall),
 			p.media.ID,
-			"jpeg",
+			"webp",
 		)
 	}
 
@@ -309,7 +309,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 		string(TypeAttachment),
 		string(SizeSmall),
 		p.media.ID,
-		"jpeg",
+		"webp",
 	)
 
 	// Get mimetype for the file container
@@ -317,7 +317,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	p.media.File.ContentType = getMimeType(ext)
 
 	// Set the known thumbnail content type.
-	p.media.Thumbnail.ContentType = "image/jpeg"
+	p.media.Thumbnail.ContentType = "image/webp"
 
 	// We can now consider this cached.
 	p.media.Cached = util.Ptr(true)

internal/media/util.go

@@ -22,13 +22,15 @@ import (
 	"errors"
 	"fmt"
 	"image"
-	"image/jpeg"
 	"io"
 	"os"
 
+	"golang.org/x/image/webp"
+
 	"codeberg.org/gruf/go-bytesize"
 	"codeberg.org/gruf/go-iotools"
 	"codeberg.org/gruf/go-mimetypes"
+
 	"github.com/buckket/go-blurhash"
 	"github.com/disintegration/imaging"
 )
@@ -63,8 +65,8 @@ func thumbSize(width, height int) (int, int) {
 	}
 }
 
-// jpegDecode decodes the JPEG at filepath into parsed image.Image.
-func jpegDecode(filepath string) (image.Image, error) {
+// webpDecode decodes the WebP at filepath into parsed image.Image.
+func webpDecode(filepath string) (image.Image, error) {
 	// Open the file at given path.
 	file, err := os.Open(filepath)
 	if err != nil {
@@ -72,7 +74,7 @@ func jpegDecode(filepath string) (image.Image, error) {
 	}
 
 	// Decode image from file.
-	img, err := jpeg.Decode(file)
+	img, err := webp.Decode(file)
 
 	// Done with file.
 	_ = file.Close()
@@ -83,7 +85,7 @@ func jpegDecode(filepath string) (image.Image, error) {
 // generateBlurhash generates a blurhash for JPEG at filepath.
 func generateBlurhash(filepath string) (string, error) {
 	// Decode JPEG file at given path.
-	img, err := jpegDecode(filepath)
+	img, err := webpDecode(filepath)
 	if err != nil {
 		return "", err
 	}