[bugfix] determine mime-type to use during ffprobe evaluation stage, don't bother checking against file extension (#3506)

* determine mime-type to use during ffprobe evaluation stage, don't bother rechecking by file extension

* set mjpeg content-type

* fix up tests expecting differing default values

internal/api/client/instance/instancepatch_test.go

@@ -120,7 +120,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch1() {
         "image/apng",
         "audio/ogg",
         "video/ogg",
-        "audio/x-m4a",
+        "audio/mp4",
         "video/mp4",
         "video/quicktime",
         "audio/x-ms-wma",
@@ -261,7 +261,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch2() {
         "image/apng",
         "audio/ogg",
         "video/ogg",
-        "audio/x-m4a",
+        "audio/mp4",
         "video/mp4",
         "video/quicktime",
         "audio/x-ms-wma",
@@ -402,7 +402,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch3() {
         "image/apng",
         "audio/ogg",
         "video/ogg",
-        "audio/x-m4a",
+        "audio/mp4",
         "video/mp4",
         "video/quicktime",
         "audio/x-ms-wma",
@@ -594,7 +594,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch6() {
         "image/apng",
         "audio/ogg",
         "video/ogg",
-        "audio/x-m4a",
+        "audio/mp4",
         "video/mp4",
         "video/quicktime",
         "audio/x-ms-wma",
@@ -757,7 +757,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch8() {
         "image/apng",
         "audio/ogg",
         "video/ogg",
-        "audio/x-m4a",
+        "audio/mp4",
         "video/mp4",
         "video/quicktime",
         "audio/x-ms-wma",
@@ -939,7 +939,7 @@ func (suite *InstancePatchTestSuite) TestInstancePatch9() {
         "image/apng",
         "audio/ogg",
         "video/ogg",
-        "audio/x-m4a",
+        "audio/mp4",
         "video/mp4",
         "video/quicktime",
         "audio/x-ms-wma",

internal/media/ffmpeg.go

@@ -323,14 +323,14 @@ type videoStream struct {
 //
 // Note the checks for (len(res.video) > 0) may catch some audio files with embedded
 // album art as video, but i blame that on the hellscape that is media filetypes.
-//
-// TODO: we can update this code to also return a mimetype and avoid later parsing!
-func (res *result) GetFileType() (gtsmodel.FileType, string) {
+func (res *result) GetFileType() (gtsmodel.FileType, string, string) {
 	switch res.format {
 	case "mpeg":
-		return gtsmodel.FileTypeVideo, "mpeg"
+		return gtsmodel.FileTypeVideo,
+			"video/mpeg", "mpeg"
 	case "mjpeg":
-		return gtsmodel.FileTypeVideo, "mjpeg"
+		return gtsmodel.FileTypeVideo,
+			"video/x-motion-jpeg", "mjpeg"
 	case "mov,mp4,m4a,3gp,3g2,mj2":
 		switch {
 		case len(res.video) > 0:
@@ -338,55 +338,70 @@ func (res *result) GetFileType() (gtsmodel.FileType, string) {
 				res.duration <= 30 {
 				// Short, soundless
 				// video file aka gifv.
-				return gtsmodel.FileTypeGifv, "mp4"
+				return gtsmodel.FileTypeGifv,
+					"video/mp4", "mp4"
 			} else {
 				// Video file (with or without audio).
-				return gtsmodel.FileTypeVideo, "mp4"
+				return gtsmodel.FileTypeVideo,
+					"video/mp4", "mp4"
 			}
 		case len(res.audio) > 0 &&
 			res.audio[0].codec == "aac":
 			// m4a only supports [aac] audio.
-			return gtsmodel.FileTypeAudio, "m4a"
+			return gtsmodel.FileTypeAudio,
+				"audio/mp4", "m4a"
 		}
 	case "apng":
-		return gtsmodel.FileTypeImage, "apng"
+		return gtsmodel.FileTypeImage,
+			"image/apng", "apng"
 	case "png_pipe":
-		return gtsmodel.FileTypeImage, "png"
+		return gtsmodel.FileTypeImage,
+			"image/png", "png"
 	case "image2", "image2pipe", "jpeg_pipe":
-		return gtsmodel.FileTypeImage, "jpeg"
+		return gtsmodel.FileTypeImage,
+			"image/jpeg", "jpeg"
 	case "webp", "webp_pipe":
-		return gtsmodel.FileTypeImage, "webp"
+		return gtsmodel.FileTypeImage,
+			"image/webp", "webp"
 	case "gif":
-		return gtsmodel.FileTypeImage, "gif"
+		return gtsmodel.FileTypeImage,
+			"image/gif", "gif"
 	case "mp3":
 		if len(res.audio) > 0 {
 			switch res.audio[0].codec {
 			case "mp2":
-				return gtsmodel.FileTypeAudio, "mp2"
+				return gtsmodel.FileTypeAudio,
+					"audio/mp2", "mp2"
 			case "mp3":
-				return gtsmodel.FileTypeAudio, "mp3"
+				return gtsmodel.FileTypeAudio,
+					"audio/mp3", "mp3"
 			}
 		}
 	case "asf":
 		switch {
 		case len(res.video) > 0:
-			return gtsmodel.FileTypeVideo, "wmv"
+			return gtsmodel.FileTypeVideo,
+				"video/x-ms-wmv", "wmv"
 		case len(res.audio) > 0:
-			return gtsmodel.FileTypeAudio, "wma"
+			return gtsmodel.FileTypeAudio,
+				"audio/x-ms-wma", "wma"
 		}
 	case "ogg":
 		if len(res.video) > 0 {
 			switch res.video[0].codec {
 			case "theora", "dirac": // daala, tarkin
-				return gtsmodel.FileTypeVideo, "ogv"
+				return gtsmodel.FileTypeVideo,
+					"video/ogg", "ogv"
 			}
 		}
 		if len(res.audio) > 0 {
 			switch res.audio[0].codec {
 			case "opus", "libopus":
-				return gtsmodel.FileTypeAudio, "opus"
+				return gtsmodel.FileTypeAudio,
+					"audio/opus", "opus"
 			default:
-				return gtsmodel.FileTypeAudio, "ogg"
+				return gtsmodel.FileTypeAudio,
+					"audio/ogg", "ogg"
 			}
 		}
 	case "matroska,webm":
@@ -411,21 +426,27 @@ func (res *result) GetFileType() (gtsmodel.FileType, string) {
 			}
 
 			if isWebm {
-				// Check for valid webm codec config.
-				return gtsmodel.FileTypeVideo, "webm"
+				// Check valid webm codec config.
+				return gtsmodel.FileTypeVideo,
+					"video/webm", "webm"
 			}
 
 			// All else falls under generic mkv.
-			return gtsmodel.FileTypeVideo, "mkv"
+			return gtsmodel.FileTypeVideo,
+				"video/x-matroska", "mkv"
 		case len(res.audio) > 0:
-			return gtsmodel.FileTypeAudio, "mka"
+			return gtsmodel.FileTypeAudio,
+				"audio/x-matroska", "mka"
 		}
 	case "avi":
-		return gtsmodel.FileTypeVideo, "avi"
+		return gtsmodel.FileTypeVideo,
+			"video/x-msvideo", "avi"
 	case "flac":
-		return gtsmodel.FileTypeAudio, "flac"
+		return gtsmodel.FileTypeAudio,
+			"audio/flac", "flac"
 	}
-	return gtsmodel.FileTypeUnknown, res.format
+	return gtsmodel.FileTypeUnknown,
+		"", res.format
 }
 
 // ImageMeta extracts image metadata contained within ffprobe'd media result streams.

internal/media/manager.go

@@ -56,7 +56,7 @@ var SupportedMIMETypes = []string{
 	"video/ogg", // .ogv
 
 	// mpeg4 types
-	"audio/x-m4a",     // .m4a
+	"audio/mp4",       // .m4a
 	"video/mp4",       // .mp4
 	"video/quicktime", // .mov
 

internal/media/manager_test.go

@@ -664,7 +664,7 @@ func (suite *ManagerTestSuite) TestOpusProcess() {
 		Duration: util.Ptr(float32(122.10006)),
 		Bitrate:  util.Ptr(uint64(116426)),
 	}, attachment.FileMeta.Original)
-	suite.Equal("audio/ogg", attachment.File.ContentType)
+	suite.Equal("audio/opus", attachment.File.ContentType)
 	suite.Equal(1776956, attachment.File.FileSize)
 	suite.Empty(attachment.Blurhash)
 

internal/media/processingemoji.go

@@ -163,9 +163,10 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 	}
 
 	var ext string
+	var fileType gtsmodel.FileType
 
-	// Get type from ffprobe format data.
-	fileType, ext := result.GetFileType()
+	// Get abstract file type, mimetype and ext from ffprobe data.
+	fileType, p.emoji.ImageContentType, ext = result.GetFileType()
 	if fileType != gtsmodel.FileTypeImage {
 		return gtserror.Newf("unsupported emoji filetype: %s (%s)", fileType, ext)
 	}
@@ -216,10 +217,6 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 		"png",
 	)
 
-	// Get mimetype for the file container
-	// type, falling back to generic data.
-	p.emoji.ImageContentType = getMimeType(ext)
-
 	// Set the known emoji static content type.
 	p.emoji.ImageStaticContentType = "image/png"
 

internal/media/processingmedia.go

@@ -186,8 +186,8 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	p.media.FileMeta.Original.Duration = util.PtrIf(float32(result.duration))
 	p.media.FileMeta.Original.Bitrate = util.PtrIf(result.bitrate)
 
-	// Set media type from ffprobe format data.
-	p.media.Type, ext = result.GetFileType()
+	// Set generic media type and mimetype from ffprobe format data.
+	p.media.Type, p.media.File.ContentType, ext = result.GetFileType()
 
 	// Add file extension to path.
 	newpath := temppath + "." + ext
@@ -236,10 +236,10 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 
 		// Determine if blurhash needs generating.
 		needBlurhash := (p.media.Blurhash == "")
-		var newBlurhash string
+		var newBlurhash, mimeType string
 
-		// Generate thumbnail, and new blurhash if need from media.
-		thumbpath, newBlurhash, err = generateThumb(ctx, temppath,
+		// Generate thumbnail, and new blurhash if needed from temp media.
+		thumbpath, mimeType, newBlurhash, err = generateThumb(ctx, temppath,
 			thumbWidth,
 			thumbHeight,
 			result.orientation,
@@ -250,6 +250,9 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 			return gtserror.Newf("error generating image thumb: %w", err)
 		}
 
+		// Set generated thumbnail's mimetype.
+		p.media.Thumbnail.ContentType = mimeType
+
 		if needBlurhash {
 			// Set newly determined blurhash.
 			p.media.Blurhash = newBlurhash
@@ -265,10 +268,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 		ext,
 	)
 
-	// Get mimetype for the file container
-	// type, falling back to generic data.
-	p.media.File.ContentType = getMimeType(ext)
-
 	// Copy temporary file into storage at path.
 	filesz, err := p.mgr.state.Storage.PutFile(ctx,
 		p.media.File.Path,
@@ -295,9 +294,6 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 			thumbExt,
 		)
 
-		// Determine thumbnail content-type from thumb ext.
-		p.media.Thumbnail.ContentType = getMimeType(thumbExt)
-
 		// Copy thumbnail file into storage at path.
 		thumbsz, err := p.mgr.state.Storage.PutFile(ctx,
 			p.media.Thumbnail.Path,

internal/media/thumbnail.go

@@ -84,17 +84,21 @@ func generateThumb(
 	needBlurhash bool,
 ) (
 	outpath string,
+	mimeType string,
 	blurhash string,
 	err error,
 ) {
 	var ext string
 
+	// Default type is webp.
+	mimeType = "image/webp"
+
 	// Generate thumb output path REPLACING extension.
 	if i := strings.IndexByte(filepath, '.'); i != -1 {
 		outpath = filepath[:i] + "_thumb.webp"
 		ext = filepath[i+1:] // old extension
 	} else {
-		return "", "", gtserror.New("input file missing extension")
+		return "", "", "", gtserror.New("input file missing extension")
 	}
 
 	// Check for the few media types we
@@ -106,6 +110,7 @@ func generateThumb(
 		// Replace the "webp" with "jpeg", as we'll
 		// use our native Go thumbnailing generation.
 		outpath = outpath[:len(outpath)-4] + "jpeg"
+		mimeType = "image/jpeg"
 
 		log.Debug(ctx, "generating thumb from jpeg")
 		blurhash, err := generateNativeThumb(
@@ -117,7 +122,7 @@ func generateThumb(
 			jpeg.Decode,
 			needBlurhash,
 		)
-		return outpath, blurhash, err
+		return outpath, mimeType, blurhash, err
 
 	// We specifically only allow generating native
 	// thumbnails from gif IF it doesn't contain an
@@ -128,6 +133,7 @@ func generateThumb(
 		// Replace the "webp" with "jpeg", as we'll
 		// use our native Go thumbnailing generation.
 		outpath = outpath[:len(outpath)-4] + "jpeg"
+		mimeType = "image/jpeg"
 
 		log.Debug(ctx, "generating thumb from gif")
 		blurhash, err := generateNativeThumb(
@@ -139,7 +145,7 @@ func generateThumb(
 			gif.Decode,
 			needBlurhash,
 		)
-		return outpath, blurhash, err
+		return outpath, mimeType, blurhash, err
 
 	// We specifically only allow generating native
 	// thumbnails from png IF it doesn't contain an
@@ -150,6 +156,7 @@ func generateThumb(
 		// Replace the "webp" with "jpeg", as we'll
 		// use our native Go thumbnailing generation.
 		outpath = outpath[:len(outpath)-4] + "jpeg"
+		mimeType = "image/jpeg"
 
 		log.Debug(ctx, "generating thumb from png")
 		blurhash, err := generateNativeThumb(
@@ -161,7 +168,7 @@ func generateThumb(
 			png.Decode,
 			needBlurhash,
 		)
-		return outpath, blurhash, err
+		return outpath, mimeType, blurhash, err
 
 	// We specifically only allow generating native
 	// thumbnails from webp IF it doesn't contain an
@@ -172,6 +179,7 @@ func generateThumb(
 		// Replace the "webp" with "jpeg", as we'll
 		// use our native Go thumbnailing generation.
 		outpath = outpath[:len(outpath)-4] + "jpeg"
+		mimeType = "image/jpeg"
 
 		log.Debug(ctx, "generating thumb from webp")
 		blurhash, err := generateNativeThumb(
@@ -183,7 +191,7 @@ func generateThumb(
 			webp.Decode,
 			needBlurhash,
 		)
-		return outpath, blurhash, err
+		return outpath, mimeType, blurhash, err
 	}
 
 	// The fallback for thumbnail generation, which
@@ -196,18 +204,18 @@ func generateThumb(
 		height,
 		pixfmt,
 	); err != nil {
-		return outpath, "", err
+		return outpath, "", "", err
 	}
 
 	if needBlurhash {
 		// Generate new blurhash from webp output thumb.
 		blurhash, err = generateWebpBlurhash(outpath)
 		if err != nil {
-			return outpath, "", gtserror.Newf("error generating blurhash: %w", err)
+			return outpath, "", "", gtserror.Newf("error generating blurhash: %w", err)
 		}
 	}
 
-	return outpath, blurhash, err
+	return outpath, mimeType, blurhash, nil
 }
 
 // generateNativeThumb generates a thumbnail

internal/media/util.go

@@ -18,7 +18,6 @@
 package media
 
 import (
-	"cmp"
 	"errors"
 	"fmt"
 	"io"
@@ -28,7 +27,6 @@ import (
 
 	"codeberg.org/gruf/go-bytesize"
 	"codeberg.org/gruf/go-iotools"
-	"codeberg.org/gruf/go-mimetypes"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 )
 
@@ -87,12 +85,6 @@ func getExtension(path string) string {
 	return ""
 }
 
-// getMimeType returns a suitable mimetype for file extension.
-func getMimeType(ext string) string {
-	const defaultType = "application/octet-stream"
-	return cmp.Or(mimetypes.MimeTypes[ext], defaultType)
-}
-
 // drainToTmp drains data from given reader into a new temp file
 // and closes it, returning the path of the resulting temp file.
 //

internal/typeutils/internaltofrontend_test.go

@@ -1958,7 +1958,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV1ToFrontend() {
         "image/apng",
         "audio/ogg",
         "video/ogg",
-        "audio/x-m4a",
+        "audio/mp4",
         "video/mp4",
         "video/quicktime",
         "audio/x-ms-wma",
@@ -2103,7 +2103,7 @@ func (suite *InternalToFrontendTestSuite) TestInstanceV2ToFrontend() {
         "image/apng",
         "audio/ogg",
         "video/ogg",
-        "audio/x-m4a",
+        "audio/mp4",
         "video/mp4",
         "video/quicktime",
         "audio/x-ms-wma",