[chore] media pipeline improvements (#3110)

* don't set emoji / media image paths on failed download, migrate FileType from string to integer

* fix incorrect uses of util.PtrOr, fix returned frontend media

* fix migration not setting arguments correctly in where clause

* fix not providing default with not null column

* whoops

* ensure a default gets set for media attachment file type

* remove the exclusive flag from writing files in disk storage

* rename PtrOr -> PtrOrZero, and rename PtrValueOr -> PtrOrValue to match

* slight wording changes

* use singular / plural word forms (no parentheses), is better for screen readers

* update testmodels with unknown media type to have unset file details, update attachment focus handling converting to frontend, update tests

* store first instance in ffmpeg wasm pool, fill remaining with closed instances

internal/media/ffmpeg/pool.go

@@ -34,14 +34,33 @@ type wasmInstancePool struct {
 }
 
 func (p *wasmInstancePool) Init(ctx context.Context, sz int) error {
-	p.pool = make(chan *wasm.Instance, sz)
-	for i := 0; i < sz; i++ {
-		inst, err := p.inst.New(ctx)
-		if err != nil {
-			return err
-		}
-		p.pool <- inst
+	// Initialize for first time
+	// to preload module into the
+	// wazero compilation cache.
+	inst, err := p.inst.New(ctx)
+	if err != nil {
+		return err
 	}
+
+	// Clamp to 1.
+	if sz <= 0 {
+		sz = 1
+	}
+
+	// Allocate new pool instance channel.
+	p.pool = make(chan *wasm.Instance, sz)
+
+	// Store only one
+	// open instance
+	// at init time.
+	p.pool <- inst
+
+	// Fill reminaing with closed
+	// instances for later opening.
+	for i := 0; i < sz-1; i++ {
+		p.pool <- new(wasm.Instance)
+	}
+
 	return nil
 }
 

internal/media/manager.go

@@ -102,74 +102,19 @@ func (m *Manager) CreateMedia(
 ) {
 	now := time.Now()
 
-	// Generate new ID.
-	id := id.NewULID()
-
-	// Placeholder URL for attachment.
-	url := uris.URIForAttachment(
-		accountID,
-		string(TypeAttachment),
-		string(SizeOriginal),
-		id,
-		"unknown",
-	)
-
-	// Placeholder storage path for attachment.
-	path := uris.StoragePathForAttachment(
-		accountID,
-		string(TypeAttachment),
-		string(SizeOriginal),
-		id,
-		"unknown",
-	)
-
-	// Calculate attachment thumbnail file path
-	thumbPath := uris.StoragePathForAttachment(
-		accountID,
-		string(TypeAttachment),
-		string(SizeSmall),
-		id,
-
-		// Always encode attachment
-		// thumbnails as jpeg.
-		"jpeg",
-	)
-
-	// Calculate attachment thumbnail URL.
-	thumbURL := uris.URIForAttachment(
-		accountID,
-		string(TypeAttachment),
-		string(SizeSmall),
-		id,
-
-		// Always encode attachment
-		// thumbnails as jpeg.
-		"jpeg",
-	)
-
 	// Populate initial fields on the new media,
 	// leaving out fields with values we don't know
 	// yet. These will be overwritten as we go.
 	attachment := &gtsmodel.MediaAttachment{
-		ID:         id,
+		ID:         id.NewULID(),
+		AccountID:  accountID,
+		Type:       gtsmodel.FileTypeUnknown,
+		Processing: gtsmodel.ProcessingStatusReceived,
+		Avatar:     util.Ptr(false),
+		Header:     util.Ptr(false),
+		Cached:     util.Ptr(false),
 		CreatedAt:  now,
 		UpdatedAt:  now,
-		URL:        url,
-		Type:       gtsmodel.FileTypeUnknown,
-		AccountID:  accountID,
-		Processing: gtsmodel.ProcessingStatusReceived,
-		File: gtsmodel.File{
-			ContentType: "application/octet-stream",
-			Path:        path,
-		},
-		Thumbnail: gtsmodel.Thumbnail{
-			ContentType: "image/jpeg",
-			Path:        thumbPath,
-			URL:         thumbURL,
-		},
-		Avatar: util.Ptr(false),
-		Header: util.Ptr(false),
-		Cached: util.Ptr(false),
 	}
 
 	// Check if we were provided additional info
@@ -252,56 +197,23 @@ func (m *Manager) CreateEmoji(
 	// Generate new ID.
 	id := id.NewULID()
 
-	// Fetch the local instance account for emoji path generation.
-	instanceAcc, err := m.state.DB.GetInstanceAccount(ctx, "")
-	if err != nil {
-		return nil, gtserror.Newf("error fetching instance account: %w", err)
-	}
-
 	if domain == "" && info.URI == nil {
 		// Generate URI for local emoji.
 		uri := uris.URIForEmoji(id)
 		info.URI = &uri
 	}
 
-	// Generate static URL for attachment.
-	staticURL := uris.URIForAttachment(
-		instanceAcc.ID,
-		string(TypeEmoji),
-		string(SizeStatic),
-		id,
-
-		// All static emojis
-		// are encoded as png.
-		"png",
-	)
-
-	// Generate static image path for attachment.
-	staticPath := uris.StoragePathForAttachment(
-		instanceAcc.ID,
-		string(TypeEmoji),
-		string(SizeStatic),
-		id,
-
-		// All static emojis
-		// are encoded as png.
-		"png",
-	)
-
 	// Populate initial fields on the new emoji,
 	// leaving out fields with values we don't know
 	// yet. These will be overwritten as we go.
 	emoji := &gtsmodel.Emoji{
-		ID:                     id,
-		Shortcode:              shortcode,
-		Domain:                 domain,
-		ImageStaticURL:         staticURL,
-		ImageStaticPath:        staticPath,
-		ImageStaticContentType: "image/png",
-		Disabled:               util.Ptr(false),
-		VisibleInPicker:        util.Ptr(true),
-		CreatedAt:              now,
-		UpdatedAt:              now,
+		ID:              id,
+		Shortcode:       shortcode,
+		Domain:          domain,
+		Disabled:        util.Ptr(false),
+		VisibleInPicker: util.Ptr(true),
+		CreatedAt:       now,
+		UpdatedAt:       now,
 	}
 
 	// Finally, create new emoji.
@@ -327,12 +239,6 @@ func (m *Manager) RefreshEmoji(
 	*ProcessingEmoji,
 	error,
 ) {
-	// Fetch the local instance account for emoji path generation.
-	instanceAcc, err := m.state.DB.GetInstanceAccount(ctx, "")
-	if err != nil {
-		return nil, gtserror.Newf("error fetching instance account: %w", err)
-	}
-
 	// Create references to old emoji image
 	// paths before they get updated with new
 	// path ID. These are required for later
@@ -380,38 +286,6 @@ func (m *Manager) RefreshEmoji(
 		return rct, nil
 	}
 
-	// Use a new ID to create a new path
-	// for the new images, to get around
-	// needing to do cache invalidation.
-	newPathID, err := id.NewRandomULID()
-	if err != nil {
-		return nil, gtserror.Newf("error generating newPathID for emoji refresh: %s", err)
-	}
-
-	// Generate new static URL for emoji.
-	emoji.ImageStaticURL = uris.URIForAttachment(
-		instanceAcc.ID,
-		string(TypeEmoji),
-		string(SizeStatic),
-		newPathID,
-
-		// All static emojis
-		// are encoded as png.
-		"png",
-	)
-
-	// Generate new static image storage path for emoji.
-	emoji.ImageStaticPath = uris.StoragePathForAttachment(
-		instanceAcc.ID,
-		string(TypeEmoji),
-		string(SizeStatic),
-		newPathID,
-
-		// All static emojis
-		// are encoded as png.
-		"png",
-	)
-
 	// Finally, create new emoji in database.
 	processingEmoji, err := m.createEmoji(ctx,
 		func(ctx context.Context, emoji *gtsmodel.Emoji) error {
@@ -425,8 +299,8 @@ func (m *Manager) RefreshEmoji(
 		return nil, err
 	}
 
-	// Set the refreshed path ID used.
-	processingEmoji.newPathID = newPathID
+	// Generate a new path ID to use instead.
+	processingEmoji.newPathID = id.NewULID()
 
 	return processingEmoji, nil
 }
@@ -441,6 +315,12 @@ func (m *Manager) createEmoji(
 	*ProcessingEmoji,
 	error,
 ) {
+	// Fetch the local instance account for emoji path generation.
+	instanceAcc, err := m.state.DB.GetInstanceAccount(ctx, "")
+	if err != nil {
+		return nil, gtserror.Newf("error fetching instance account: %w", err)
+	}
+
 	// Check if we have additional info to add to the emoji,
 	// and overwrite some of the emoji fields if so.
 	if info.URI != nil {
@@ -475,9 +355,10 @@ func (m *Manager) createEmoji(
 
 	// Return wrapped emoji for later processing.
 	processingEmoji := &ProcessingEmoji{
-		emoji:  emoji,
-		dataFn: data,
-		mgr:    m,
+		instAccID: instanceAcc.ID,
+		emoji:     emoji,
+		dataFn:    data,
+		mgr:       m,
 	}
 
 	return processingEmoji, nil

internal/media/manager_test.go

@@ -358,11 +358,10 @@ func (suite *ManagerTestSuite) TestPDFProcess() {
 	suite.Equal(processing.ID(), attachment.ID)
 	suite.Equal(accountID, attachment.AccountID)
 
-	// file meta should be correctly derived from the image
 	suite.Zero(attachment.FileMeta)
-	suite.Equal("application/octet-stream", attachment.File.ContentType)
-	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
-	suite.Empty(attachment.Blurhash)
+	suite.Zero(attachment.File.ContentType)
+	suite.Zero(attachment.Thumbnail.ContentType)
+	suite.Zero(attachment.Blurhash)
 
 	// now make sure the attachment is in the database
 	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachment.ID)
@@ -376,7 +375,6 @@ func (suite *ManagerTestSuite) TestPDFProcess() {
 	stored, err := suite.storage.Has(ctx, attachment.File.Path)
 	suite.NoError(err)
 	suite.False(stored)
-
 	stored, err = suite.storage.Has(ctx, attachment.Thumbnail.Path)
 	suite.NoError(err)
 	suite.False(stored)

internal/media/processingemoji.go

@@ -26,7 +26,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
-	"github.com/superseriousbusiness/gotosocial/internal/regexes"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
@@ -36,6 +35,7 @@ import (
 // various functions for retrieving data from the process.
 type ProcessingEmoji struct {
 	emoji     *gtsmodel.Emoji   // processing emoji details
+	instAccID string            // instance account ID
 	newPathID string            // new emoji path ID to use when being refreshed
 	dataFn    DataFunc          // load-data function, returns media stream
 	done      bool              // done is set when process finishes with non ctx canceled type error
@@ -191,21 +191,24 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 		pathID = p.emoji.ID
 	}
 
-	// Determine instance account ID from generated image static path.
-	instanceAccID, ok := getInstanceAccountID(p.emoji.ImageStaticPath)
-	if !ok {
-		return gtserror.Newf("invalid emoji static path; no instance account id: %s", p.emoji.ImageStaticPath)
-	}
-
-	// Calculate final media attachment file path.
+	// Calculate final emoji media file path.
 	p.emoji.ImagePath = uris.StoragePathForAttachment(
-		instanceAccID,
+		p.instAccID,
 		string(TypeEmoji),
 		string(SizeOriginal),
 		pathID,
 		ext,
 	)
 
+	// Calculate final emoji static media file path.
+	p.emoji.ImageStaticPath = uris.StoragePathForAttachment(
+		p.instAccID,
+		string(TypeEmoji),
+		string(SizeStatic),
+		pathID,
+		"png",
+	)
+
 	// Copy temporary file into storage at path.
 	filesz, err := p.mgr.state.Storage.PutFile(ctx,
 		p.emoji.ImagePath,
@@ -228,19 +231,31 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 	p.emoji.ImageFileSize = int(filesz)
 	p.emoji.ImageStaticFileSize = int(staticsz)
 
-	// Fill in remaining emoji data now it's stored.
+	// Generate an emoji media static URL.
 	p.emoji.ImageURL = uris.URIForAttachment(
-		instanceAccID,
+		p.instAccID,
 		string(TypeEmoji),
 		string(SizeOriginal),
 		pathID,
 		ext,
 	)
 
+	// Generate an emoji image static URL.
+	p.emoji.ImageStaticURL = uris.URIForAttachment(
+		p.instAccID,
+		string(TypeEmoji),
+		string(SizeStatic),
+		pathID,
+		"png",
+	)
+
 	// Get mimetype for the file container
 	// type, falling back to generic data.
 	p.emoji.ImageContentType = getMimeType(ext)
 
+	// Set the known emoji static content type.
+	p.emoji.ImageStaticContentType = "image/png"
+
 	// We can now consider this cached.
 	p.emoji.Cached = util.Ptr(true)
 
@@ -268,16 +283,16 @@ func (p *ProcessingEmoji) cleanup(ctx context.Context) {
 		}
 	}
 
+	// Unset processor-calculated fields.
+	p.emoji.ImageStaticContentType = ""
+	p.emoji.ImageStaticFileSize = 0
+	p.emoji.ImageStaticPath = ""
+	p.emoji.ImageStaticURL = ""
+	p.emoji.ImageContentType = ""
+	p.emoji.ImageFileSize = 0
+	p.emoji.ImagePath = ""
+	p.emoji.ImageURL = ""
+
 	// Ensure marked as not cached.
 	p.emoji.Cached = util.Ptr(false)
 }
-
-// getInstanceAccountID determines the instance account ID from
-// emoji static image storage path. returns false on failure.
-func getInstanceAccountID(staticPath string) (string, bool) {
-	matches := regexes.FilePath.FindStringSubmatch(staticPath)
-	if len(matches) < 2 {
-		return "", false
-	}
-	return matches[1], true
-}

internal/media/processingmedia.go

@@ -248,6 +248,15 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 				return gtserror.Newf("error generating thumb blurhash: %w", err)
 			}
 		}
+
+		// Calculate final media attachment thumbnail path.
+		p.media.Thumbnail.Path = uris.StoragePathForAttachment(
+			p.media.AccountID,
+			string(TypeAttachment),
+			string(SizeSmall),
+			p.media.ID,
+			"jpeg",
+		)
 	}
 
 	// Calculate final media attachment file path.
@@ -285,8 +294,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 		p.media.Thumbnail.FileSize = int(thumbsz)
 	}
 
-	// Fill in correct attachment
-	// data now we've parsed it.
+	// Generate a media attachment URL.
 	p.media.URL = uris.URIForAttachment(
 		p.media.AccountID,
 		string(TypeAttachment),
@@ -295,10 +303,22 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 		ext,
 	)
 
+	// Generate a media attachment thumbnail URL.
+	p.media.Thumbnail.URL = uris.URIForAttachment(
+		p.media.AccountID,
+		string(TypeAttachment),
+		string(SizeSmall),
+		p.media.ID,
+		"jpeg",
+	)
+
 	// Get mimetype for the file container
 	// type, falling back to generic data.
 	p.media.File.ContentType = getMimeType(ext)
 
+	// Set the known thumbnail content type.
+	p.media.Thumbnail.ContentType = "image/jpeg"
+
 	// We can now consider this cached.
 	p.media.Cached = util.Ptr(true)
 
@@ -329,6 +349,18 @@ func (p *ProcessingMedia) cleanup(ctx context.Context) {
 		}
 	}
 
+	// Unset all processor-calculated media fields.
+	p.media.FileMeta.Original = gtsmodel.Original{}
+	p.media.FileMeta.Small = gtsmodel.Small{}
+	p.media.File.ContentType = ""
+	p.media.File.FileSize = 0
+	p.media.File.Path = ""
+	p.media.Thumbnail.FileSize = 0
+	p.media.Thumbnail.ContentType = ""
+	p.media.Thumbnail.Path = ""
+	p.media.Thumbnail.URL = ""
+	p.media.URL = ""
+
 	// Also ensure marked as unknown and finished
 	// processing so gets inserted as placeholder URL.
 	p.media.Processing = gtsmodel.ProcessingStatusProcessed