[bugfix] media.Processor{}.GetFile() returning 404s on first call, correctly loading on 2nd (#3129)

* refactor file handling a tiny bit

* whoops

* make processing media / emoji defers a bit clear to see that it's the "on finished processing" path

* some wording

* add some debug logging

* add mutex locks for processing remote media

* try removing freshness check

* fix derefMedia not being allocated

* fix log format string

* handle case of empty file paths (i.e. not stored)

* remove media / emoji once finished processing from dereferencer maps

* whoops, fix the cached / force checks

* move url parsing outside of 'process___Safely()' funcs to prevalidate url

* use emoji.ShortcodeDomain()

* update RefreshEmoji() to also match RefreshMedia() changes

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>

internal/cleaner/cleaner.go

@@ -59,12 +59,19 @@ func (c *Cleaner) Media() *Media {
 
 // haveFiles returns whether all of the provided files exist within current storage.
 func (c *Cleaner) haveFiles(ctx context.Context, files ...string) (bool, error) {
-	for _, file := range files {
+	for _, path := range files {
+		if path == "" {
+			// File not stored.
+			return false, nil
+		}
+
 		// Check whether each file exists in storage.
-		have, err := c.state.Storage.Has(ctx, file)
+		have, err := c.state.Storage.Has(ctx, path)
 		if err != nil {
-			return false, gtserror.Newf("error checking storage for %s: %w", file, err)
-		} else if !have {
+			return false, gtserror.Newf("error checking storage for %s: %w", path, err)
+		}
+
+		if !have {
 			// Missing file(s).
 			return false, nil
 		}
@@ -80,29 +87,34 @@ func (c *Cleaner) removeFiles(ctx context.Context, files ...string) (int, error)
 	}
 
 	var (
-		errs     gtserror.MultiError
-		errCount int
+		errs  gtserror.MultiError
+		count int
 	)
 
 	for _, path := range files {
+		if path == "" {
+			// not stored.
+			continue
+		}
+
 		// Remove each provided storage path.
 		log.Debugf(ctx, "removing file: %s", path)
 		err := c.state.Storage.Delete(ctx, path)
 		if err != nil && !storage.IsNotFound(err) {
 			errs.Appendf("error removing %s: %w", path, err)
-			errCount++
+			continue
 		}
-	}
 
-	// Calculate no. files removed.
-	diff := len(files) - errCount
+		// Incr.
+		count++
+	}
 
 	// Wrap the combined error slice.
 	if err := errs.Combine(); err != nil {
-		return diff, gtserror.Newf("error(s) removing files: %w", err)
+		return count, gtserror.Newf("error(s) removing files: %w", err)
 	}
 
-	return diff, nil
+	return count, nil
 }
 
 // ScheduleJobs schedules cleaning

internal/cleaner/media.go

@@ -97,7 +97,7 @@ func (m *Media) PruneOrphaned(ctx context.Context) (int, error) {
 	if err := m.state.Storage.WalkKeys(ctx, func(path string) error {
 		// Check for our expected fileserver path format.
 		if !regexes.FilePath.MatchString(path) {
-			log.Warn(ctx, "unexpected storage item: %s", path)
+			log.Warnf(ctx, "unexpected storage item: %s", path)
 			return nil
 		}