[performance] update storage backend and make use of seek syscall when available (#2924)

* update to use go-storage/ instead of go-store/v2/storage/

* pull in latest version from codeberg

* remove test output 😇

* add code comments

* set the exclusive bit when creating new files in disk config

* bump to actual release version

* bump to v0.1.1 (tis a simple no-logic change)

* update readme

* only use a temporary read seeker when decoding video if required (should only be S3 now)

* use fastcopy library to use memory pooled buffers when calling TempFileSeeker()

* update to use seek call in serveFileRange()

internal/media/manager.go

@@ -19,17 +19,16 @@ package media
 
 import (
 	"context"
-	"errors"
 	"io"
 	"time"
 
 	"codeberg.org/gruf/go-iotools"
-	"codeberg.org/gruf/go-store/v2/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
@@ -260,11 +259,11 @@ func (m *Manager) PreProcessEmoji(
 
 			// Wrap closer to cleanup old data.
 			c := iotools.CloserCallback(rc, func() {
-				if err := m.state.Storage.Delete(ctx, originalImagePath); err != nil && !errors.Is(err, storage.ErrNotFound) {
+				if err := m.state.Storage.Delete(ctx, originalImagePath); err != nil && !storage.IsNotFound(err) {
 					log.Errorf(ctx, "error removing old emoji %s@%s from storage: %v", emoji.Shortcode, emoji.Domain, err)
 				}
 
-				if err := m.state.Storage.Delete(ctx, originalImageStaticPath); err != nil && !errors.Is(err, storage.ErrNotFound) {
+				if err := m.state.Storage.Delete(ctx, originalImageStaticPath); err != nil && !storage.IsNotFound(err) {
 					log.Errorf(ctx, "error removing old static emoji %s@%s from storage: %v", emoji.Shortcode, emoji.Domain, err)
 				}
 			})

internal/media/manager_test.go

@@ -23,15 +23,15 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path"
 	"testing"
 	"time"
 
-	"codeberg.org/gruf/go-store/v2/storage"
+	"codeberg.org/gruf/go-storage/disk"
 	"github.com/stretchr/testify/suite"
 	gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/storage"
 	gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
@@ -189,9 +189,9 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingRefresh() {
 
 	// the old image files should no longer be in storage
 	_, err = suite.storage.Get(ctx, oldEmojiImagePath)
-	suite.ErrorIs(err, storage.ErrNotFound)
+	suite.True(storage.IsNotFound(err))
 	_, err = suite.storage.Get(ctx, oldEmojiImageStaticPath)
-	suite.ErrorIs(err, storage.ErrNotFound)
+	suite.True(storage.IsNotFound(err))
 }
 
 func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() {
@@ -1189,9 +1189,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
 	temp := fmt.Sprintf("%s/gotosocial-test", os.TempDir())
 	defer os.RemoveAll(temp)
 
-	disk, err := storage.OpenDisk(temp, &storage.DiskConfig{
-		LockFile: path.Join(temp, "store.lock"),
-	})
+	disk, err := disk.Open(temp, nil)
 	if err != nil {
 		panic(err)
 	}

internal/media/processingmedia.go

@@ -20,7 +20,6 @@ package media
 import (
 	"bytes"
 	"context"
-	"errors"
 	"image/jpeg"
 	"io"
 	"time"
@@ -156,7 +155,7 @@ func (p *ProcessingMedia) load(ctx context.Context) (*gtsmodel.MediaAttachment,
 		// never decoded). Try to clean up in this case.
 		if p.media.Type == gtsmodel.FileTypeUnknown {
 			deleteErr := p.mgr.state.Storage.Delete(ctx, p.media.File.Path)
-			if deleteErr != nil && !errors.Is(deleteErr, storage.ErrNotFound) {
+			if deleteErr != nil && !storage.IsNotFound(deleteErr) {
 				errs.Append(deleteErr)
 			}
 		}

internal/media/video.go

@@ -36,20 +36,30 @@ type gtsVideo struct {
 // decodeVideoFrame decodes and returns an image from a single frame in the given video stream.
 // (note: currently this only returns a blank image resized to fit video dimensions).
 func decodeVideoFrame(r io.Reader) (*gtsVideo, error) {
-	// we need a readseeker to decode the video...
-	tfs, err := iotools.TempFileSeeker(r)
-	if err != nil {
-		return nil, fmt.Errorf("error creating temp file seeker: %w", err)
-	}
-	defer func() {
-		if err := tfs.Close(); err != nil {
-			log.Errorf(nil, "error closing temp file seeker: %s", err)
+	// Check if video stream supports
+	// seeking, usually when *os.File.
+	rsc, ok := r.(io.ReadSeekCloser)
+	if !ok {
+		var err error
+
+		// Store stream to temporary location
+		// in order that we can get seek-reads.
+		rsc, err = iotools.TempFileSeeker(r)
+		if err != nil {
+			return nil, fmt.Errorf("error creating temp file seeker: %w", err)
 		}
-	}()
+
+		defer func() {
+			// Ensure temp. read seeker closed.
+			if err := rsc.Close(); err != nil {
+				log.Errorf(nil, "error closing temp file seeker: %s", err)
+			}
+		}()
+	}
 
 	// probe the video file to extract useful metadata from it; for methodology, see:
 	// https://github.com/abema/go-mp4/blob/7d8e5a7c5e644e0394261b0cf72fef79ce246d31/mp4tool/probe/probe.go#L85-L154
-	info, err := mp4.Probe(tfs)
+	info, err := mp4.Probe(rsc)
 	if err != nil {
 		return nil, fmt.Errorf("error during mp4 probe: %w", err)
 	}