rework data function to provide filesize

internal/media/image.go

@@ -30,7 +30,6 @@ import (
 
 	"github.com/buckket/go-blurhash"
 	"github.com/nfnt/resize"
-	"github.com/superseriousbusiness/exifremove/pkg/exifremove"
 )
 
 const (
@@ -197,22 +196,3 @@ func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) {
 		small: out.Bytes(),
 	}, nil
 }
-
-// purgeExif is a little wrapper for the action of removing exif data from an image.
-// Only pass pngs or jpegs to this function.
-func purgeExif(data []byte) ([]byte, error) {
-	if len(data) == 0 {
-		return nil, errors.New("passed image was not valid")
-	}
-
-	clean, err := exifremove.Remove(data)
-	if err != nil {
-		return nil, fmt.Errorf("could not purge exif from image: %s", err)
-	}
-
-	if len(clean) == 0 {
-		return nil, errors.New("purged image was not valid")
-	}
-
-	return clean, nil
-}

internal/media/manager_test.go

@@ -39,13 +39,13 @@ type ManagerTestSuite struct {
 func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
 	ctx := context.Background()
 
-	data := func(_ context.Context) (io.Reader, error) {
+	data := func(_ context.Context) (io.Reader, int, error) {
 		// load bytes from a test image
 		b, err := os.ReadFile("./test/test-jpeg.jpg")
 		if err != nil {
 			panic(err)
 		}
-		return bytes.NewBuffer(b), nil
+		return bytes.NewBuffer(b), len(b), nil
 	}
 
 	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@@ -109,13 +109,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
 func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
 	ctx := context.Background()
 
-	data := func(_ context.Context) (io.Reader, error) {
+	data := func(_ context.Context) (io.Reader, int, error) {
 		// load bytes from a test image
 		b, err := os.ReadFile("./test/test-jpeg.jpg")
 		if err != nil {
 			panic(err)
 		}
-		return bytes.NewBuffer(b), nil
+		return bytes.NewBuffer(b), len(b), nil
 	}
 
 	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@@ -192,9 +192,9 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
 		panic(err)
 	}
 
-	data := func(_ context.Context) (io.Reader, error) {
+	data := func(_ context.Context) (io.Reader, int, error) {
 		// load bytes from a test image
-		return bytes.NewReader(b), nil
+		return bytes.NewReader(b), len(b), nil
 	}
 
 	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"

internal/media/processingemoji.go

@@ -163,7 +163,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 	}
 
 	// execute the data function to get the reader out of it
-	reader, err := p.data(ctx)
+	reader, fileSize, err := p.data(ctx)
 	if err != nil {
 		return fmt.Errorf("store: error executing data function: %s", err)
 	}
@@ -194,6 +194,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 	p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), p.emoji.ID, extension)
 	p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, p.emoji.ID, extension)
 	p.emoji.ImageContentType = contentType
+	p.emoji.ImageFileSize = fileSize
 
 	// concatenate the first bytes with the existing bytes still in the reader (thanks Mara)
 	multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
@@ -202,7 +203,6 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
 	if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil {
 		return fmt.Errorf("store: error storing stream: %s", err)
 	}
-	p.emoji.ImageFileSize = 36702 // TODO: set this based on the result of PutStream
 
 	// if the original reader is a readcloser, close it since we're done with it now
 	if rc, ok := reader.(io.ReadCloser); ok {

internal/media/processingmedia.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"codeberg.org/gruf/go-store/kv"
+	terminator "github.com/superseriousbusiness/exif-terminator"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
@@ -239,7 +240,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	}
 
 	// execute the data function to get the reader out of it
-	reader, err := p.data(ctx)
+	reader, fileSize, err := p.data(ctx)
 	if err != nil {
 		return fmt.Errorf("store: error executing data function: %s", err)
 	}
@@ -268,22 +269,36 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	}
 	extension := split[1] // something like 'jpeg'
 
-	// set some additional fields on the attachment now that
-	// we know more about what the underlying media actually is
-	if extension == mimeGif {
+	// concatenate the cleaned up first bytes with the existing bytes still in the reader (thanks Mara)
+	multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
+
+	// we'll need to clean exif data from the first bytes; while we're
+	// here, we can also use the extension to derive the attachment type
+	var clean io.Reader
+	switch extension {
+	case mimeGif:
 		p.attachment.Type = gtsmodel.FileTypeGif
-	} else {
+		clean = multiReader // nothing to clean from a gif
+	case mimeJpeg, mimePng:
 		p.attachment.Type = gtsmodel.FileTypeImage
+		purged, err := terminator.Terminate(multiReader, fileSize, extension)
+		if err != nil {
+			return fmt.Errorf("store: exif error: %s", err)
+		}
+		clean = purged
+	default:
+		return fmt.Errorf("store: couldn't process %s", extension)
 	}
+
+	// now set some additional fields on the attachment since
+	// we know more about what the underlying media actually is
 	p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension)
 	p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension)
 	p.attachment.File.ContentType = contentType
-
-	// concatenate the first bytes with the existing bytes still in the reader (thanks Mara)
-	multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
+	p.attachment.File.FileSize = fileSize
 
 	// store this for now -- other processes can pull it out of storage as they please
-	if err := p.storage.PutStream(p.attachment.File.Path, multiReader); err != nil {
+	if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
 		return fmt.Errorf("store: error storing stream: %s", err)
 	}
 

internal/media/types.go

@@ -118,4 +118,4 @@ type AdditionalEmojiInfo struct {
 }
 
 // DataFunc represents a function used to retrieve the raw bytes of a piece of media.
-type DataFunc func(ctx context.Context) (io.Reader, error)
+type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int, err error)