mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-25 21:23:42 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Media attachment placeholders (#2331)

Browse source 
* [feature] Use placeholders for unknown media types

* fix read of underreported small files

* switch to reduce nesting

* simplify cleanup
This commit is contained in:
tobi 2023-11-10 19:29:26 +01:00 • committed by GitHub
commit ba9d6b467a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 1472 additions and 841 deletions
Download patch file Download diff file Expand all files Collapse all files

91
internal/media/processingemoji.go
View file

@ -20,7 +20,6 @@ package media
import (
"bytes"
"context"
"fmt"
"io"
"codeberg.org/gruf/go-bytesize"
@ -33,6 +32,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/regexes"
"github.com/superseriousbusiness/gotosocial/internal/uris"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// ProcessingEmoji represents an emoji currently processing. It exposes
@ -156,14 +156,51 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
}
}()
// Byte buffer to read file header into.
// See: https://en.wikipedia.org/wiki/File_format#File_header
// and https://github.com/h2non/filetype
hdrBuf := make([]byte, 261)
var maxSize bytesize.Size
// Read the first 261 header bytes into buffer.
if _, err := io.ReadFull(rc, hdrBuf); err != nil {
return gtserror.Newf("error reading incoming media: %w", err)
if p.emoji.Domain == "" {
// this is a local emoji upload
maxSize = config.GetMediaEmojiLocalMaxSize()
} else {
// this is a remote incoming emoji
maxSize = config.GetMediaEmojiRemoteMaxSize()
}
// Check that provided size isn't beyond max. We check beforehand
// so that we don't attempt to stream the emoji into storage if not needed.
if size := bytesize.Size(sz); sz > 0 && size > maxSize {
return gtserror.Newf("given emoji size %s greater than max allowed %s", size, maxSize)
}
// Prepare to read bytes from
// file header or magic number.
fileSize := int(sz)
hdrBuf := newHdrBuf(fileSize)
// Read into buffer as much as possible.
//
// UnexpectedEOF means we couldn't read up to the
// given size, but we may still have read something.
//
// EOF means we couldn't read anything at all.
//
// Any other error likely means the connection messed up.
//
// In other words, rather counterintuitively, we
// can only proceed on no error or unexpected error!
n, err := io.ReadFull(rc, hdrBuf)
if err != nil {
if err != io.ErrUnexpectedEOF {
return gtserror.Newf("error reading first bytes of incoming media: %w", err)
}
// Initial file size was misreported, so we didn't read
// fully into hdrBuf. Reslice it to the size we did read.
log.Warnf(ctx,
"recovered from misreported file size; reported %d; read %d",
fileSize, n,
)
hdrBuf = hdrBuf[:n]
}
// Parse file type info from header buffer.
@ -184,24 +221,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
// Recombine header bytes with remaining stream
r := io.MultiReader(bytes.NewReader(hdrBuf), rc)
var maxSize bytesize.Size
if p.emoji.Domain == "" {
// this is a local emoji upload
maxSize = config.GetMediaEmojiLocalMaxSize()
} else {
// this is a remote incoming emoji
maxSize = config.GetMediaEmojiRemoteMaxSize()
}
// Check that provided size isn't beyond max. We check beforehand
// so that we don't attempt to stream the emoji into storage if not needed.
if size := bytesize.Size(sz); sz > 0 && size > maxSize {
return gtserror.Newf("given emoji size %s greater than max allowed %s", size, maxSize)
}
var pathID string
if p.newPathID != "" {
// This is a refreshed emoji with a new
// path ID that this will be stored under.
@ -215,11 +235,10 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
instanceAccID := regexes.FilePath.FindStringSubmatch(p.emoji.ImageStaticPath)[1]
// Calculate emoji file path.
p.emoji.ImagePath = fmt.Sprintf(
"%s/%s/%s/%s.%s",
p.emoji.ImagePath = uris.StoragePathForAttachment(
instanceAccID,
TypeEmoji,
SizeOriginal,
string(TypeEmoji),
string(SizeOriginal),
pathID,
info.Extension,
)
@ -235,14 +254,13 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
}
// Write the final image reader stream to our storage.
sz, err = p.mgr.state.Storage.PutStream(ctx, p.emoji.ImagePath, r)
wroteSize, err := p.mgr.state.Storage.PutStream(ctx, p.emoji.ImagePath, r)
if err != nil {
return gtserror.Newf("error writing emoji to storage: %w", err)
}
// Once again check size in case none was provided previously.
if size := bytesize.Size(sz); size > maxSize {
if size := bytesize.Size(wroteSize); size > maxSize {
if err := p.mgr.state.Storage.Delete(ctx, p.emoji.ImagePath); err != nil {
log.Errorf(ctx, "error removing too-large-emoji from storage: %v", err)
}
@ -251,7 +269,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
}
// Fill in remaining attachment data now it's stored.
p.emoji.ImageURL = uris.GenerateURIForAttachment(
p.emoji.ImageURL = uris.URIForAttachment(
instanceAccID,
string(TypeEmoji),
string(SizeOriginal),
@ -259,11 +277,8 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
info.Extension,
)
p.emoji.ImageContentType = info.MIME.Value
p.emoji.ImageFileSize = int(sz)
p.emoji.Cached = func() *bool {
ok := true
return &ok
}()
p.emoji.ImageFileSize = int(wroteSize)
p.emoji.Cached = util.Ptr(true)
return nil
}