mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-26 01:33:31 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] update config types to use bytesize.Size (#828)

Browse source 
* update config size types to use bytesize.Size

* submit unchecked-out file ... 🤦

* fix bytesize config var decoding

* bump bytesize version

* update kim's libraries in readme

* update envparse.sh to output more useful errors

* improve envparse.sh

* remove reliance on jq

* instead, use uint64 for bytesize flag types

* remove redundant type

* fix viper unmarshaling

* Update envparsing.sh

* fix envparsing test

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tobi <31960611+tsmethurst@users.noreply.github.com>
This commit is contained in:
kim 2022-09-29 21:50:43 +01:00 • committed by GitHub
commit 1d999712e6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 223 additions and 169 deletions
Download patch file Download diff file Expand all files Collapse all files

46
internal/media/manager_test.go
View file

@ -43,13 +43,13 @@ type ManagerTestSuite struct {
func (suite *ManagerTestSuite) TestEmojiProcessBlocking() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/rainbow-original.png")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
emojiID := "01GDQ9G782X42BAMFASKP64343"
@ -104,13 +104,13 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlocking() {
func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/big-panda.gif")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
emojiID := "01GDQ9G782X42BAMFASKP64343"
@ -128,13 +128,13 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() {
func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/big-panda.gif")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
emojiID := "01GDQ9G782X42BAMFASKP64343"
@ -152,7 +152,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() {
func (suite *ManagerTestSuite) TestEmojiProcessBlockingNoFileSizeGiven() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/rainbow-original.png")
if err != nil {
@ -214,13 +214,13 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingNoFileSizeGiven() {
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@ -286,7 +286,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
@ -359,7 +359,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingReadCloser() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// open test image as a file
f, err := os.Open("./test/test-jpeg.jpg")
if err != nil {
@ -432,13 +432,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingReadCloser() {
func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-png-noalphachannel.png")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@ -504,13 +504,13 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-png-alphachannel.png")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@ -576,13 +576,13 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
// test the callback function by setting a simple boolean
@ -659,13 +659,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@ -744,9 +744,9 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
panic(err)
}
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
return bytes.NewReader(b), len(b), nil
return bytes.NewReader(b), int64(len(b)), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
@ -820,13 +820,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
ctx := context.Background()
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("./test/test-jpeg.jpg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
accountID := "01FS1X72SK9ZPW0J1QQ68BD264"

8
internal/media/processingemoji.go
View file

@ -210,11 +210,11 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
// concatenate the first bytes with the existing bytes still in the reader (thanks Mara)
readerToStore := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
var maxEmojiSize int
var maxEmojiSize int64
if p.emoji.Domain == "" {
maxEmojiSize = config.GetMediaEmojiLocalMaxSize()
maxEmojiSize = int64(config.GetMediaEmojiLocalMaxSize())
} else {
maxEmojiSize = config.GetMediaEmojiRemoteMaxSize()
maxEmojiSize = int64(config.GetMediaEmojiRemoteMaxSize())
}
// if we know the fileSize already, make sure it's not bigger than our limit
@ -241,7 +241,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
return fmt.Errorf("store: discovered emoji fileSize (%db) is larger than allowed emojiRemoteMaxSize (%db)", fileSize, maxEmojiSize)
}
p.emoji.ImageFileSize = fileSize
p.emoji.ImageFileSize = int(fileSize)
p.read = true
if p.postData != nil {

4
internal/media/processingmedia.go
View file

@ -315,7 +315,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
p.attachment.Type = gtsmodel.FileTypeImage
if fileSize > 0 {
var err error
readerToStore, err = terminator.Terminate(readerToStore, fileSize, extension)
readerToStore, err = terminator.Terminate(readerToStore, int(fileSize), extension)
if err != nil {
return fmt.Errorf("store: exif error: %s", err)
}
@ -344,7 +344,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
cached := true
p.attachment.Cached = &cached
p.attachment.File.FileSize = fileSize
p.attachment.File.FileSize = int(fileSize)
p.read = true
if p.postData != nil {

4
internal/media/pruneremote_test.go
View file

@ -74,13 +74,13 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.ErrorIs(err, storage.ErrNotFound)
// now recache the image....
data := func(_ context.Context) (io.Reader, int, error) {
data := func(_ context.Context) (io.Reader, int64, error) {
// load bytes from a test image
b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpeg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
return bytes.NewBuffer(b), int64(len(b)), nil
}
processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, testAttachment.ID)
suite.NoError(err)

2
internal/media/types.go
View file

@ -118,7 +118,7 @@ type AdditionalEmojiInfo struct {
}
// DataFunc represents a function used to retrieve the raw bytes of a piece of media.
type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int, err error)
type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int64, err error)
// PostDataCallbackFunc represents a function executed after the DataFunc has been executed,
// and the returned reader has been read. It can be used to clean up any remaining resources.

6
internal/media/util.go
View file

@ -151,19 +151,19 @@ func parseOlderThan(olderThanDays int) (time.Time, error) {
// lengthReader wraps a reader and reads the length of total bytes written as it goes.
type lengthReader struct {
source io.Reader
length int
length int64
}
func (r *lengthReader) Read(b []byte) (int, error) {
n, err := r.source.Read(b)
r.length += n
r.length += int64(n)
return n, err
}
// putStream either puts a file with a known fileSize into storage directly, and returns the
// fileSize unchanged, or it wraps the reader with a lengthReader and returns the discovered
// fileSize.
func putStream(ctx context.Context, storage storage.Driver, key string, r io.Reader, fileSize int) (int, error) {
func putStream(ctx context.Context, storage storage.Driver, key string, r io.Reader, fileSize int64) (int64, error) {
if fileSize > 0 {
return fileSize, storage.PutStream(ctx, key, r)
}