mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-12-12 10:18:06 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] S3 support (#674)

Browse source 
* feat: vendor minio client

* feat: introduce storage package with s3 support

* feat: serve s3 files directly

this saves a lot of bandwith as the files are fetched from the object
store directly

* fix: use explicit local storage in tests

* feat: integrate s3 storage with the main server

* fix: add s3 config to cli tests

* docs: explicitly set values in example config

also adds license header to the storage package

* fix: use better http status code on s3 redirect

HTTP 302 Found is the best fit, as it signifies that the resource
requested was found but not under its presumed URL

307/TemporaryRedirect would mean that this resource is usually located
here, not in this case

303/SeeOther indicates that the redirection does not link to the
requested resource but to another page

* refactor: use context in storage driver interface
This commit is contained in:
Dominik Süß 2022-07-03 12:08:30 +02:00 • committed by GitHub
commit 9d0df426da
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
250 changed files with 77798 additions and 185 deletions
Download patch file Download diff file Expand all files Collapse all files

6
internal/api/client/account/account_test.go
View file

@ -6,7 +6,6 @@ import (
"net/http"
"net/http/httptest"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/account"
@ -20,6 +19,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -27,7 +27,7 @@ type AccountStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@ -65,7 +65,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)

6
internal/api/client/admin/admin_test.go
View file

@ -24,7 +24,6 @@ import (
"net/http"
"net/http/httptest"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
@ -38,6 +37,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -45,7 +45,7 @@ type AdminStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@ -83,7 +83,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)

4
internal/api/client/admin/emojicreate_test.go
View file

@ -103,10 +103,10 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() {
suite.Empty(dbEmoji.CategoryID)
// emoji should be in storage
emojiBytes, err := suite.storage.Get(dbEmoji.ImagePath)
emojiBytes, err := suite.storage.Get(ctx, dbEmoji.ImagePath)
suite.NoError(err)
suite.Len(emojiBytes, dbEmoji.ImageFileSize)
emojiStaticBytes, err := suite.storage.Get(dbEmoji.ImageStaticPath)
emojiStaticBytes, err := suite.storage.Get(ctx, dbEmoji.ImageStaticPath)
suite.NoError(err)
suite.Len(emojiStaticBytes, dbEmoji.ImageStaticFileSize)
}

6
internal/api/client/auth/auth_test.go
View file

@ -24,7 +24,6 @@ import (
"fmt"
"net/http/httptest"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-contrib/sessions"
"github.com/gin-contrib/sessions/memstore"
"github.com/gin-gonic/gin"
@ -42,13 +41,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oidc"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type AuthStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@ -88,7 +88,7 @@ func (suite *AuthStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)

5
internal/api/client/fileserver/servefile.go
View file

@ -84,6 +84,11 @@ func (m *FileServer) ServeFile(c *gin.Context) {
return
}
if content.URL != nil {
c.Redirect(http.StatusFound, content.URL.String())
return
}
defer func() {
// if the content is a ReadCloser, close it when we're done
if closer, ok := content.Content.(io.ReadCloser); ok {

10
internal/api/client/fileserver/servefile_test.go
View file

@ -26,7 +26,6 @@ import (
"net/http/httptest"
"testing"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@ -40,6 +39,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -48,7 +48,7 @@ type ServeFileTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
federator federation.Federator
tc typeutils.TypeConverter
processor processing.Processor
@ -81,7 +81,7 @@ func (suite *ServeFileTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
@ -160,7 +160,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
suite.NoError(err)
suite.NotNil(b)
fileInStorage, err := suite.storage.Get(targetAttachment.File.Path)
fileInStorage, err := suite.storage.Get(ctx, targetAttachment.File.Path)
suite.NoError(err)
suite.NotNil(fileInStorage)
suite.Equal(b, fileInStorage)
@ -206,7 +206,7 @@ func (suite *ServeFileTestSuite) TestServeSmallFileSuccessful() {
suite.NoError(err)
suite.NotNil(b)
fileInStorage, err := suite.storage.Get(targetAttachment.Thumbnail.Path)
fileInStorage, err := suite.storage.Get(ctx, targetAttachment.Thumbnail.Path)
suite.NoError(err)
suite.NotNil(fileInStorage)
suite.Equal(b, fileInStorage)

6
internal/api/client/followrequest/followrequest_test.go
View file

@ -23,7 +23,6 @@ import (
"fmt"
"net/http/httptest"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/followrequest"
@ -37,13 +36,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type FollowRequestStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@ -80,7 +80,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)

6
internal/api/client/instance/instance_test.go
View file

@ -23,7 +23,6 @@ import (
"fmt"
"net/http/httptest"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/instance"
@ -37,6 +36,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -44,7 +44,7 @@ type InstanceStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
federator federation.Federator
processor processing.Processor
@ -82,7 +82,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)

10
internal/api/client/media/mediacreate_test.go
View file

@ -30,7 +30,6 @@ import (
"net/http/httptest"
"testing"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@ -46,6 +45,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -54,7 +54,7 @@ type MediaCreateTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
storage *kv.KVStore
storage *storage.Local
mediaManager media.Manager
federator federation.Federator
tc typeutils.TypeConverter
@ -87,7 +87,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
@ -138,7 +138,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {
// see what's in storage *before* the request
storageKeysBeforeRequest := []string{}
iter, err := suite.storage.Iterator(nil)
iter, err := suite.storage.KVStore.Iterator(nil)
if err != nil {
panic(err)
}
@ -164,7 +164,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {
// check what's in storage *after* the request
storageKeysAfterRequest := []string{}
iter, err = suite.storage.Iterator(nil)
iter, err = suite.storage.KVStore.Iterator(nil)
if err != nil {
panic(err)
}

6
internal/api/client/media/mediaupdate_test.go
View file

@ -28,7 +28,6 @@ import (
"net/http/httptest"
"testing"
"codeberg.org/gruf/go-store/kv"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
@ -44,6 +43,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -52,7 +52,7 @@ type MediaUpdateTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
federator federation.Federator
tc typeutils.TypeConverter
mediaManager media.Manager
@ -85,7 +85,7 @@ func (suite *MediaUpdateTestSuite) SetupSuite() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)

6
internal/api/client/status/status_test.go
View file

@ -19,7 +19,6 @@
package status_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
@ -30,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -43,7 +43,7 @@ type StatusStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage *kv.KVStore
storage storage.Driver
// standard suite models
testTokens map[string]*gtsmodel.Token
@ -76,7 +76,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")

6
internal/api/client/user/user_test.go
View file

@ -19,7 +19,6 @@
package user_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/user"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
@ -30,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -42,7 +42,7 @@ type UserStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage *kv.KVStore
storage storage.Driver
testTokens map[string]*gtsmodel.Token
testClients map[string]*gtsmodel.Client
@ -66,7 +66,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.testUsers = testrig.NewTestUsers()
suite.testAccounts = testrig.NewTestAccounts()
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)

7
internal/api/model/content.go
View file

@ -18,7 +18,10 @@
package model
import "io"
import (
"io"
"net/url"
)
// Content wraps everything needed to serve a blob of content (some kind of media) through the API.
type Content struct {
@ -28,6 +31,8 @@ type Content struct {
ContentLength int64
// Actual content
Content io.Reader
// Resource URL to forward to if the file can be fetched from the storage directly (e.g signed S3 URL)
URL *url.URL
}
// GetContentRequestForm describes a piece of content desired by the caller of the fileserver API.

6
internal/api/s2s/user/user_test.go
View file

@ -19,7 +19,6 @@
package user_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
"github.com/superseriousbusiness/gotosocial/internal/api/security"
@ -32,6 +31,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -45,7 +45,7 @@ type UserStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage *kv.KVStore
storage storage.Driver
oauthServer oauth.Server
securityModule *security.Module
@ -83,7 +83,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)

6
internal/api/s2s/webfinger/webfinger_test.go
View file

@ -23,7 +23,6 @@ import (
"crypto/rsa"
"time"
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger"
@ -37,6 +36,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -50,7 +50,7 @@ type WebfingerStandardTestSuite struct {
federator federation.Federator
emailSender email.Sender
processor processing.Processor
storage *kv.KVStore
storage storage.Driver
oauthServer oauth.Server
securityModule *security.Module
@ -86,7 +86,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)

5
internal/config/config.go
View file

@ -82,6 +82,11 @@ type Configuration struct {
StorageBackend string `name:"storage-backend" usage:"Storage backend to use for media attachments"`
StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."`
StorageS3Endpoint string `name:"storage-s3-endpoint" usage:"S3 Endpoint URL (e.g 'minio.example.org:9000')"`
StorageS3AccessKey string `name:"storage-s3-access-key" usage:"S3 Access Key"`
StorageS3SecretKey string `name:"storage-s3-secret-key" usage:"S3 Secret Key"`
StorageS3UseSSL bool `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
StorageS3BucketName string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
StatusesMaxChars int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses"`
StatusesCWMaxChars int `name:"statuses-cw-max-chars" usage:"Max permitted characters for content/spoiler warnings on statuses"`

1
internal/config/defaults.go
View file

@ -61,6 +61,7 @@ var Defaults = Configuration{
StorageBackend: "local",
StorageLocalBasePath: "/gotosocial/storage",
StorageS3UseSSL: true,
StatusesMaxChars: 5000,
StatusesCWMaxChars: 100,

125
internal/config/helpers.gen.go
View file

@ -843,6 +843,131 @@ func GetStorageLocalBasePath() string { return global.GetStorageLocalBasePath()
// SetStorageLocalBasePath safely sets the value for global configuration 'StorageLocalBasePath' field
func SetStorageLocalBasePath(v string) { global.SetStorageLocalBasePath(v) }
// GetStorageS3Endpoint safely fetches the Configuration value for state's 'StorageS3Endpoint' field
func (st *ConfigState) GetStorageS3Endpoint() (v string) {
st.mutex.Lock()
v = st.config.StorageS3Endpoint
st.mutex.Unlock()
return
}
// SetStorageS3Endpoint safely sets the Configuration value for state's 'StorageS3Endpoint' field
func (st *ConfigState) SetStorageS3Endpoint(v string) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.StorageS3Endpoint = v
st.reloadToViper()
}
// StorageS3EndpointFlag returns the flag name for the 'StorageS3Endpoint' field
func StorageS3EndpointFlag() string { return "storage-s3-endpoint" }
// GetStorageS3Endpoint safely fetches the value for global configuration 'StorageS3Endpoint' field
func GetStorageS3Endpoint() string { return global.GetStorageS3Endpoint() }
// SetStorageS3Endpoint safely sets the value for global configuration 'StorageS3Endpoint' field
func SetStorageS3Endpoint(v string) { global.SetStorageS3Endpoint(v) }
// GetStorageS3AccessKey safely fetches the Configuration value for state's 'StorageS3AccessKey' field
func (st *ConfigState) GetStorageS3AccessKey() (v string) {
st.mutex.Lock()
v = st.config.StorageS3AccessKey
st.mutex.Unlock()
return
}
// SetStorageS3AccessKey safely sets the Configuration value for state's 'StorageS3AccessKey' field
func (st *ConfigState) SetStorageS3AccessKey(v string) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.StorageS3AccessKey = v
st.reloadToViper()
}
// StorageS3AccessKeyFlag returns the flag name for the 'StorageS3AccessKey' field
func StorageS3AccessKeyFlag() string { return "storage-s3-access-key" }
// GetStorageS3AccessKey safely fetches the value for global configuration 'StorageS3AccessKey' field
func GetStorageS3AccessKey() string { return global.GetStorageS3AccessKey() }
// SetStorageS3AccessKey safely sets the value for global configuration 'StorageS3AccessKey' field
func SetStorageS3AccessKey(v string) { global.SetStorageS3AccessKey(v) }
// GetStorageS3SecretKey safely fetches the Configuration value for state's 'StorageS3SecretKey' field
func (st *ConfigState) GetStorageS3SecretKey() (v string) {
st.mutex.Lock()
v = st.config.StorageS3SecretKey
st.mutex.Unlock()
return
}
// SetStorageS3SecretKey safely sets the Configuration value for state's 'StorageS3SecretKey' field
func (st *ConfigState) SetStorageS3SecretKey(v string) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.StorageS3SecretKey = v
st.reloadToViper()
}
// StorageS3SecretKeyFlag returns the flag name for the 'StorageS3SecretKey' field
func StorageS3SecretKeyFlag() string { return "storage-s3-secret-key" }
// GetStorageS3SecretKey safely fetches the value for global configuration 'StorageS3SecretKey' field
func GetStorageS3SecretKey() string { return global.GetStorageS3SecretKey() }
// SetStorageS3SecretKey safely sets the value for global configuration 'StorageS3SecretKey' field
func SetStorageS3SecretKey(v string) { global.SetStorageS3SecretKey(v) }
// GetStorageS3UseSSL safely fetches the Configuration value for state's 'StorageS3UseSSL' field
func (st *ConfigState) GetStorageS3UseSSL() (v bool) {
st.mutex.Lock()
v = st.config.StorageS3UseSSL
st.mutex.Unlock()
return
}
// SetStorageS3UseSSL safely sets the Configuration value for state's 'StorageS3UseSSL' field
func (st *ConfigState) SetStorageS3UseSSL(v bool) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.StorageS3UseSSL = v
st.reloadToViper()
}
// StorageS3UseSSLFlag returns the flag name for the 'StorageS3UseSSL' field
func StorageS3UseSSLFlag() string { return "storage-s3-use-ssl" }
// GetStorageS3UseSSL safely fetches the value for global configuration 'StorageS3UseSSL' field
func GetStorageS3UseSSL() bool { return global.GetStorageS3UseSSL() }
// SetStorageS3UseSSL safely sets the value for global configuration 'StorageS3UseSSL' field
func SetStorageS3UseSSL(v bool) { global.SetStorageS3UseSSL(v) }
// GetStorageS3BucketName safely fetches the Configuration value for state's 'StorageS3BucketName' field
func (st *ConfigState) GetStorageS3BucketName() (v string) {
st.mutex.Lock()
v = st.config.StorageS3BucketName
st.mutex.Unlock()
return
}
// SetStorageS3BucketName safely sets the Configuration value for state's 'StorageS3BucketName' field
func (st *ConfigState) SetStorageS3BucketName(v string) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.StorageS3BucketName = v
st.reloadToViper()
}
// StorageS3BucketNameFlag returns the flag name for the 'StorageS3BucketName' field
func StorageS3BucketNameFlag() string { return "storage-s3-bucket" }
// GetStorageS3BucketName safely fetches the value for global configuration 'StorageS3BucketName' field
func GetStorageS3BucketName() string { return global.GetStorageS3BucketName() }
// SetStorageS3BucketName safely sets the value for global configuration 'StorageS3BucketName' field
func SetStorageS3BucketName(v string) { global.SetStorageS3BucketName(v) }
// GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field
func (st *ConfigState) GetStatusesMaxChars() (v int) {
st.mutex.Lock()

5
internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go
View file

@ -59,6 +59,11 @@ func init() {
up := func(ctx context.Context, db *bun.DB) error {
l := logrus.WithField("migration", "20220612091800_duplicated_media_cleanup")
if config.GetStorageBackend() != "local" {
// this migration only affects versions which only supported local storage
return nil
}
storageBasePath := config.GetStorageLocalBasePath()
if storageBasePath == "" {
return fmt.Errorf("%s must be set to do storage migration", config.StorageLocalBasePathFlag())

6
internal/federation/dereferencing/dereferencer_test.go
View file

@ -19,7 +19,6 @@
package dereferencing_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
@ -27,13 +26,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type DereferencerStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
testRemoteStatuses map[string]vocab.ActivityStreamsNote
testRemotePeople map[string]vocab.ActivityStreamsPerson
@ -57,7 +57,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
suite.testRemoteAttachments = testrig.NewTestFediAttachments("../../../testrig/media")
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)), testrig.NewTestMediaManager(suite.db, suite.storage))
testrig.StandardDBSetup(suite.db, nil)
}

6
internal/federation/federator_test.go
View file

@ -19,11 +19,11 @@
package federation_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -31,7 +31,7 @@ import (
type FederatorStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
tc typeutils.TypeConverter
testAccounts map[string]*gtsmodel.Account
testStatuses map[string]*gtsmodel.Status
@ -41,7 +41,7 @@ type FederatorStandardTestSuite struct {
// SetupSuite sets some variables on the suite that we can use as consts (more or less) throughout
func (suite *FederatorStandardTestSuite) SetupSuite() {
// setup standard items
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.testAccounts = testrig.NewTestAccounts()
suite.testStatuses = testrig.NewTestStatuses()

6
internal/media/manager.go
View file

@ -23,12 +23,12 @@ import (
"fmt"
"time"
"codeberg.org/gruf/go-store/kv"
"github.com/robfig/cron/v3"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/storage"
)
// selectPruneLimit is the amount of media entries to select at a time from the db when pruning
@ -98,7 +98,7 @@ type Manager interface {
type manager struct {
db db.DB
storage *kv.KVStore
storage storage.Driver
emojiWorker *concurrency.WorkerPool[*ProcessingEmoji]
mediaWorker *concurrency.WorkerPool[*ProcessingMedia]
stopCronJobs func() error
@ -110,7 +110,7 @@ type manager struct {
// a limited number of media will be processed in parallel. The numbers of workers
// is determined from the $GOMAXPROCS environment variable (usually no. CPU cores).
// See internal/concurrency.NewWorkerPool() documentation for further information.
func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
func NewManager(database db.DB, storage storage.Driver) (Manager, error) {
m := &manager{
db: database,
storage: storage,

27
internal/media/manager_test.go
View file

@ -33,6 +33,7 @@ import (
"github.com/stretchr/testify/suite"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
)
type ManagerTestSuite struct {
@ -87,7 +88,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@ -100,7 +101,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@ -159,7 +160,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@ -172,7 +173,7 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@ -231,7 +232,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@ -244,7 +245,7 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@ -314,7 +315,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@ -327,7 +328,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@ -393,7 +394,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@ -406,7 +407,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@ -474,7 +475,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
suite.NotNil(dbAttachment)
// make sure the processed file is in storage
processedFullBytes, err := suite.storage.Get(attachment.File.Path)
processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
suite.NoError(err)
suite.NotEmpty(processedFullBytes)
@ -487,7 +488,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() {
suite.Equal(processedFullBytesExpected, processedFullBytes)
// now do the same for the thumbnail and make sure it's what we expected
processedThumbnailBytes, err := suite.storage.Get(attachment.Thumbnail.Path)
processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
suite.NoError(err)
suite.NotEmpty(processedThumbnailBytes)
@ -523,7 +524,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
panic(err)
}
diskManager, err := media.NewManager(suite.db, diskStorage)
diskManager, err := media.NewManager(suite.db, &gtsstorage.Local{KVStore: diskStorage})
if err != nil {
panic(err)
}

6
internal/media/media_test.go
View file

@ -19,11 +19,11 @@
package media_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -31,7 +31,7 @@ type MediaStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
manager media.Manager
testAttachments map[string]*gtsmodel.MediaAttachment
testAccounts map[string]*gtsmodel.Account
@ -42,7 +42,7 @@ func (suite *MediaStandardTestSuite) SetupSuite() {
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
}
func (suite *MediaStandardTestSuite) SetupTest() {

10
internal/media/processingemoji.go
View file

@ -28,10 +28,10 @@ import (
"sync/atomic"
"time"
"codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
@ -64,7 +64,7 @@ type ProcessingEmoji struct {
*/
database db.DB
storage *kv.KVStore
storage storage.Driver
err error // error created during processing, if any
@ -113,7 +113,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
switch processState(staticState) {
case received:
// stream the original file out of storage...
stored, err := p.storage.GetStream(p.emoji.ImagePath)
stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath)
if err != nil {
p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
@ -135,7 +135,7 @@ func (p *ProcessingEmoji) loadStatic(ctx context.Context) error {
}
// put the static in storage
if err := p.storage.Put(p.emoji.ImageStaticPath, static.small); err != nil {
if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil {
p.err = fmt.Errorf("loadStatic: error storing static: %s", err)
atomic.StoreInt32(&p.staticState, int32(errored))
return p.err
@ -211,7 +211,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error {
multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), reader)
// store this for now -- other processes can pull it out of storage as they please
if err := p.storage.PutStream(p.emoji.ImagePath, multiReader); err != nil {
if err := p.storage.PutStream(ctx, p.emoji.ImagePath, multiReader); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}

12
internal/media/processingmedia.go
View file

@ -28,12 +28,12 @@ import (
"sync/atomic"
"time"
"codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
terminator "github.com/superseriousbusiness/exif-terminator"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
@ -61,7 +61,7 @@ type ProcessingMedia struct {
*/
database db.DB
storage *kv.KVStore
storage storage.Driver
err error // error created during processing, if any
@ -138,7 +138,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
// stream the original file out of storage
logrus.Tracef("loadThumb: fetching attachment from storage %s", p.attachment.URL)
stored, err := p.storage.GetStream(p.attachment.File.Path)
stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
@ -164,7 +164,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
// put the thumbnail in storage
logrus.Tracef("loadThumb: storing new thumbnail %s", p.attachment.URL)
if err := p.storage.Put(p.attachment.Thumbnail.Path, thumb.small); err != nil {
if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil {
p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err)
atomic.StoreInt32(&p.thumbState, int32(errored))
return p.err
@ -203,7 +203,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
var decoded *imageMeta
// stream the original file out of storage...
stored, err := p.storage.GetStream(p.attachment.File.Path)
stored, err := p.storage.GetStream(ctx, p.attachment.File.Path)
if err != nil {
p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err)
atomic.StoreInt32(&p.fullSizeState, int32(errored))
@ -343,7 +343,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
p.attachment.File.FileSize = fileSize
// store this for now -- other processes can pull it out of storage as they please
if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
if err := p.storage.PutStream(ctx, p.attachment.File.Path, clean); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
p.attachment.Cached = true

4
internal/media/prunemeta.go
View file

@ -69,7 +69,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.File.Path)
if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
}
@ -77,7 +77,7 @@ func (m *manager) pruneOneAvatarOrHeader(ctx context.Context, attachment *gtsmod
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("pruneOneAvatarOrHeader: deleting %s", attachment.Thumbnail.Path)
if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
}

16
internal/media/prunemeta_test.go
View file

@ -49,13 +49,13 @@ func (suite *PruneMetaTestSuite) TestPruneMeta() {
suite.Equal(2, totalPruned)
// media should no longer be stored
_, err = suite.storage.Get(zorkOldAvatar.File.Path)
_, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
_, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldHeader.File.Path)
_, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
_, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// attachments should no longer be in the db
@ -110,13 +110,13 @@ func (suite *PruneMetaTestSuite) TestPruneMetaMultipleAccounts() {
suite.Equal(2, totalPruned)
// media should no longer be stored
_, err = suite.storage.Get(zorkOldAvatar.File.Path)
_, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldAvatar.Thumbnail.Path)
_, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldHeader.File.Path)
_, err = suite.storage.Get(ctx, zorkOldHeader.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(zorkOldHeader.Thumbnail.Path)
_, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// attachments should no longer be in the db

4
internal/media/pruneremote.go
View file

@ -67,7 +67,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("pruneOneRemote: deleting %s", attachment.File.Path)
if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
@ -76,7 +76,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path)
if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false

10
internal/media/pruneremote_test.go
View file

@ -68,9 +68,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.Equal(2, totalPruned)
// media should no longer be stored
_, err = suite.storage.Get(testAttachment.File.Path)
_, err = suite.storage.Get(ctx, testAttachment.File.Path)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(testAttachment.Thumbnail.Path)
_, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path)
suite.ErrorIs(err, storage.ErrNotFound)
// now recache the image....
@ -98,9 +98,9 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same
// recached files should be back in storage
_, err = suite.storage.Get(recachedAttachment.File.Path)
_, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
suite.NoError(err)
_, err = suite.storage.Get(recachedAttachment.Thumbnail.Path)
_, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
suite.NoError(err)
}
@ -112,7 +112,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
suite.NoError(err)
suite.True(media.Cached)
err = suite.storage.Delete(media.File.Path)
err = suite.storage.Delete(ctx, media.File.Path)
suite.NoError(err)
// Now attempt to prune remote for item with db entry no file

6
internal/processing/account/account_test.go
View file

@ -21,7 +21,6 @@ package account_test
import (
"context"
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -33,6 +32,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/processing/account"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -43,7 +43,7 @@ type AccountStandardTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
oauthServer oauth.Server
fromClientAPIChan chan messages.FromClientAPI
@ -91,7 +91,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)

4
internal/processing/media/delete.go
View file

@ -24,14 +24,14 @@ func (p *processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr
// delete the thumbnail from storage
if attachment.Thumbnail.Path != "" {
if err := p.storage.Delete(attachment.Thumbnail.Path); err != nil {
if err := p.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil {
errs = append(errs, fmt.Sprintf("remove thumbnail at path %s: %s", attachment.Thumbnail.Path, err))
}
}
// delete the file from storage
if attachment.File.Path != "" {
if err := p.storage.Delete(attachment.File.Path); err != nil {
if err := p.storage.Delete(ctx, attachment.File.Path); err != nil {
errs = append(errs, fmt.Sprintf("remove file at path %s: %s", attachment.File.Path, err))
}
}

14
internal/processing/media/getfile.go
View file

@ -113,7 +113,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount
// if we have the media cached on our server already, we can now simply return it from storage
if a.Cached {
return p.streamFromStorage(storagePath, attachmentContent)
return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}
// if we don't have it cached, then we can assume two things:
@ -221,7 +221,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
}
// ... so now we can safely return it
return p.streamFromStorage(storagePath, attachmentContent)
return p.retrieveFromStorage(ctx, storagePath, attachmentContent)
}
return attachmentContent, nil
@ -253,11 +253,15 @@ func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, e
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize))
}
return p.streamFromStorage(storagePath, emojiContent)
return p.retrieveFromStorage(ctx, storagePath, emojiContent)
}
func (p *processor) streamFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
reader, err := p.storage.GetStream(storagePath)
func (p *processor) retrieveFromStorage(ctx context.Context, storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
if url := p.storage.URL(ctx, storagePath); url != nil {
content.URL = url
return content, nil
}
reader, err := p.storage.GetStream(ctx, storagePath)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err))
}

18
internal/processing/media/getfile_test.go
View file

@ -70,9 +70,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
testAttachment.Cached = false
err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.File.Path)
err = suite.storage.Delete(ctx, testAttachment.File.Path)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch it
@ -106,7 +106,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
suite.True(dbAttachment.Cached)
// the file should be back in storage at the same path as before
refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path)
suite.NoError(err)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
}
@ -119,9 +119,9 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() {
testAttachment.Cached = false
err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.File.Path)
err = suite.storage.Delete(ctx, testAttachment.File.Path)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch it
@ -156,7 +156,7 @@ func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() {
suite.True(dbAttachment.Cached)
// the file should be back in storage at the same path as before
refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
refreshedBytes, err := suite.storage.Get(ctx, testAttachment.File.Path)
suite.NoError(err)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
}
@ -166,16 +166,16 @@ func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() {
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
// fetch the existing thumbnail bytes from storage first
thumbnailBytes, err := suite.storage.Get(testAttachment.Thumbnail.Path)
thumbnailBytes, err := suite.storage.Get(ctx, testAttachment.Thumbnail.Path)
suite.NoError(err)
// uncache the file from local
testAttachment.Cached = false
err = suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.File.Path)
err = suite.storage.Delete(ctx, testAttachment.File.Path)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
err = suite.storage.Delete(ctx, testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch the thumbnail

6
internal/processing/media/media.go
View file

@ -21,12 +21,12 @@ package media
import (
"context"
"codeberg.org/gruf/go-store/kv"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@ -51,12 +51,12 @@ type processor struct {
tc typeutils.TypeConverter
mediaManager media.Manager
transportController transport.Controller
storage *kv.KVStore
storage storage.Driver
db db.DB
}
// New returns a new media processor.
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *kv.KVStore) Processor {
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage storage.Driver) Processor {
return &processor{
tc: tc,
mediaManager: mediaManager,

6
internal/processing/media/media_test.go
View file

@ -19,7 +19,6 @@
package media_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -27,6 +26,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -37,7 +37,7 @@ type MediaStandardTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
transportController transport.Controller
@ -72,7 +72,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1))
suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage)

6
internal/processing/processor.go
View file

@ -23,7 +23,6 @@ import (
"net/http"
"net/url"
"codeberg.org/gruf/go-store/kv"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -41,6 +40,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/processing/status"
"github.com/superseriousbusiness/gotosocial/internal/processing/streaming"
"github.com/superseriousbusiness/gotosocial/internal/processing/user"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/stream"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@ -251,7 +251,7 @@ type processor struct {
tc typeutils.TypeConverter
oauthServer oauth.Server
mediaManager media.Manager
storage *kv.KVStore
storage storage.Driver
statusTimelines timeline.Manager
db db.DB
filter visibility.Filter
@ -275,7 +275,7 @@ func NewProcessor(
federator federation.Federator,
oauthServer oauth.Server,
mediaManager media.Manager,
storage *kv.KVStore,
storage storage.Driver,
db db.DB,
emailSender email.Sender,
clientWorker *concurrency.WorkerPool[messages.FromClientAPI],

6
internal/processing/processor_test.go
View file

@ -19,7 +19,6 @@
package processing_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -30,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -39,7 +39,7 @@ type ProcessingStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
typeconverter typeutils.TypeConverter
httpClient *testrig.MockHTTPClient
@ -91,7 +91,7 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.typeconverter = testrig.NewTestTypeConverter(suite.db)
suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media")

6
internal/processing/status/status_test.go
View file

@ -19,7 +19,6 @@
package status_test
import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -29,6 +28,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/processing/status"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
@ -39,7 +39,7 @@ type StatusStandardTestSuite struct {
db db.DB
typeConverter typeutils.TypeConverter
tc transport.Controller
storage *kv.KVStore
storage storage.Driver
mediaManager media.Manager
federator federation.Federator
clientWorker *concurrency.WorkerPool[messages.FromClientAPI]
@ -81,7 +81,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
suite.clientWorker = concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker)
suite.storage = testrig.NewTestStorage()
suite.storage = testrig.NewInMemoryStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker)
suite.status = status.New(suite.db, suite.typeConverter, suite.clientWorker, processing.GetParseMentionFunc(suite.db, suite.federator))

50
internal/storage/local.go Normal file
View file

@ -0,0 +1,50 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package storage
import (
"context"
"io"
"net/url"
"codeberg.org/gruf/go-store/kv"
)
type Local struct {
KVStore *kv.KVStore
}
func (l *Local) Get(ctx context.Context, key string) ([]byte, error) {
return l.KVStore.Get(key)
}
func (l *Local) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
return l.KVStore.GetStream(key)
}
func (l *Local) PutStream(ctx context.Context, key string, r io.Reader) error {
return l.KVStore.PutStream(key, r)
}
func (l *Local) Put(ctx context.Context, key string, value []byte) error {
return l.KVStore.Put(key, value)
}
func (l *Local) Delete(ctx context.Context, key string) error {
return l.KVStore.Delete(key)
}
func (l *Local) URL(ctx context.Context, key string) *url.URL {
return nil
}

87
internal/storage/s3.go Normal file
View file

@ -0,0 +1,87 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package storage
import (
"bytes"
"context"
"fmt"
"io"
"mime"
"net/url"
"path"
"time"
"github.com/minio/minio-go/v7"
)
type S3 struct {
mc *minio.Client
bucket string
}
func NewS3(mc *minio.Client, bucket string) *S3 {
return &S3{
mc: mc,
bucket: bucket,
}
}
func (s *S3) Get(ctx context.Context, key string) ([]byte, error) {
r, err := s.GetStream(ctx, key)
if err != nil {
return nil, err
}
defer r.Close()
b, err := io.ReadAll(r)
if err != nil {
return nil, fmt.Errorf("reading data from s3: %w", err)
}
return b, nil
}
func (s *S3) GetStream(ctx context.Context, key string) (io.ReadCloser, error) {
o, err := s.mc.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
err = fmt.Errorf("retrieving object from s3: %w", err)
}
return o, err
}
func (s *S3) PutStream(ctx context.Context, key string, r io.Reader) error {
if _, err := s.mc.PutObject(ctx, s.bucket, key, r, -1, minio.PutObjectOptions{}); err != nil {
return fmt.Errorf("uploading data stream: %w", err)
}
return nil
}
func (s *S3) Put(ctx context.Context, key string, value []byte) error {
if _, err := s.mc.PutObject(ctx, s.bucket, key, bytes.NewBuffer(value), -1, minio.PutObjectOptions{}); err != nil {
return fmt.Errorf("uploading data slice: %w", err)
}
return nil
}
func (s *S3) Delete(ctx context.Context, key string) error {
return s.mc.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
}
func (s *S3) URL(ctx context.Context, key string) *url.URL {
// it's safe to ignore the error here, as we just fall back to fetching the
// file if the url request fails
url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{
"response-content-type": []string{mime.TypeByExtension(path.Ext(key))},
})
return url
}

78
internal/storage/storage.go Normal file
View file

@ -0,0 +1,78 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package storage
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"path"
"codeberg.org/gruf/go-store/kv"
"codeberg.org/gruf/go-store/storage"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/superseriousbusiness/gotosocial/internal/config"
)
var (
ErrNotSupported = errors.New("driver does not suppport functionality")
)
// Driver implements the functionality to store and retrieve blobs
// (images,video,audio)
type Driver interface {
Get(ctx context.Context, key string) ([]byte, error)
GetStream(ctx context.Context, key string) (io.ReadCloser, error)
PutStream(ctx context.Context, key string, r io.Reader) error
Put(ctx context.Context, key string, value []byte) error
Delete(ctx context.Context, key string) error
URL(ctx context.Context, key string) *url.URL
}
func AutoConfig() (Driver, error) {
switch config.GetStorageBackend() {
case "s3":
mc, err := minio.New(config.GetStorageS3Endpoint(), &minio.Options{
Creds: credentials.NewStaticV4(config.GetStorageS3AccessKey(), config.GetStorageS3SecretKey(), ""),
Secure: config.GetStorageS3UseSSL(),
})
if err != nil {
return nil, fmt.Errorf("creating minio client: %w", err)
}
return NewS3(mc, config.GetStorageS3BucketName()), nil
case "local":
storageBasePath := config.GetStorageLocalBasePath()
storage, err := kv.OpenFile(storageBasePath, &storage.DiskConfig{
// Put the store lockfile in the storage dir itself.
// Normally this would not be safe, since we could end up
// overwriting the lockfile if we store a file called 'store.lock'.
// However, in this case it's OK because the keys are set by
// GtS and not the user, so we know we're never going to overwrite it.
LockFile: path.Join(storageBasePath, "store.lock"),
})
if err != nil {
return nil, fmt.Errorf("error creating storage backend: %s", err)
}
return &Local{KVStore: storage}, nil
}
return nil, fmt.Errorf("invalid storage backend %s", config.GetStorageBackend())
}