mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-25 23:13:32 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Clean up/uncache remote media (#407)

Browse source 
* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 • committed by GitHub
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/db/bundb/basic_test.go
View file

@ -43,7 +43,7 @@ func (suite *BasicTestSuite) TestGetAllStatuses() {
s := []*gtsmodel.Status{}
err := suite.db.GetAll(context.Background(), &s)
suite.NoError(err)
suite.Len(s, 14)
suite.Len(s, 15)
}
func (suite *BasicTestSuite) TestGetAllNotNull() {

3
internal/db/bundb/bundb_test.go
View file

@ -55,9 +55,8 @@ func (suite *BunDBStandardTestSuite) SetupSuite() {
}
func (suite *BunDBStandardTestSuite) SetupTest() {
testrig.InitTestLog()
testrig.InitTestConfig()
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
testrig.StandardDBSetup(suite.db, suite.testAccounts)
}

24
internal/db/bundb/media.go
View file

@ -20,6 +20,7 @@ package bundb
import (
"context"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -48,3 +49,26 @@ func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.M
}
return attachment, nil
}
func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
attachments := []*gtsmodel.MediaAttachment{}
q := m.conn.
NewSelect().
Model(&attachments).
Where("media_attachment.cached = true").
Where("media_attachment.avatar = false").
Where("media_attachment.header = false").
Where("media_attachment.created_at < ?", olderThan).
WhereGroup(" AND ", whereNotEmptyAndNotNull("media_attachment.remote_url")).
Order("media_attachment.created_at DESC")
if limit != 0 {
q = q.Limit(limit)
}
if err := q.Scan(ctx); err != nil {
return nil, m.conn.ProcessError(err)
}
return attachments, nil
}

48
internal/db/bundb/media_test.go Normal file
View file

@ -0,0 +1,48 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package bundb_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
type MediaTestSuite struct {
BunDBStandardTestSuite
}
func (suite *MediaTestSuite) TestGetAttachmentByID() {
testAttachment := suite.testAttachments["admin_account_status_1_attachment_1"]
attachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
suite.NoError(err)
suite.NotNil(attachment)
}
func (suite *MediaTestSuite) TestGetOlder() {
attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20)
suite.NoError(err)
suite.Len(attachments, 1)
}
func TestMediaTestSuite(t *testing.T) {
suite.Run(t, new(MediaTestSuite))
}

172
internal/db/bundb/migrations/20220214175650_media_cleanup.go Normal file
View file

@ -0,0 +1,172 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package migrations
import (
"context"
"database/sql"
"time"
previousgtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init"
newgtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20220214175650_media_cleanup"
"github.com/uptrace/bun"
)
func init() {
const batchSize = 100
up := func(ctx context.Context, db *bun.DB) error {
// we need to migrate media attachments into a new table
// see section 6 here: https://www.sqlite.org/lang_altertable.html
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
// create the new media attachments table
if _, err := tx.
NewCreateTable().
ModelTableExpr("new_media_attachments").
Model(&newgtsmodel.MediaAttachment{}).
IfNotExists().
Exec(ctx); err != nil {
return err
}
offset := time.Now()
// migrate existing media attachments into new table
migrateLoop:
for {
oldAttachments := []*previousgtsmodel.MediaAttachment{}
err := tx.
NewSelect().
Model(&oldAttachments).
// subtract a millisecond from the offset just to make sure we're not getting double entries (this happens sometimes)
Where("media_attachment.created_at < ?", offset.Add(-1*time.Millisecond)).
Order("media_attachment.created_at DESC").
Limit(batchSize).
Scan(ctx)
if err != nil && err != sql.ErrNoRows {
// there's been a real error
return err
}
if err == sql.ErrNoRows || len(oldAttachments) == 0 {
// we're finished migrating
break migrateLoop
}
// update the offset to the createdAt time of the oldest media attachment in the slice
offset = oldAttachments[len(oldAttachments)-1].CreatedAt
// for every old attachment, we need to make a new attachment out of it by taking the same values
newAttachments := []*newgtsmodel.MediaAttachment{}
for _, old := range oldAttachments {
new := &newgtsmodel.MediaAttachment{
ID: old.ID,
CreatedAt: old.CreatedAt,
UpdatedAt: old.UpdatedAt,
StatusID: old.StatusID,
URL: old.URL,
RemoteURL: old.RemoteURL,
Type: newgtsmodel.FileType(old.Type),
FileMeta: newgtsmodel.FileMeta{
Original: newgtsmodel.Original{
Width: old.FileMeta.Original.Width,
Height: old.FileMeta.Original.Height,
Size: old.FileMeta.Original.Size,
Aspect: old.FileMeta.Original.Aspect,
},
Small: newgtsmodel.Small{
Width: old.FileMeta.Small.Width,
Height: old.FileMeta.Small.Height,
Size: old.FileMeta.Small.Size,
Aspect: old.FileMeta.Small.Aspect,
},
Focus: newgtsmodel.Focus{
X: old.FileMeta.Focus.X,
Y: old.FileMeta.Focus.Y,
},
},
AccountID: old.AccountID,
Description: old.Description,
ScheduledStatusID: old.ScheduledStatusID,
Blurhash: old.Blurhash,
Processing: newgtsmodel.ProcessingStatus(old.Processing),
File: newgtsmodel.File{
Path: old.File.Path,
ContentType: old.File.ContentType,
FileSize: old.File.FileSize,
UpdatedAt: old.File.UpdatedAt,
},
Thumbnail: newgtsmodel.Thumbnail{
Path: old.Thumbnail.Path,
ContentType: old.Thumbnail.ContentType,
FileSize: old.Thumbnail.FileSize,
UpdatedAt: old.Thumbnail.UpdatedAt,
URL: old.Thumbnail.URL,
RemoteURL: old.Thumbnail.RemoteURL,
},
Avatar: old.Avatar,
Header: old.Header,
Cached: true,
}
newAttachments = append(newAttachments, new)
}
// insert this batch of new attachments, and then continue the loop
if _, err := tx.
NewInsert().
Model(&newAttachments).
ModelTableExpr("new_media_attachments").
Exec(ctx); err != nil {
return err
}
}
// we have all the data we need from the old table, so we can safely drop it now
if _, err := tx.NewDropTable().Model(&previousgtsmodel.MediaAttachment{}).Exec(ctx); err != nil {
return err
}
// rename the new table to the same name as the old table was
if _, err := tx.QueryContext(ctx, "ALTER TABLE new_media_attachments RENAME TO media_attachments;"); err != nil {
return err
}
// add an index to the new table
if _, err := tx.
NewCreateIndex().
Model(&newgtsmodel.MediaAttachment{}).
Index("media_attachments_id_idx").
Column("id").
Exec(ctx); err != nil {
return err
}
return nil
})
}
down := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
return nil
})
}
if err := Migrations.Register(up, down); err != nil {
panic(err)
}
}

117
internal/db/bundb/migrations/20220214175650_media_cleanup/account.go Normal file
View file

@ -0,0 +1,117 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Package gtsmodel contains types used *internally* by GoToSocial and added/removed/selected from the database.
// These types should never be serialized and/or sent out via public APIs, as they contain sensitive information.
// The annotation used on these structs is for handling them via the bun-db ORM.
// See here for more info on bun model annotations: https://bun.uptrace.dev/guide/models.html
package gtsmodel
import (
"crypto/rsa"
"time"
)
// Account represents either a local or a remote fediverse account, gotosocial or otherwise (mastodon, pleroma, etc).
type Account struct {
ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
Username string `validate:"required" bun:",nullzero,notnull,unique:userdomain"` // Username of the account, should just be a string of [a-zA-Z0-9_]. Can be added to domain to create the full username in the form ``[username]@[domain]`` eg., ``user_96@example.org``. Username and domain should be unique *with* each other
Domain string `validate:"omitempty,fqdn" bun:",nullzero,unique:userdomain"` // Domain of the account, will be null if this is a local account, otherwise something like ``example.org``. Should be unique with username.
AvatarMediaAttachmentID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Database ID of the media attachment, if present
AvatarMediaAttachment *MediaAttachment `validate:"-" bun:"rel:belongs-to"` // MediaAttachment corresponding to avatarMediaAttachmentID
AvatarRemoteURL string `validate:"omitempty,url" bun:",nullzero"` // For a non-local account, where can the header be fetched?
HeaderMediaAttachmentID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Database ID of the media attachment, if present
HeaderMediaAttachment *MediaAttachment `validate:"-" bun:"rel:belongs-to"` // MediaAttachment corresponding to headerMediaAttachmentID
HeaderRemoteURL string `validate:"omitempty,url" bun:",nullzero"` // For a non-local account, where can the header be fetched?
DisplayName string `validate:"-" bun:""` // DisplayName for this account. Can be empty, then just the Username will be used for display purposes.
Fields []Field `validate:"-"` // a key/value map of fields that this account has added to their profile
Note string `validate:"-" bun:""` // A note that this account has on their profile (ie., the account's bio/description of themselves)
Memorial bool `validate:"-" bun:",default:false"` // Is this a memorial account, ie., has the user passed away?
AlsoKnownAs string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // This account is associated with x account id (TODO: migrate to be AlsoKnownAsID)
MovedToAccountID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // This account has moved this account id in the database
Bot bool `validate:"-" bun:",default:false"` // Does this account identify itself as a bot?
Reason string `validate:"-" bun:""` // What reason was given for signing up when this account was created?
Locked bool `validate:"-" bun:",default:true"` // Does this account need an approval for new followers?
Discoverable bool `validate:"-" bun:",default:false"` // Should this account be shown in the instance's profile directory?
Privacy Visibility `validate:"required_without=Domain,omitempty,oneof=public unlocked followers_only mutuals_only direct" bun:",nullzero"` // Default post privacy for this account
Sensitive bool `validate:"-" bun:",default:false"` // Set posts from this account to sensitive by default?
Language string `validate:"omitempty,bcp47_language_tag" bun:",nullzero,notnull,default:'en'"` // What language does this account post in?
URI string `validate:"required,url" bun:",nullzero,notnull,unique"` // ActivityPub URI for this account.
URL string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Web URL for this account's profile
LastWebfingeredAt time.Time `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"` // Last time this account was refreshed/located with webfinger.
InboxURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Address of this account's ActivityPub inbox, for sending activity to
OutboxURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Address of this account's activitypub outbox
FollowingURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URI for getting the following list of this account
FollowersURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URI for getting the followers list of this account
FeaturedCollectionURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URL for getting the featured collection list of this account
ActorType string `validate:"oneof=Application Group Organization Person Service" bun:",nullzero,notnull"` // What type of activitypub actor is this account?
PrivateKey *rsa.PrivateKey `validate:"required_without=Domain"` // Privatekey for validating activitypub requests, will only be defined for local accounts
PublicKey *rsa.PublicKey `validate:"required"` // Publickey for encoding activitypub requests, will be defined for both local and remote accounts
PublicKeyURI string `validate:"required,url" bun:",nullzero,notnull,unique"` // Web-reachable location of this account's public key
SensitizedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account set to have all its media shown as sensitive?
SilencedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account silenced (eg., statuses only visible to followers, not public)?
SuspendedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account suspended (eg., don't allow it to log in/post, don't accept media/posts from this account)
HideCollections bool `validate:"-" bun:",default:false"` // Hide this account's collections
SuspensionOrigin string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // id of the database entry that caused this account to become suspended -- can be an account ID or a domain block ID
}
// Field represents a key value field on an account, for things like pronouns, website, etc.
// VerifiedAt is optional, to be used only if Value is a URL to a webpage that contains the
// username of the user.
type Field struct {
Name string `validate:"required"` // Name of this field.
Value string `validate:"required"` // Value of this field.
VerifiedAt time.Time `validate:"-" bun:",nullzero"` // This field was verified at (optional).
}
// Relationship describes a requester's relationship with another account.
type Relationship struct {
ID string // The account id.
Following bool // Are you following this user?
ShowingReblogs bool // Are you receiving this user's boosts in your home timeline?
Notifying bool // Have you enabled notifications for this user?
FollowedBy bool // Are you followed by this user?
Blocking bool // Are you blocking this user?
BlockedBy bool // Is this user blocking you?
Muting bool // Are you muting this user?
MutingNotifications bool // Are you muting notifications from this user?
Requested bool // Do you have a pending follow request for this user?
DomainBlocking bool // Are you blocking this user's domain?
Endorsed bool // Are you featuring this user on your profile?
Note string // Your note on this account.
}
// Visibility represents the visibility granularity of a status.
type Visibility string
const (
// VisibilityPublic means this status will be visible to everyone on all timelines.
VisibilityPublic Visibility = "public"
// VisibilityUnlocked means this status will be visible to everyone, but will only show on home timeline to followers, and in lists.
VisibilityUnlocked Visibility = "unlocked"
// VisibilityFollowersOnly means this status is viewable to followers only.
VisibilityFollowersOnly Visibility = "followers_only"
// VisibilityMutualsOnly means this status is visible to mutual followers only.
VisibilityMutualsOnly Visibility = "mutuals_only"
// VisibilityDirect means this status is visible only to mentioned recipients.
VisibilityDirect Visibility = "direct"
// VisibilityDefault is used when no other setting can be found.
VisibilityDefault Visibility = VisibilityUnlocked
)

116
internal/db/bundb/migrations/20220214175650_media_cleanup/mediaattachment.go Normal file
View file

@ -0,0 +1,116 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gtsmodel
import (
"time"
)
type MediaAttachment struct {
ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
StatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // ID of the status to which this is attached
URL string `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on *this* server
RemoteURL string `validate:"required_without=URL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on a remote server (empty for local media)
Type FileType `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"` // Type of file (image/gif/audio/video)
FileMeta FileMeta `validate:"required" bun:",embed:filemeta_,nullzero,notnull"` // Metadata about the file
AccountID string `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"` // To which account does this attachment belong
Account *Account `validate:"-" bun:"rel:has-one"` // Account corresponding to accountID
Description string `validate:"-" bun:""` // Description of the attachment (for screenreaders)
ScheduledStatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // To which scheduled status does this attachment belong
Blurhash string `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment
Processing ProcessingStatus `validate:"oneof=0 1 2 666" bun:",notnull,default:2"` // What is the processing status of this attachment
File File `validate:"required" bun:",embed:file_,notnull,nullzero"` // metadata for the whole file
Thumbnail Thumbnail `validate:"required" bun:",embed:thumbnail_,notnull,nullzero"` // small image thumbnail derived from a larger image, video, or audio file.
Avatar bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as an avatar?
Header bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as a header?
Cached bool `validate:"-" bun:",notnull"` // Is this attachment currently cached by our instance?
}
// File refers to the metadata for the whole file
type File struct {
Path string `validate:"required,file" bun:",nullzero,notnull"` // Path of the file in storage.
ContentType string `validate:"required" bun:",nullzero,notnull"` // MIME content type of the file.
FileSize int `validate:"required" bun:",notnull"` // File size in bytes
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the file last updated.
}
// Thumbnail refers to a small image thumbnail derived from a larger image, video, or audio file.
type Thumbnail struct {
Path string `validate:"required,file" bun:",nullzero,notnull"` // Path of the file in storage.
ContentType string `validate:"required" bun:",nullzero,notnull"` // MIME content type of the file.
FileSize int `validate:"required" bun:",notnull"` // File size in bytes
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the file last updated.
URL string `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"` // What is the URL of the thumbnail on the local server
RemoteURL string `validate:"required_without=URL,omitempty,url" bun:",nullzero"` // What is the remote URL of the thumbnail (empty for local media)
}
// ProcessingStatus refers to how far along in the processing stage the attachment is.
type ProcessingStatus int
// MediaAttachment processing states.
const (
ProcessingStatusReceived ProcessingStatus = 0 // ProcessingStatusReceived indicates the attachment has been received and is awaiting processing. No thumbnail available yet.
ProcessingStatusProcessing ProcessingStatus = 1 // ProcessingStatusProcessing indicates the attachment is currently being processed. Thumbnail is available but full media is not.
ProcessingStatusProcessed ProcessingStatus = 2 // ProcessingStatusProcessed indicates the attachment has been fully processed and is ready to be served.
ProcessingStatusError ProcessingStatus = 666 // ProcessingStatusError indicates something went wrong processing the attachment and it won't be tried again--these can be deleted.
)
// FileType refers to the file type of the media attaachment.
type FileType string
// MediaAttachment file types.
const (
FileTypeImage FileType = "Image" // FileTypeImage is for jpegs and pngs
FileTypeGif FileType = "Gif" // FileTypeGif is for native gifs and soundless videos that have been converted to gifs
FileTypeAudio FileType = "Audio" // FileTypeAudio is for audio-only files (no video)
FileTypeVideo FileType = "Video" // FileTypeVideo is for files with audio + visual
FileTypeUnknown FileType = "Unknown" // FileTypeUnknown is for unknown file types (surprise surprise!)
)
// FileMeta describes metadata about the actual contents of the file.
type FileMeta struct {
Original Original `validate:"required" bun:"embed:original_"`
Small Small `bun:"embed:small_"`
Focus Focus `bun:"embed:focus_"`
}
// Small can be used for a thumbnail of any media type
type Small struct {
Width int `validate:"required_with=Height Size Aspect"` // width in pixels
Height int `validate:"required_with=Width Size Aspect"` // height in pixels
Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
Aspect float64 `validate:"required_with=Widhth Height Size"` // aspect ratio (width / height)
}
// Original can be used for original metadata for any media type
type Original struct {
Width int `validate:"required_with=Height Size Aspect"` // width in pixels
Height int `validate:"required_with=Width Size Aspect"` // height in pixels
Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
Aspect float64 `validate:"required_with=Widhth Height Size"` // aspect ratio (width / height)
}
// Focus describes the 'center' of the image for display purposes.
// X and Y should each be between -1 and 1
type Focus struct {
X float32 `validate:"omitempty,max=1,min=-1"`
Y float32 `validate:"omitempty,max=1,min=-1"`
}

14
internal/db/bundb/util.go
View file

@ -37,6 +37,20 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
}
}
// whereNotEmptyAndNotNull is a convenience function to return a bun WhereGroup that specifies
// that the given column should be NEITHER an empty string NOR null.
//
// Use it as follows:
//
// q = q.WhereGroup(" AND ", whereNotEmptyAndNotNull("whatever_column"))
func whereNotEmptyAndNotNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
return func(q *bun.SelectQuery) *bun.SelectQuery {
return q.
Where("? IS NOT NULL", bun.Ident(column)).
Where("? != ''", bun.Ident(column))
}
}
// updateWhere parses []db.Where and adds it to the given update query.
func updateWhere(q *bun.UpdateQuery, where []db.Where) {
for _, w := range where {

7
internal/db/media.go
View file

@ -20,6 +20,7 @@ package db
import (
"context"
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
@ -28,4 +29,10 @@ import (
type Media interface {
// GetAttachmentByID gets a single attachment by its ID
GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
// GetRemoteOlderThan gets limit n remote media attachments older than the given olderThan time.
// These will be returned in order of attachment.created_at descending (newest to oldest in other words).
//
// The selected media attachments will be those with both a URL and a RemoteURL filled in.
// In other words, media attachments that originated remotely, and that we currently have cached locally.
GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error)
}