mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 14:52:26 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[chore/performance] Update media prune logic, add extra CLI command (#1474)

Browse source 
* start updating media prune stuff a wee bit

* continue prune / uncache work

* more tidying + consistency stuff

* add prune CLI command

* docs

* arg
This commit is contained in:
tobi 2023-02-11 12:48:38 +01:00 • committed by GitHub
commit 40bc03e717
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 1113 additions and 1090 deletions
Download patch file Download diff file Expand all files Collapse all files

46
internal/db/bundb/media.go
View file

@ -72,6 +72,23 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l
return attachments, nil
}
func (m *mediaDB) CountRemoteOlderThan(ctx context.Context, olderThan time.Time) (int, db.Error) {
q := m.conn.
NewSelect().
TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
Column("media_attachment.id").
Where("? = ?", bun.Ident("media_attachment.cached"), true).
Where("? < ?", bun.Ident("media_attachment.created_at"), olderThan).
WhereGroup(" AND ", whereNotEmptyAndNotNull("media_attachment.remote_url"))
count, err := q.Count(ctx)
if err != nil {
return 0, m.conn.ProcessError(err)
}
return count, nil
}
func (m *mediaDB) GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
attachments := []*gtsmodel.MediaAttachment{}
@ -98,7 +115,7 @@ func (m *mediaDB) GetAvatarsAndHeaders(ctx context.Context, maxID string, limit
return attachments, nil
}
func (m *mediaDB) GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
func (m *mediaDB) GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
attachments := []*gtsmodel.MediaAttachment{}
q := m.newMediaQ(&attachments).
@ -107,11 +124,8 @@ func (m *mediaDB) GetLocalUnattachedOlderThan(ctx context.Context, olderThan tim
Where("? = ?", bun.Ident("media_attachment.header"), false).
Where("? < ?", bun.Ident("media_attachment.created_at"), olderThan).
Where("? IS NULL", bun.Ident("media_attachment.remote_url")).
Where("? IS NULL", bun.Ident("media_attachment.status_id"))
if maxID != "" {
q = q.Where("? < ?", bun.Ident("media_attachment.id"), maxID)
}
Where("? IS NULL", bun.Ident("media_attachment.status_id")).
Order("media_attachment.created_at DESC")
if limit != 0 {
q = q.Limit(limit)
@ -123,3 +137,23 @@ func (m *mediaDB) GetLocalUnattachedOlderThan(ctx context.Context, olderThan tim
return attachments, nil
}
func (m *mediaDB) CountLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time) (int, db.Error) {
q := m.conn.
NewSelect().
TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
Column("media_attachment.id").
Where("? = ?", bun.Ident("media_attachment.cached"), true).
Where("? = ?", bun.Ident("media_attachment.avatar"), false).
Where("? = ?", bun.Ident("media_attachment.header"), false).
Where("? < ?", bun.Ident("media_attachment.created_at"), olderThan).
Where("? IS NULL", bun.Ident("media_attachment.remote_url")).
Where("? IS NULL", bun.Ident("media_attachment.status_id"))
count, err := q.Count(ctx)
if err != nil {
return 0, m.conn.ProcessError(err)
}
return count, nil
}

2
internal/db/bundb/media_test.go
View file

@ -55,7 +55,7 @@ func (suite *MediaTestSuite) TestGetAvisAndHeaders() {
func (suite *MediaTestSuite) TestGetLocalUnattachedOlderThan() {
ctx := context.Background()
attachments, err := suite.db.GetLocalUnattachedOlderThan(ctx, testrig.TimeMustParse("2090-06-04T13:12:00Z"), "", 10)
attachments, err := suite.db.GetLocalUnattachedOlderThan(ctx, testrig.TimeMustParse("2090-06-04T13:12:00Z"), 10)
suite.NoError(err)
suite.Len(attachments, 1)
}

13
internal/db/media.go
View file

@ -37,6 +37,11 @@ type Media interface {
// In other words, media attachments that originated remotely, and that we currently have cached locally.
GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error)
// CountRemoteOlderThan is like GetRemoteOlderThan, except instead of getting limit n attachments,
// it just counts how many remote attachments in the database (including avatars and headers) meet
// the olderThan criteria.
CountRemoteOlderThan(ctx context.Context, olderThan time.Time) (int, Error)
// GetAvatarsAndHeaders fetches limit n avatars and headers with an id < maxID. These headers
// and avis may be in use or not; the caller should check this if it's important.
GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
@ -44,5 +49,11 @@ type Media interface {
// GetLocalUnattachedOlderThan fetches limit n local media attachments (including avatars and headers), older than
// the given time, which aren't header or avatars, and aren't attached to a status. In other words, attachments which were
// uploaded but never used for whatever reason, or attachments that were attached to a status which was subsequently deleted.
GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
//
// These will be returned in order of attachment.created_at descending (newest to oldest in other words).
GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error)
// CountLocalUnattachedOlderThan is like GetLocalUnattachedOlderThan, except instead of getting limit n attachments,
// it just counts how many local attachments in the database meet the olderThan criteria.
CountLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time) (int, Error)
}