mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-10-29 14:52:26 -05:00
[performance] cache media attachments (#1525)
* replace concurrency worker pools with base models in State.Workers, update code and tests accordingly * add media attachment caching, slightly tweak default cache config * further tweak default cache config values * replace other media attachment db calls to go through cache * update envparsing test * fix delete media attachment sql * fix media sql query * invalidate cached media entries during status create / update * fix envparsing test * fix typo in panic log message... * add 'updated_at' column during UpdateAttachment * remove unused func --------- Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
GitHub
parent
5be59f4a25
commit
a8e6bdfa33
15 changed files with 235 additions and 61 deletions
|
|
@ -34,39 +34,69 @@ type mediaDB struct {
|
|||
state *state.State
|
||||
}
|
||||
|
||||
func (m *mediaDB) newMediaQ(i *gtsmodel.MediaAttachment) *bun.SelectQuery {
|
||||
return m.conn.
|
||||
NewSelect().
|
||||
Model(i)
|
||||
}
|
||||
|
||||
func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, db.Error) {
|
||||
return m.getAttachment(
|
||||
ctx,
|
||||
"ID",
|
||||
func(attachment *gtsmodel.MediaAttachment) error {
|
||||
return m.newMediaQ(attachment).Where("? = ?", bun.Ident("media_attachment.id"), id).Scan(ctx)
|
||||
return m.conn.NewSelect().
|
||||
Model(attachment).
|
||||
Where("? = ?", bun.Ident("media_attachment.id"), id).
|
||||
Scan(ctx)
|
||||
},
|
||||
id,
|
||||
)
|
||||
}
|
||||
|
||||
func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) {
|
||||
attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids))
|
||||
func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) {
|
||||
return m.state.Caches.GTS.Media().Load(lookup, func() (*gtsmodel.MediaAttachment, error) {
|
||||
var attachment gtsmodel.MediaAttachment
|
||||
|
||||
for _, id := range ids {
|
||||
// Attempt fetch from DB
|
||||
attachment, err := m.GetAttachmentByID(ctx, id)
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "error getting attachment %q: %v", id, err)
|
||||
continue
|
||||
// Not cached! Perform database query
|
||||
if err := dbQuery(&attachment); err != nil {
|
||||
return nil, m.conn.ProcessError(err)
|
||||
}
|
||||
|
||||
// Append attachment
|
||||
attachments = append(attachments, attachment)
|
||||
return &attachment, nil
|
||||
}, keyParts...)
|
||||
}
|
||||
|
||||
func (m *mediaDB) PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error {
|
||||
return m.state.Caches.GTS.Media().Store(media, func() error {
|
||||
_, err := m.conn.NewInsert().Model(media).Exec(ctx)
|
||||
return m.conn.ProcessError(err)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *mediaDB) UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error {
|
||||
media.UpdatedAt = time.Now()
|
||||
if len(columns) > 0 {
|
||||
// If we're updating by column, ensure "updated_at" is included.
|
||||
columns = append(columns, "updated_at")
|
||||
}
|
||||
|
||||
return attachments, nil
|
||||
return m.state.Caches.GTS.Media().Store(media, func() error {
|
||||
_, err := m.conn.NewUpdate().
|
||||
Model(media).
|
||||
Where("? = ?", bun.Ident("media_attachment.id"), media.ID).
|
||||
Column(columns...).
|
||||
Exec(ctx)
|
||||
return m.conn.ProcessError(err)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error {
|
||||
// Attempt to delete from database.
|
||||
if _, err := m.conn.NewDelete().
|
||||
TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
|
||||
Where("? = ?", bun.Ident("media_attachment.id"), id).
|
||||
Exec(ctx); err != nil {
|
||||
return m.conn.ProcessError(err)
|
||||
}
|
||||
|
||||
// Invalidate this media item from the cache.
|
||||
m.state.Caches.GTS.Media().Invalidate("ID", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
|
||||
|
|
@ -183,14 +213,20 @@ func (m *mediaDB) CountLocalUnattachedOlderThan(ctx context.Context, olderThan t
|
|||
return count, nil
|
||||
}
|
||||
|
||||
func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) {
|
||||
// Fetch attachment from database
|
||||
// todo: cache this lookup
|
||||
attachment := new(gtsmodel.MediaAttachment)
|
||||
func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) {
|
||||
attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids))
|
||||
|
||||
if err := dbQuery(attachment); err != nil {
|
||||
return nil, m.conn.ProcessError(err)
|
||||
for _, id := range ids {
|
||||
// Attempt fetch from DB
|
||||
attachment, err := m.GetAttachmentByID(ctx, id)
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "error getting attachment %q: %v", id, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Append attachment
|
||||
attachments = append(attachments, attachment)
|
||||
}
|
||||
|
||||
return attachment, nil
|
||||
return attachments, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g
|
|||
}
|
||||
|
||||
func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Error {
|
||||
return s.state.Caches.GTS.Status().Store(status, func() error {
|
||||
err := s.state.Caches.GTS.Status().Store(status, func() error {
|
||||
// It is safe to run this database transaction within cache.Store
|
||||
// as the cache does not attempt a mutex lock until AFTER hook.
|
||||
//
|
||||
|
|
@ -248,6 +248,17 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Er
|
|||
return err
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
// already processed
|
||||
return err
|
||||
}
|
||||
|
||||
for _, id := range status.AttachmentIDs {
|
||||
// Clear updated media attachment IDs from cache
|
||||
s.state.Caches.GTS.Media().Invalidate("ID", id)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) db.Error {
|
||||
|
|
@ -317,11 +328,18 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co
|
|||
Exec(ctx)
|
||||
return err
|
||||
}); err != nil {
|
||||
// already processed
|
||||
return err
|
||||
}
|
||||
|
||||
// Drop any old value from cache by this ID
|
||||
for _, id := range status.AttachmentIDs {
|
||||
// Clear updated media attachment IDs from cache
|
||||
s.state.Caches.GTS.Media().Invalidate("ID", id)
|
||||
}
|
||||
|
||||
// Drop any old status value from cache by this ID
|
||||
s.state.Caches.GTS.Status().Invalidate("ID", status.ID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,9 +27,18 @@ import (
|
|||
|
||||
// Media contains functions related to creating/getting/removing media attachments.
|
||||
type Media interface {
|
||||
// GetAttachmentByID gets a single attachment by its ID
|
||||
// GetAttachmentByID gets a single attachment by its ID.
|
||||
GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
|
||||
|
||||
// PutAttachment inserts the given attachment into the database.
|
||||
PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error
|
||||
|
||||
// UpdateAttachment will update the given attachment in the database.
|
||||
UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error
|
||||
|
||||
// DeleteAttachment deletes the attachment with given ID from the database.
|
||||
DeleteAttachment(ctx context.Context, id string) error
|
||||
|
||||
// GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given
|
||||
// olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words).
|
||||
//
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue