mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-30 23:23:32 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[performance] cache media attachments (#1525)

Browse source 
* replace concurrency worker pools with base models in State.Workers, update code and tests accordingly

* add media attachment caching, slightly tweak default cache config

* further tweak default cache config values

* replace other media attachment db calls to go through cache

* update envparsing test

* fix delete media attachment sql

* fix media sql query

* invalidate cached media entries during status create / update

* fix envparsing test

* fix typo in panic log message...

* add 'updated_at' column during UpdateAttachment

* remove unused func

---------

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2023-03-03 23:02:23 +00:00 • committed by GitHub
commit a8e6bdfa33
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 235 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

24
internal/cache/gts.go vendored
View file

@ -54,6 +54,9 @@ type GTSCaches interface {
// Mention provides access to the gtsmodel Mention database cache.
Mention() *result.Cache[*gtsmodel.Mention]
// Media provides access to the gtsmodel Media database cache.
Media() *result.Cache[*gtsmodel.MediaAttachment]
// Notification provides access to the gtsmodel Notification database cache.
Notification() *result.Cache[*gtsmodel.Notification]
@ -81,6 +84,7 @@ type gtsCaches struct {
domainBlock *domain.BlockCache
emoji *result.Cache[*gtsmodel.Emoji]
emojiCategory *result.Cache[*gtsmodel.EmojiCategory]
media *result.Cache[*gtsmodel.MediaAttachment]
mention *result.Cache[*gtsmodel.Mention]
notification *result.Cache[*gtsmodel.Notification]
report *result.Cache[*gtsmodel.Report]
@ -95,6 +99,7 @@ func (c *gtsCaches) Init() {
c.initDomainBlock()
c.initEmoji()
c.initEmojiCategory()
c.initMedia()
c.initMention()
c.initNotification()
c.initReport()
@ -119,6 +124,9 @@ func (c *gtsCaches) Start() {
tryUntil("starting gtsmodel.EmojiCategory cache", 5, func() bool {
return c.emojiCategory.Start(config.GetCacheGTSEmojiCategorySweepFreq())
})
tryUntil("starting gtsmodel.MediaAttachment cache", 5, func() bool {
return c.media.Start(config.GetCacheGTSMediaSweepFreq())
})
tryUntil("starting gtsmodel.Mention cache", 5, func() bool {
return c.mention.Start(config.GetCacheGTSMentionSweepFreq())
})
@ -145,6 +153,7 @@ func (c *gtsCaches) Stop() {
tryUntil("stopping gtsmodel.DomainBlock cache", 5, c.domainBlock.Stop)
tryUntil("stopping gtsmodel.Emoji cache", 5, c.emoji.Stop)
tryUntil("stopping gtsmodel.EmojiCategory cache", 5, c.emojiCategory.Stop)
tryUntil("stopping gtsmodel.MediaAttachment cache", 5, c.media.Stop)
tryUntil("stopping gtsmodel.Mention cache", 5, c.mention.Stop)
tryUntil("stopping gtsmodel.Notification cache", 5, c.notification.Stop)
tryUntil("stopping gtsmodel.Report cache", 5, c.report.Stop)
@ -173,6 +182,10 @@ func (c *gtsCaches) EmojiCategory() *result.Cache[*gtsmodel.EmojiCategory] {
return c.emojiCategory
}
func (c *gtsCaches) Media() *result.Cache[*gtsmodel.MediaAttachment] {
return c.media
}
func (c *gtsCaches) Mention() *result.Cache[*gtsmodel.Mention] {
return c.mention
}
@ -258,6 +271,17 @@ func (c *gtsCaches) initEmojiCategory() {
c.emojiCategory.SetTTL(config.GetCacheGTSEmojiCategoryTTL(), true)
}
func (c *gtsCaches) initMedia() {
c.media = result.New([]result.Lookup{
{Name: "ID"},
}, func(m1 *gtsmodel.MediaAttachment) *gtsmodel.MediaAttachment {
m2 := new(gtsmodel.MediaAttachment)
*m2 = *m1
return m2
}, config.GetCacheGTSMediaMaxSize())
c.media.SetTTL(config.GetCacheGTSMediaTTL(), true)
}
func (c *gtsCaches) initMention() {
c.mention = result.New([]result.Lookup{
{Name: "ID"},

4
internal/config/config.go
View file

@ -177,6 +177,10 @@ type GTSCacheConfiguration struct {
EmojiCategoryTTL time.Duration `name:"emoji-category-ttl"`
EmojiCategorySweepFreq time.Duration `name:"emoji-category-sweep-freq"`
MediaMaxSize int `name:"media-max-size"`
MediaTTL time.Duration `name:"media-ttl"`
MediaSweepFreq time.Duration `name:"media-sweep-freq"`
MentionMaxSize int `name:"mention-max-size"`
MentionTTL time.Duration `name:"mention-ttl"`
MentionSweepFreq time.Duration `name:"mention-sweep-freq"`

26
internal/config/defaults.go
View file

@ -116,13 +116,13 @@ var Defaults = Configuration{
Cache: CacheConfiguration{
GTS: GTSCacheConfiguration{
AccountMaxSize: 100,
AccountMaxSize: 500,
AccountTTL: time.Minute * 5,
AccountSweepFreq: time.Second * 10,
AccountSweepFreq: time.Second * 30,
BlockMaxSize: 100,
BlockTTL: time.Minute * 5,
BlockSweepFreq: time.Second * 10,
BlockSweepFreq: time.Second * 30,
DomainBlockMaxSize: 1000,
DomainBlockTTL: time.Hour * 24,
@ -130,35 +130,39 @@ var Defaults = Configuration{
EmojiMaxSize: 500,
EmojiTTL: time.Minute * 5,
EmojiSweepFreq: time.Second * 10,
EmojiSweepFreq: time.Second * 30,
EmojiCategoryMaxSize: 100,
EmojiCategoryTTL: time.Minute * 5,
EmojiCategorySweepFreq: time.Second * 10,
EmojiCategorySweepFreq: time.Second * 30,
MediaMaxSize: 500,
MediaTTL: time.Minute * 5,
MediaSweepFreq: time.Second * 30,
MentionMaxSize: 500,
MentionTTL: time.Minute * 5,
MentionSweepFreq: time.Second * 10,
MentionSweepFreq: time.Second * 30,
NotificationMaxSize: 500,
NotificationTTL: time.Minute * 5,
NotificationSweepFreq: time.Second * 10,
NotificationSweepFreq: time.Second * 30,
ReportMaxSize: 100,
ReportTTL: time.Minute * 5,
ReportSweepFreq: time.Second * 10,
ReportSweepFreq: time.Second * 30,
StatusMaxSize: 500,
StatusTTL: time.Minute * 5,
StatusSweepFreq: time.Second * 10,
StatusSweepFreq: time.Second * 30,
TombstoneMaxSize: 100,
TombstoneTTL: time.Minute * 5,
TombstoneSweepFreq: time.Second * 10,
TombstoneSweepFreq: time.Second * 30,
UserMaxSize: 100,
UserTTL: time.Minute * 5,
UserSweepFreq: time.Second * 10,
UserSweepFreq: time.Second * 30,
},
},

75
internal/config/helpers.gen.go
View file

@ -2426,6 +2426,81 @@ func GetCacheGTSEmojiCategorySweepFreq() time.Duration {
// SetCacheGTSEmojiCategorySweepFreq safely sets the value for global configuration 'Cache.GTS.EmojiCategorySweepFreq' field
func SetCacheGTSEmojiCategorySweepFreq(v time.Duration) { global.SetCacheGTSEmojiCategorySweepFreq(v) }
// GetCacheGTSMediaMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
func (st *ConfigState) GetCacheGTSMediaMaxSize() (v int) {
st.mutex.Lock()
v = st.config.Cache.GTS.MediaMaxSize
st.mutex.Unlock()
return
}
// SetCacheGTSMediaMaxSize safely sets the Configuration value for state's 'Cache.GTS.MediaMaxSize' field
func (st *ConfigState) SetCacheGTSMediaMaxSize(v int) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.MediaMaxSize = v
st.reloadToViper()
}
// CacheGTSMediaMaxSizeFlag returns the flag name for the 'Cache.GTS.MediaMaxSize' field
func CacheGTSMediaMaxSizeFlag() string { return "cache-gts-media-max-size" }
// GetCacheGTSMediaMaxSize safely fetches the value for global configuration 'Cache.GTS.MediaMaxSize' field
func GetCacheGTSMediaMaxSize() int { return global.GetCacheGTSMediaMaxSize() }
// SetCacheGTSMediaMaxSize safely sets the value for global configuration 'Cache.GTS.MediaMaxSize' field
func SetCacheGTSMediaMaxSize(v int) { global.SetCacheGTSMediaMaxSize(v) }
// GetCacheGTSMediaTTL safely fetches the Configuration value for state's 'Cache.GTS.MediaTTL' field
func (st *ConfigState) GetCacheGTSMediaTTL() (v time.Duration) {
st.mutex.Lock()
v = st.config.Cache.GTS.MediaTTL
st.mutex.Unlock()
return
}
// SetCacheGTSMediaTTL safely sets the Configuration value for state's 'Cache.GTS.MediaTTL' field
func (st *ConfigState) SetCacheGTSMediaTTL(v time.Duration) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.MediaTTL = v
st.reloadToViper()
}
// CacheGTSMediaTTLFlag returns the flag name for the 'Cache.GTS.MediaTTL' field
func CacheGTSMediaTTLFlag() string { return "cache-gts-media-ttl" }
// GetCacheGTSMediaTTL safely fetches the value for global configuration 'Cache.GTS.MediaTTL' field
func GetCacheGTSMediaTTL() time.Duration { return global.GetCacheGTSMediaTTL() }
// SetCacheGTSMediaTTL safely sets the value for global configuration 'Cache.GTS.MediaTTL' field
func SetCacheGTSMediaTTL(v time.Duration) { global.SetCacheGTSMediaTTL(v) }
// GetCacheGTSMediaSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field
func (st *ConfigState) GetCacheGTSMediaSweepFreq() (v time.Duration) {
st.mutex.Lock()
v = st.config.Cache.GTS.MediaSweepFreq
st.mutex.Unlock()
return
}
// SetCacheGTSMediaSweepFreq safely sets the Configuration value for state's 'Cache.GTS.MediaSweepFreq' field
func (st *ConfigState) SetCacheGTSMediaSweepFreq(v time.Duration) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.MediaSweepFreq = v
st.reloadToViper()
}
// CacheGTSMediaSweepFreqFlag returns the flag name for the 'Cache.GTS.MediaSweepFreq' field
func CacheGTSMediaSweepFreqFlag() string { return "cache-gts-media-sweep-freq" }
// GetCacheGTSMediaSweepFreq safely fetches the value for global configuration 'Cache.GTS.MediaSweepFreq' field
func GetCacheGTSMediaSweepFreq() time.Duration { return global.GetCacheGTSMediaSweepFreq() }
// SetCacheGTSMediaSweepFreq safely sets the value for global configuration 'Cache.GTS.MediaSweepFreq' field
func SetCacheGTSMediaSweepFreq(v time.Duration) { global.SetCacheGTSMediaSweepFreq(v) }
// GetCacheGTSMentionMaxSize safely fetches the Configuration value for state's 'Cache.GTS.MentionMaxSize' field
func (st *ConfigState) GetCacheGTSMentionMaxSize() (v int) {
st.mutex.Lock()

86
internal/db/bundb/media.go
View file

@ -34,39 +34,69 @@ type mediaDB struct {
state *state.State
}
func (m *mediaDB) newMediaQ(i *gtsmodel.MediaAttachment) *bun.SelectQuery {
return m.conn.
NewSelect().
Model(i)
}
func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, db.Error) {
return m.getAttachment(
ctx,
"ID",
func(attachment *gtsmodel.MediaAttachment) error {
return m.newMediaQ(attachment).Where("? = ?", bun.Ident("media_attachment.id"), id).Scan(ctx)
return m.conn.NewSelect().
Model(attachment).
Where("? = ?", bun.Ident("media_attachment.id"), id).
Scan(ctx)
},
id,
)
}
func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) {
attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids))
func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) {
return m.state.Caches.GTS.Media().Load(lookup, func() (*gtsmodel.MediaAttachment, error) {
var attachment gtsmodel.MediaAttachment
for _, id := range ids {
// Attempt fetch from DB
attachment, err := m.GetAttachmentByID(ctx, id)
if err != nil {
log.Errorf(ctx, "error getting attachment %q: %v", id, err)
continue
// Not cached! Perform database query
if err := dbQuery(&attachment); err != nil {
return nil, m.conn.ProcessError(err)
}
// Append attachment
attachments = append(attachments, attachment)
return &attachment, nil
}, keyParts...)
}
func (m *mediaDB) PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error {
return m.state.Caches.GTS.Media().Store(media, func() error {
_, err := m.conn.NewInsert().Model(media).Exec(ctx)
return m.conn.ProcessError(err)
})
}
func (m *mediaDB) UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error {
media.UpdatedAt = time.Now()
if len(columns) > 0 {
// If we're updating by column, ensure "updated_at" is included.
columns = append(columns, "updated_at")
}
return attachments, nil
return m.state.Caches.GTS.Media().Store(media, func() error {
_, err := m.conn.NewUpdate().
Model(media).
Where("? = ?", bun.Ident("media_attachment.id"), media.ID).
Column(columns...).
Exec(ctx)
return m.conn.ProcessError(err)
})
}
func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error {
// Attempt to delete from database.
if _, err := m.conn.NewDelete().
TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
Where("? = ?", bun.Ident("media_attachment.id"), id).
Exec(ctx); err != nil {
return m.conn.ProcessError(err)
}
// Invalidate this media item from the cache.
m.state.Caches.GTS.Media().Invalidate("ID", id)
return nil
}
func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
@ -183,14 +213,20 @@ func (m *mediaDB) CountLocalUnattachedOlderThan(ctx context.Context, olderThan t
return count, nil
}
func (m *mediaDB) getAttachment(ctx context.Context, lookup string, dbQuery func(*gtsmodel.MediaAttachment) error, keyParts ...any) (*gtsmodel.MediaAttachment, db.Error) {
// Fetch attachment from database
// todo: cache this lookup
attachment := new(gtsmodel.MediaAttachment)
func (m *mediaDB) getAttachments(ctx context.Context, ids []string) ([]*gtsmodel.MediaAttachment, db.Error) {
attachments := make([]*gtsmodel.MediaAttachment, 0, len(ids))
if err := dbQuery(attachment); err != nil {
return nil, m.conn.ProcessError(err)
for _, id := range ids {
// Attempt fetch from DB
attachment, err := m.GetAttachmentByID(ctx, id)
if err != nil {
log.Errorf(ctx, "error getting attachment %q: %v", id, err)
continue
}
// Append attachment
attachments = append(attachments, attachment)
}
return attachment, nil
return attachments, nil
}

22
internal/db/bundb/status.go
View file

@ -188,7 +188,7 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g
}
func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Error {
return s.state.Caches.GTS.Status().Store(status, func() error {
err := s.state.Caches.GTS.Status().Store(status, func() error {
// It is safe to run this database transaction within cache.Store
// as the cache does not attempt a mutex lock until AFTER hook.
//
@ -248,6 +248,17 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Er
return err
})
})
if err != nil {
// already processed
return err
}
for _, id := range status.AttachmentIDs {
// Clear updated media attachment IDs from cache
s.state.Caches.GTS.Media().Invalidate("ID", id)
}
return nil
}
func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, columns ...string) db.Error {
@ -317,11 +328,18 @@ func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status, co
Exec(ctx)
return err
}); err != nil {
// already processed
return err
}
// Drop any old value from cache by this ID
for _, id := range status.AttachmentIDs {
// Clear updated media attachment IDs from cache
s.state.Caches.GTS.Media().Invalidate("ID", id)
}
// Drop any old status value from cache by this ID
s.state.Caches.GTS.Status().Invalidate("ID", status.ID)
return nil
}

11
internal/db/media.go
View file

@ -27,9 +27,18 @@ import (
// Media contains functions related to creating/getting/removing media attachments.
type Media interface {
// GetAttachmentByID gets a single attachment by its ID
// GetAttachmentByID gets a single attachment by its ID.
GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
// PutAttachment inserts the given attachment into the database.
PutAttachment(ctx context.Context, media *gtsmodel.MediaAttachment) error
// UpdateAttachment will update the given attachment in the database.
UpdateAttachment(ctx context.Context, media *gtsmodel.MediaAttachment, columns ...string) error
// DeleteAttachment deletes the attachment with given ID from the database.
DeleteAttachment(ctx context.Context, id string) error
// GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given
// olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words).
//

8
internal/media/processingmedia.go
View file

@ -123,13 +123,13 @@ func (p *ProcessingMedia) load(ctx context.Context) (*gtsmodel.MediaAttachment,
}
if p.recache {
// Existing attachment we're recaching, so only need to update.
err = p.mgr.state.DB.UpdateByID(ctx, p.media, p.media.ID)
// Existing attachment we're recaching, so only update.
err = p.mgr.state.DB.UpdateAttachment(ctx, p.media)
return err
}
// New attachment, first time caching.
err = p.mgr.state.DB.Put(ctx, p.media)
// First time caching this attachment, insert it.
err = p.mgr.state.DB.PutAttachment(ctx, p.media)
return err
})

4
internal/media/prune.go
View file

@ -320,7 +320,7 @@ func (m *manager) deleteAttachment(ctx context.Context, attachment *gtsmodel.Med
}
// Delete attachment completely.
return m.state.DB.DeleteByID(ctx, attachment.ID, attachment)
return m.state.DB.DeleteAttachment(ctx, attachment.ID)
}
func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
@ -332,7 +332,7 @@ func (m *manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.Me
attachment.UpdatedAt = time.Now()
cached := false
attachment.Cached = &cached
return m.state.DB.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached")
return m.state.DB.UpdateAttachment(ctx, attachment, "updated_at", "cached")
}
func (m *manager) removeFiles(ctx context.Context, keys ...string) (int, error) {

2
internal/processing/instance.go
View file

@ -233,7 +233,7 @@ func (p *Processor) InstancePatch(ctx context.Context, form *apimodel.InstanceSe
} else if form.AvatarDescription != nil && ia.AvatarMediaAttachment != nil {
// process just the description for the existing avatar
ia.AvatarMediaAttachment.Description = *form.AvatarDescription
if err := p.state.DB.UpdateByID(ctx, ia.AvatarMediaAttachment, ia.AvatarMediaAttachmentID, "description"); err != nil {
if err := p.state.DB.UpdateAttachment(ctx, ia.AvatarMediaAttachment, "description"); err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error updating instance avatar description: %s", err))
}
}

2
internal/processing/media/delete.go
View file

@ -40,7 +40,7 @@ func (p *Processor) Delete(ctx context.Context, mediaAttachmentID string) gtserr
}
// delete the attachment
if err := p.state.DB.DeleteByID(ctx, mediaAttachmentID, attachment); err != nil && !errors.Is(err, db.ErrNoEntries) {
if err := p.state.DB.DeleteAttachment(ctx, mediaAttachmentID); err != nil && !errors.Is(err, db.ErrNoEntries) {
errs = append(errs, fmt.Sprintf("remove attachment: %s", err))
}

2
internal/processing/media/unattach.go
View file

@ -49,7 +49,7 @@ func (p *Processor) Unattach(ctx context.Context, account *gtsmodel.Account, med
attachment.UpdatedAt = time.Now()
attachment.StatusID = ""
if err := p.state.DB.UpdateByID(ctx, attachment, attachment.ID, updatingColumns...); err != nil {
if err := p.state.DB.UpdateAttachment(ctx, attachment, updatingColumns...); err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("db error updating attachment: %s", err))
}