[performance] cache more database calls, reduce required database calls overall (#3290)

* improvements to caching for lists and relationship to accounts / follows

* fix nil panic in AddToList()

* ensure list related caches are correctly invalidated

* ensure returned ID lists are ordered correctly

* bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded)

* remove zero checks in uncached key load functions (go-structr now handles this)

* fix issues after rebase on upstream/main

* update the expected return order of CSV exports (since list entries are now down by entry creation date)

* rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests

* use returning statements on delete to get cache invalidation info

* fixes to recent database delete changes

* fix broken list entries delete sql

* remove unused db function

* update remainder of delete functions to behave in similar way, some other small tweaks

* fix delete user sql, allow returning on err no entries

* uncomment + fix list database tests

* update remaining list tests

* update envparsing test

* add comments to each specific key being invalidated

* add more cache invalidation explanatory comments

* whoops; actually delete poll votes from database in the DeletePollByID() func

* remove added but-commented-out field

* improved comment regarding paging being disabled

* make cache invalidation comments match what's actually happening

* fix up delete query comments to match what is happening

* rename function to read a bit better

* don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call)

* update function name in test

* move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic()

* use the poll_id column in poll_votes for selecting votes in poll ID

* fix function name
This commit is contained in:
kim 2024-09-16 16:46:09 +00:00 committed by GitHub
commit 84279f6a6a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
68 changed files with 1683 additions and 2135 deletions

View file

@ -70,7 +70,7 @@ func (t *timelineDB) GetHomeTimeline(ctx context.Context, accountID string, maxI
// To take account of exclusive lists, get all of
// this account's lists, so we can filter out follows
// that are in contained in exclusive lists.
lists, err := t.state.DB.GetListsForAccountID(ctx, accountID)
lists, err := t.state.DB.GetListsByAccountID(ctx, accountID)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.Newf("db error getting lists for account %s: %w", accountID, err)
}
@ -84,9 +84,15 @@ func (t *timelineDB) GetHomeTimeline(ctx context.Context, accountID string, maxI
continue
}
// Fetch all follow IDs of the entries ccontained in this list.
listFollowIDs, err := t.state.DB.GetFollowIDsInList(ctx, list.ID, nil)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.Newf("db error getting list entry follow ids: %w", err)
}
// Exclusive list, index all its follow IDs.
for _, listEntry := range list.ListEntries {
ignoreFollowIDs[listEntry.FollowID] = struct{}{}
for _, followID := range listFollowIDs {
ignoreFollowIDs[followID] = struct{}{}
}
}
@ -370,30 +376,20 @@ func (t *timelineDB) GetListTimeline(
frontToBack = true
)
// Fetch all listEntries entries from the database.
listEntries, err := t.state.DB.GetListEntries(
// Don't need actual follows
// for this, just the IDs.
gtscontext.SetBarebones(ctx),
listID,
"", "", "", 0,
// Fetch all follow IDs contained in list from DB.
followIDs, err := t.state.DB.GetFollowIDsInList(
ctx, listID, nil,
)
if err != nil {
return nil, fmt.Errorf("error getting entries for list %s: %w", listID, err)
return nil, fmt.Errorf("error getting follows in list: %w", err)
}
// If there's no list entries we can't
// If there's no list follows we can't
// possibly return anything for this list.
if len(listEntries) == 0 {
if len(followIDs) == 0 {
return make([]*gtsmodel.Status, 0), nil
}
// Extract just the IDs of each follow.
followIDs := make([]string, 0, len(listEntries))
for _, listEntry := range listEntries {
followIDs = append(followIDs, listEntry.FollowID)
}
// Select target account IDs from follows.
subQ := t.db.
NewSelect().