mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-12-07 20:58:06 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[performance] add caching of status fave, boost of, in reply to ID lists (#2060)

Browse source
This commit is contained in:
kim 2023-08-04 12:28:33 +01:00 • committed by GitHub
commit 9a291dea84
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 610 additions and 406 deletions
Download patch file Download diff file Expand all files Collapse all files

160
internal/db/bundb/status.go
View file

@ -20,7 +20,6 @@ package bundb
import (
"container/list"
"context"
"database/sql"
"errors"
"time"
@ -96,6 +95,26 @@ func (s *statusDB) GetStatusByURL(ctx context.Context, url string) (*gtsmodel.St
)
}
func (s *statusDB) GetStatusBoost(ctx context.Context, boostOfID string, byAccountID string) (*gtsmodel.Status, error) {
return s.getStatus(
ctx,
"BoostOfID.AccountID",
func(status *gtsmodel.Status) error {
return s.newStatusQ(status).
Where("status.boost_of_id = ?", boostOfID).
Where("status.account_id = ?", byAccountID).
// Our old code actually allowed a status to
// be boosted multiple times by the same author,
// so limit our query + order to fetch latest.
Order("status.id DESC"). // our IDs are timestamped
Limit(1).
Scan(ctx)
},
boostOfID, byAccountID,
)
}
func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*gtsmodel.Status) error, keyParts ...any) (*gtsmodel.Status, error) {
// Fetch status from database cache with loader callback
status, err := s.state.Caches.GTS.Status().Load(lookup, func() (*gtsmodel.Status, error) {
@ -245,11 +264,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
}
}
if err := errs.Combine(); err != nil {
return gtserror.Newf("%w", err)
}
return nil
return errs.Combine()
}
func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) error {
@ -506,25 +521,17 @@ func (s *statusDB) GetStatusChildren(ctx context.Context, status *gtsmodel.Statu
}
func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status, foundStatuses *list.List, onlyDirect bool, minID string) {
var childIDs []string
q := s.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
Column("status.id").
Where("? = ?", bun.Ident("status.in_reply_to_id"), status.ID)
if minID != "" {
q = q.Where("? > ?", bun.Ident("status.id"), minID)
}
if err := q.Scan(ctx, &childIDs); err != nil {
if err != sql.ErrNoRows {
log.Errorf(ctx, "error getting children for %q: %v", status.ID, err)
}
childIDs, err := s.getStatusReplyIDs(ctx, status.ID)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
log.Errorf(ctx, "error getting status %s children: %v", status.ID, err)
return
}
for _, id := range childIDs {
if id <= minID {
continue
}
// Fetch child with ID from database
child, err := s.GetStatusByID(ctx, id)
if err != nil {
@ -553,48 +560,80 @@ func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status,
}
}
func (s *statusDB) CountStatusReplies(ctx context.Context, status *gtsmodel.Status) (int, error) {
return s.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
Where("? = ?", bun.Ident("status.in_reply_to_id"), status.ID).
Count(ctx)
func (s *statusDB) GetStatusReplies(ctx context.Context, statusID string) ([]*gtsmodel.Status, error) {
statusIDs, err := s.getStatusReplyIDs(ctx, statusID)
if err != nil {
return nil, err
}
return s.GetStatusesByIDs(ctx, statusIDs)
}
func (s *statusDB) CountStatusReblogs(ctx context.Context, status *gtsmodel.Status) (int, error) {
return s.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
Where("? = ?", bun.Ident("status.boost_of_id"), status.ID).
Count(ctx)
func (s *statusDB) CountStatusReplies(ctx context.Context, statusID string) (int, error) {
statusIDs, err := s.getStatusReplyIDs(ctx, statusID)
return len(statusIDs), err
}
func (s *statusDB) CountStatusFaves(ctx context.Context, status *gtsmodel.Status) (int, error) {
return s.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("status_faves"), bun.Ident("status_fave")).
Where("? = ?", bun.Ident("status_fave.status_id"), status.ID).
Count(ctx)
func (s *statusDB) getStatusReplyIDs(ctx context.Context, statusID string) ([]string, error) {
return s.state.Caches.GTS.InReplyToIDs().Load(statusID, func() ([]string, error) {
var statusIDs []string
// Status reply IDs not in cache, perform DB query!
if err := s.db.
NewSelect().
Table("statuses").
Column("id").
Where("? = ?", bun.Ident("in_reply_to_id"), statusID).
Order("id DESC").
Scan(ctx, &statusIDs); err != nil {
return nil, s.db.ProcessError(err)
}
return statusIDs, nil
})
}
func (s *statusDB) IsStatusFavedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) {
q := s.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("status_faves"), bun.Ident("status_fave")).
Where("? = ?", bun.Ident("status_fave.status_id"), status.ID).
Where("? = ?", bun.Ident("status_fave.account_id"), accountID)
return s.db.Exists(ctx, q)
func (s *statusDB) GetStatusBoosts(ctx context.Context, statusID string) ([]*gtsmodel.Status, error) {
statusIDs, err := s.getStatusBoostIDs(ctx, statusID)
if err != nil {
return nil, err
}
return s.GetStatusesByIDs(ctx, statusIDs)
}
func (s *statusDB) IsStatusRebloggedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) {
q := s.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
Where("? = ?", bun.Ident("status.boost_of_id"), status.ID).
Where("? = ?", bun.Ident("status.account_id"), accountID)
func (s *statusDB) IsStatusBoostedBy(ctx context.Context, statusID string, accountID string) (bool, error) {
boost, err := s.GetStatusBoost(
gtscontext.SetBarebones(ctx),
statusID,
accountID,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return false, err
}
return (boost != nil), nil
}
return s.db.Exists(ctx, q)
func (s *statusDB) CountStatusBoosts(ctx context.Context, statusID string) (int, error) {
statusIDs, err := s.getStatusBoostIDs(ctx, statusID)
return len(statusIDs), err
}
func (s *statusDB) getStatusBoostIDs(ctx context.Context, statusID string) ([]string, error) {
return s.state.Caches.GTS.BoostOfIDs().Load(statusID, func() ([]string, error) {
var statusIDs []string
// Status boost IDs not in cache, perform DB query!
if err := s.db.
NewSelect().
Table("statuses").
Column("id").
Where("? = ?", bun.Ident("boost_of_id"), statusID).
Order("id DESC").
Scan(ctx, &statusIDs); err != nil {
return nil, s.db.ProcessError(err)
}
return statusIDs, nil
})
}
func (s *statusDB) IsStatusMutedBy(ctx context.Context, status *gtsmodel.Status, accountID string) (bool, error) {
@ -616,16 +655,3 @@ func (s *statusDB) IsStatusBookmarkedBy(ctx context.Context, status *gtsmodel.St
return s.db.Exists(ctx, q)
}
func (s *statusDB) GetStatusReblogs(ctx context.Context, status *gtsmodel.Status) ([]*gtsmodel.Status, error) {
reblogs := []*gtsmodel.Status{}
q := s.
newStatusQ(&reblogs).
Where("? = ?", bun.Ident("status.boost_of_id"), status.ID)
if err := q.Scan(ctx); err != nil {
return nil, s.db.ProcessError(err)
}
return reblogs, nil
}

216
internal/db/bundb/statusfave.go
View file

@ -19,6 +19,7 @@ package bundb
import (
"context"
"database/sql"
"errors"
"fmt"
@ -44,8 +45,14 @@ func (s *statusFaveDB) GetStatusFave(ctx context.Context, accountID string, stat
return s.db.
NewSelect().
Model(fave).
Where("? = ?", bun.Ident("account_id"), accountID).
Where("? = ?", bun.Ident("status_id"), statusID).
Where("status_fave.account_id = ?", accountID).
Where("status_fave.status_id = ?", statusID).
// Our old code actually allowed a status to
// be faved multiple times by the same author,
// so limit our query + order to fetch latest.
Order("status_fave.id DESC"). // our IDs are timestamped
Limit(1).
Scan(ctx)
},
accountID,
@ -89,63 +96,68 @@ func (s *statusFaveDB) getStatusFave(ctx context.Context, lookup string, dbQuery
return fave, nil
}
// Fetch the status fave author account.
fave.Account, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
fave.AccountID,
)
if err != nil {
return nil, fmt.Errorf("error getting status fave account %q: %w", fave.AccountID, err)
}
// Fetch the status fave target account.
fave.TargetAccount, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
fave.TargetAccountID,
)
if err != nil {
return nil, fmt.Errorf("error getting status fave target account %q: %w", fave.TargetAccountID, err)
}
// Fetch the status fave target status.
fave.Status, err = s.state.DB.GetStatusByID(
gtscontext.SetBarebones(ctx),
fave.StatusID,
)
if err != nil {
return nil, fmt.Errorf("error getting status fave status %q: %w", fave.StatusID, err)
// Populate the status favourite model.
if err := s.PopulateStatusFave(ctx, fave); err != nil {
return nil, fmt.Errorf("error(s) populating status fave: %w", err)
}
return fave, nil
}
func (s *statusFaveDB) GetStatusFavesForStatus(ctx context.Context, statusID string) ([]*gtsmodel.StatusFave, error) {
ids := []string{}
if err := s.db.
NewSelect().
Table("status_faves").
Column("id").
Where("? = ?", bun.Ident("status_id"), statusID).
Scan(ctx, &ids); err != nil {
return nil, s.db.ProcessError(err)
func (s *statusFaveDB) GetStatusFaves(ctx context.Context, statusID string) ([]*gtsmodel.StatusFave, error) {
// Fetch the status fave IDs for status.
faveIDs, err := s.getStatusFaveIDs(ctx, statusID)
if err != nil {
return nil, err
}
faves := make([]*gtsmodel.StatusFave, 0, len(ids))
// Preallocate a slice of expected status fave capacity.
faves := make([]*gtsmodel.StatusFave, 0, len(faveIDs))
for _, id := range ids {
for _, id := range faveIDs {
// Fetch status fave model for each ID.
fave, err := s.GetStatusFaveByID(ctx, id)
if err != nil {
log.Errorf(ctx, "error getting status fave %q: %v", id, err)
continue
}
faves = append(faves, fave)
}
return faves, nil
}
func (s *statusFaveDB) IsStatusFavedBy(ctx context.Context, statusID string, accountID string) (bool, error) {
fave, err := s.GetStatusFave(ctx, accountID, statusID)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return false, err
}
return (fave != nil), nil
}
func (s *statusFaveDB) CountStatusFaves(ctx context.Context, statusID string) (int, error) {
faveIDs, err := s.getStatusFaveIDs(ctx, statusID)
return len(faveIDs), err
}
func (s *statusFaveDB) getStatusFaveIDs(ctx context.Context, statusID string) ([]string, error) {
return s.state.Caches.GTS.StatusFaveIDs().Load(statusID, func() ([]string, error) {
var faveIDs []string
// Status fave IDs not in cache, perform DB query!
if err := s.db.
NewSelect().
Table("status_faves").
Column("id").
Where("? = ?", bun.Ident("status_id"), statusID).
Scan(ctx, &faveIDs); err != nil {
return nil, s.db.ProcessError(err)
}
return faveIDs, nil
})
}
func (s *statusFaveDB) PopulateStatusFave(ctx context.Context, statusFave *gtsmodel.StatusFave) error {
var (
err error
@ -203,26 +215,32 @@ func (s *statusFaveDB) PutStatusFave(ctx context.Context, fave *gtsmodel.StatusF
}
func (s *statusFaveDB) DeleteStatusFaveByID(ctx context.Context, id string) error {
defer s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
var statusID string
// Load fave into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := s.GetStatusFaveByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
// Perform DELETE on status fave,
// returning the status ID it was for.
if _, err := s.db.NewDelete().
Table("status_faves").
Where("id = ?", id).
Returning("status_id").
Exec(ctx, &statusID); err != nil {
if err == sql.ErrNoRows {
// Not an issue, only due
// to us doing a RETURNING.
err = nil
}
return err
return s.db.ProcessError(err)
}
// Finally delete fave from DB.
_, err = s.db.NewDelete().
Table("status_faves").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx)
return s.db.ProcessError(err)
if statusID != "" {
// Invalidate any cached status faves for this status.
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
// Invalidate any cached status fave IDs for this status.
s.state.Caches.GTS.StatusFaveIDs().Invalidate(statusID)
}
return nil
}
func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID string, originAccountID string) error {
@ -230,12 +248,13 @@ func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID st
return errors.New("DeleteStatusFaves: one of targetAccountID or originAccountID must be set")
}
var faveIDs []string
var statusIDs []string
q := s.db.
NewSelect().
Column("id").
Table("status_faves")
// Prepare DELETE query returning
// the deleted faves for status IDs.
q := s.db.NewDelete().
Table("status_faves").
Returning("status_id")
if targetAccountID != "" {
q = q.Where("? = ?", bun.Ident("target_account_id"), targetAccountID)
@ -245,69 +264,46 @@ func (s *statusFaveDB) DeleteStatusFaves(ctx context.Context, targetAccountID st
q = q.Where("? = ?", bun.Ident("account_id"), originAccountID)
}
if _, err := q.Exec(ctx, &faveIDs); err != nil {
// Execute query, store favourited status IDs.
if _, err := q.Exec(ctx, &statusIDs); err != nil {
if err == sql.ErrNoRows {
// Not an issue, only due
// to us doing a RETURNING.
err = nil
}
return s.db.ProcessError(err)
}
defer func() {
// Invalidate all IDs on return.
for _, id := range faveIDs {
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
}
}()
// Collate (deduplicating) status IDs.
statusIDs = collate(func(i int) string {
return statusIDs[i]
}, len(statusIDs))
// Load all faves into cache, this *really* isn't great
// but it is the only way we can ensure we invalidate all
// related caches correctly (e.g. visibility).
for _, id := range faveIDs {
_, err := s.GetStatusFaveByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
for _, id := range statusIDs {
// Invalidate any cached status faves for this status.
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
// Invalidate any cached status fave IDs for this status.
s.state.Caches.GTS.StatusFaveIDs().Invalidate(id)
}
// Finally delete all from DB.
_, err := s.db.NewDelete().
Table("status_faves").
Where("? IN (?)", bun.Ident("id"), bun.In(faveIDs)).
Exec(ctx)
return s.db.ProcessError(err)
return nil
}
func (s *statusFaveDB) DeleteStatusFavesForStatus(ctx context.Context, statusID string) error {
// Capture fave IDs in a RETURNING statement.
var faveIDs []string
q := s.db.
NewSelect().
Column("id").
// Delete all status faves for status.
if _, err := s.db.NewDelete().
Table("status_faves").
Where("? = ?", bun.Ident("status_id"), statusID)
if _, err := q.Exec(ctx, &faveIDs); err != nil {
Where("status_id = ?", statusID).
Exec(ctx); err != nil {
return s.db.ProcessError(err)
}
defer func() {
// Invalidate all IDs on return.
for _, id := range faveIDs {
s.state.Caches.GTS.StatusFave().Invalidate("ID", id)
}
}()
// Invalidate any cached status faves for this status.
s.state.Caches.GTS.StatusFave().Invalidate("ID", statusID)
// Load all faves into cache, this *really* isn't great
// but it is the only way we can ensure we invalidate all
// related caches correctly (e.g. visibility).
for _, id := range faveIDs {
_, err := s.GetStatusFaveByID(ctx, id)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return err
}
}
// Invalidate any cached status fave IDs for this status.
s.state.Caches.GTS.StatusFaveIDs().Invalidate(statusID)
// Finally delete all from DB.
_, err := s.db.NewDelete().
Table("status_faves").
Where("? IN (?)", bun.Ident("id"), bun.In(faveIDs)).
Exec(ctx)
return s.db.ProcessError(err)
return nil
}

4
internal/db/bundb/statusfave_test.go
View file

@ -35,7 +35,7 @@ type StatusFaveTestSuite struct {
func (suite *StatusFaveTestSuite) TestGetStatusFaves() {
testStatus := suite.testStatuses["admin_account_status_1"]
faves, err := suite.db.GetStatusFavesForStatus(context.Background(), testStatus.ID)
faves, err := suite.db.GetStatusFaves(context.Background(), testStatus.ID)
if err != nil {
suite.FailNow(err.Error())
}
@ -51,7 +51,7 @@ func (suite *StatusFaveTestSuite) TestGetStatusFaves() {
func (suite *StatusFaveTestSuite) TestGetStatusFavesNone() {
testStatus := suite.testStatuses["admin_account_status_4"]
faves, err := suite.db.GetStatusFavesForStatus(context.Background(), testStatus.ID)
faves, err := suite.db.GetStatusFaves(context.Background(), testStatus.ID)
if err != nil {
suite.FailNow(err.Error())
}