mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-12-10 04:38:08 -06:00
[performance] filter model and database table improvements (#4277)
- removes unnecessary fields / columns (created_at, updated_at)
- replaces filter.context_* columns with singular filter.contexts bit field which should save both struct memory and database space
- replaces filter.action string with integer enum type which should save both struct memory and database space
- adds links from filter to filter_* tables with Filter{}.KeywordIDs and Filter{}.StatusIDs fields (this also means we now have those ID slices cached, which reduces some lookups)
- removes account_id fields from filter_* tables, since there's a more direct connection between filter and filter_* tables, and filter.account_id already exists
- refactors a bunch of the filter processor logic to save on code repetition, factor in the above changes, fix a few bugs with missed error returns and bring it more in-line with some of our newer code
Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4277
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
tobi
parent
9d5af6c3dc
commit
996da6e029
82 changed files with 2440 additions and 1722 deletions
|
|
@ -35,6 +35,7 @@ import (
|
|||
"code.superseriousbusiness.org/gotosocial/internal/config"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/db"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/db/bundb/migrations"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/log"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/observability"
|
||||
|
|
@ -118,11 +119,17 @@ func doMigration(ctx context.Context, db *bun.DB) error {
|
|||
log.Infof(ctx, "MIGRATED DATABASE TO %s", group)
|
||||
|
||||
if db.Dialect().Name() == dialect.SQLite {
|
||||
log.Info(ctx,
|
||||
"running ANALYZE to update table and index statistics; this will take somewhere between "+
|
||||
"1-10 minutes, or maybe longer depending on your hardware and database size, please be patient",
|
||||
)
|
||||
_, err := db.ExecContext(ctx, "ANALYZE")
|
||||
// Perform a final WAL checkpoint after a migration on SQLite.
|
||||
if strings.EqualFold(config.GetDbSqliteJournalMode(), "WAL") {
|
||||
_, err := db.ExecContext(ctx, "PRAGMA wal_checkpoint(RESTART);")
|
||||
if err != nil {
|
||||
return gtserror.Newf("error performing wal_checkpoint: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Info(ctx, "running ANALYZE to update table and index statistics; this will take somewhere between "+
|
||||
"1-10 minutes, or maybe longer depending on your hardware and database size, please be patient")
|
||||
_, err := db.ExecContext(ctx, "ANALYZE;")
|
||||
if err != nil {
|
||||
log.Warnf(ctx, "ANALYZE failed, query planner may make poor life choices: %s", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"code.superseriousbusiness.org/gotosocial/internal/db"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtscontext"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
|
||||
|
|
@ -64,24 +64,14 @@ func (f *filterDB) GetFilterByID(ctx context.Context, id string) (*gtsmodel.Filt
|
|||
return filter, nil
|
||||
}
|
||||
|
||||
func (f *filterDB) GetFiltersForAccountID(ctx context.Context, accountID string) ([]*gtsmodel.Filter, error) {
|
||||
// Fetch IDs of all filters owned by this account.
|
||||
var filterIDs []string
|
||||
if err := f.db.
|
||||
NewSelect().
|
||||
Model((*gtsmodel.Filter)(nil)).
|
||||
Column("id").
|
||||
Where("? = ?", bun.Ident("account_id"), accountID).
|
||||
Scan(ctx, &filterIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(filterIDs) == 0 {
|
||||
func (f *filterDB) GetFiltersByIDs(ctx context.Context, ids []string) ([]*gtsmodel.Filter, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Get each filter by ID from the cache or DB.
|
||||
filters, err := f.state.Caches.DB.Filter.LoadIDs("ID",
|
||||
filterIDs,
|
||||
ids,
|
||||
func(uncached []string) ([]*gtsmodel.Filter, error) {
|
||||
filters := make([]*gtsmodel.Filter, 0, len(uncached))
|
||||
if err := f.db.
|
||||
|
|
@ -99,14 +89,15 @@ func (f *filterDB) GetFiltersForAccountID(ctx context.Context, accountID string)
|
|||
}
|
||||
|
||||
// Put the filter structs in the same order as the filter IDs.
|
||||
xslices.OrderBy(filters, filterIDs, func(filter *gtsmodel.Filter) string { return filter.ID })
|
||||
xslices.OrderBy(filters, ids, func(filter *gtsmodel.Filter) string { return filter.ID })
|
||||
|
||||
if gtscontext.Barebones(ctx) {
|
||||
return filters, nil
|
||||
}
|
||||
|
||||
var errs gtserror.MultiError
|
||||
|
||||
// Populate the filters. Remove any that we can't populate from the return slice.
|
||||
errs := gtserror.NewMultiError(len(filters))
|
||||
filters = slices.DeleteFunc(filters, func(filter *gtsmodel.Filter) bool {
|
||||
if err := f.populateFilter(ctx, filter); err != nil {
|
||||
errs.Appendf("error populating filter %s: %w", filter.ID, err)
|
||||
|
|
@ -118,235 +109,115 @@ func (f *filterDB) GetFiltersForAccountID(ctx context.Context, accountID string)
|
|||
return filters, errs.Combine()
|
||||
}
|
||||
|
||||
func (f *filterDB) GetFilterIDsByAccountID(ctx context.Context, accountID string) ([]string, error) {
|
||||
return f.state.Caches.DB.FilterIDs.Load(accountID, func() ([]string, error) {
|
||||
var filterIDs []string
|
||||
|
||||
if err := f.db.
|
||||
NewSelect().
|
||||
Model((*gtsmodel.Filter)(nil)).
|
||||
Column("id").
|
||||
Where("? = ?", bun.Ident("account_id"), accountID).
|
||||
Scan(ctx, &filterIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return filterIDs, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (f *filterDB) GetFiltersByAccountID(ctx context.Context, accountID string) ([]*gtsmodel.Filter, error) {
|
||||
filterIDs, err := f.GetFilterIDsByAccountID(ctx, accountID)
|
||||
if err != nil {
|
||||
return nil, gtserror.Newf("error getting filter ids: %w", err)
|
||||
}
|
||||
return f.GetFiltersByIDs(ctx, filterIDs)
|
||||
}
|
||||
|
||||
func (f *filterDB) populateFilter(ctx context.Context, filter *gtsmodel.Filter) error {
|
||||
var err error
|
||||
errs := gtserror.NewMultiError(2)
|
||||
var errs gtserror.MultiError
|
||||
|
||||
if filter.Keywords == nil {
|
||||
if !filter.KeywordsPopulated() {
|
||||
// Filter keywords are not set, fetch from the database.
|
||||
filter.Keywords, err = f.state.DB.GetFilterKeywordsForFilterID(
|
||||
gtscontext.SetBarebones(ctx),
|
||||
filter.ID,
|
||||
)
|
||||
filter.Keywords, err = f.GetFilterKeywordsByIDs(ctx, filter.KeywordIDs)
|
||||
if err != nil {
|
||||
errs.Appendf("error populating filter keywords: %w", err)
|
||||
}
|
||||
for i := range filter.Keywords {
|
||||
filter.Keywords[i].Filter = filter
|
||||
}
|
||||
}
|
||||
|
||||
if filter.Statuses == nil {
|
||||
if !filter.StatusesPopulated() {
|
||||
// Filter statuses are not set, fetch from the database.
|
||||
filter.Statuses, err = f.state.DB.GetFilterStatusesForFilterID(
|
||||
gtscontext.SetBarebones(ctx),
|
||||
filter.ID,
|
||||
)
|
||||
filter.Statuses, err = f.GetFilterStatusesByIDs(ctx, filter.StatusIDs)
|
||||
if err != nil {
|
||||
errs.Appendf("error populating filter statuses: %w", err)
|
||||
}
|
||||
for i := range filter.Statuses {
|
||||
filter.Statuses[i].Filter = filter
|
||||
}
|
||||
}
|
||||
|
||||
return errs.Combine()
|
||||
}
|
||||
|
||||
func (f *filterDB) PutFilter(ctx context.Context, filter *gtsmodel.Filter) error {
|
||||
// Pre-compile filter keyword regular expressions.
|
||||
for _, filterKeyword := range filter.Keywords {
|
||||
if err := filterKeyword.Compile(); err != nil {
|
||||
return gtserror.Newf("error compiling filter keyword regex: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update database.
|
||||
if err := f.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
if _, err := tx.
|
||||
NewInsert().
|
||||
Model(filter).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(filter.Keywords) > 0 {
|
||||
if _, err := tx.
|
||||
NewInsert().
|
||||
Model(&filter.Keywords).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(filter.Statuses) > 0 {
|
||||
if _, err := tx.
|
||||
NewInsert().
|
||||
Model(&filter.Statuses).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return f.state.Caches.DB.Filter.Store(filter, func() error {
|
||||
_, err := f.db.NewInsert().Model(filter).Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Update cache.
|
||||
f.state.Caches.DB.Filter.Put(filter)
|
||||
f.state.Caches.DB.FilterKeyword.Put(filter.Keywords...)
|
||||
f.state.Caches.DB.FilterStatus.Put(filter.Statuses...)
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (f *filterDB) UpdateFilter(
|
||||
ctx context.Context,
|
||||
filter *gtsmodel.Filter,
|
||||
filterColumns []string,
|
||||
filterKeywordColumns [][]string,
|
||||
deleteFilterKeywordIDs []string,
|
||||
deleteFilterStatusIDs []string,
|
||||
) error {
|
||||
if len(filter.Keywords) != len(filterKeywordColumns) {
|
||||
return errors.New("number of filter keywords must match number of lists of filter keyword columns")
|
||||
}
|
||||
|
||||
updatedAt := time.Now()
|
||||
filter.UpdatedAt = updatedAt
|
||||
for _, filterKeyword := range filter.Keywords {
|
||||
filterKeyword.UpdatedAt = updatedAt
|
||||
}
|
||||
for _, filterStatus := range filter.Statuses {
|
||||
filterStatus.UpdatedAt = updatedAt
|
||||
}
|
||||
|
||||
// If we're updating by column, ensure "updated_at" is included.
|
||||
if len(filterColumns) > 0 {
|
||||
filterColumns = append(filterColumns, "updated_at")
|
||||
}
|
||||
for i := range filterKeywordColumns {
|
||||
if len(filterKeywordColumns[i]) > 0 {
|
||||
filterKeywordColumns[i] = append(filterKeywordColumns[i], "updated_at")
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-compile filter keyword regular expressions.
|
||||
for _, filterKeyword := range filter.Keywords {
|
||||
if err := filterKeyword.Compile(); err != nil {
|
||||
return gtserror.Newf("error compiling filter keyword regex: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update database.
|
||||
if err := f.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
if _, err := tx.
|
||||
NewUpdate().
|
||||
func (f *filterDB) UpdateFilter(ctx context.Context, filter *gtsmodel.Filter, cols ...string) error {
|
||||
return f.state.Caches.DB.Filter.Store(filter, func() error {
|
||||
_, err := f.db.NewUpdate().
|
||||
Model(filter).
|
||||
Column(filterColumns...).
|
||||
Where("? = ?", bun.Ident("id"), filter.ID).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, filterKeyword := range filter.Keywords {
|
||||
if _, err := NewUpsert(tx).
|
||||
Model(filterKeyword).
|
||||
Constraint("id").
|
||||
Column(filterKeywordColumns[i]...).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(filter.Statuses) > 0 {
|
||||
if _, err := tx.
|
||||
NewInsert().
|
||||
Ignore().
|
||||
Model(&filter.Statuses).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(deleteFilterKeywordIDs) > 0 {
|
||||
if _, err := tx.
|
||||
NewDelete().
|
||||
Model((*gtsmodel.FilterKeyword)(nil)).
|
||||
Where("? = (?)", bun.Ident("id"), bun.In(deleteFilterKeywordIDs)).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(deleteFilterStatusIDs) > 0 {
|
||||
if _, err := tx.
|
||||
NewDelete().
|
||||
Model((*gtsmodel.FilterStatus)(nil)).
|
||||
Where("? = (?)", bun.Ident("id"), bun.In(deleteFilterStatusIDs)).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
Column(cols...).
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Update cache.
|
||||
f.state.Caches.DB.Filter.Put(filter)
|
||||
f.state.Caches.DB.FilterKeyword.Put(filter.Keywords...)
|
||||
f.state.Caches.DB.FilterStatus.Put(filter.Statuses...)
|
||||
// TODO: (Vyr) replace with cache multi-invalidate call
|
||||
for _, id := range deleteFilterKeywordIDs {
|
||||
f.state.Caches.DB.FilterKeyword.Invalidate("ID", id)
|
||||
}
|
||||
for _, id := range deleteFilterStatusIDs {
|
||||
f.state.Caches.DB.FilterStatus.Invalidate("ID", id)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (f *filterDB) DeleteFilterByID(ctx context.Context, id string) error {
|
||||
func (f *filterDB) DeleteFilter(ctx context.Context, filter *gtsmodel.Filter) error {
|
||||
if err := f.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
// Delete all keywords attached to filter.
|
||||
// Delete all keywords both known
|
||||
// by filter, and possible stragglers,
|
||||
// storing IDs in filter.KeywordIDs.
|
||||
if _, err := tx.
|
||||
NewDelete().
|
||||
Model((*gtsmodel.FilterKeyword)(nil)).
|
||||
Where("? = ?", bun.Ident("filter_id"), id).
|
||||
Exec(ctx); err != nil {
|
||||
Where("? = ?", bun.Ident("filter_id"), filter.ID).
|
||||
Returning("?", bun.Ident("id")).
|
||||
Exec(ctx, &filter.KeywordIDs); err != nil &&
|
||||
!errors.Is(err, db.ErrNoEntries) {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete all statuses attached to filter.
|
||||
// Delete all statuses both known
|
||||
// by filter, and possible stragglers.
|
||||
// storing IDs in filter.StatusIDs.
|
||||
if _, err := tx.
|
||||
NewDelete().
|
||||
Model((*gtsmodel.FilterStatus)(nil)).
|
||||
Where("? = ?", bun.Ident("filter_id"), id).
|
||||
Exec(ctx); err != nil {
|
||||
Where("? = ?", bun.Ident("filter_id"), filter.ID).
|
||||
Returning("?", bun.Ident("id")).
|
||||
Exec(ctx, &filter.StatusIDs); err != nil &&
|
||||
!errors.Is(err, db.ErrNoEntries) {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete the filter itself.
|
||||
// Delete filter itself.
|
||||
_, err := tx.
|
||||
NewDelete().
|
||||
Model((*gtsmodel.Filter)(nil)).
|
||||
Where("? = ?", bun.Ident("id"), id).
|
||||
Where("? = ?", bun.Ident("id"), filter.ID).
|
||||
Exec(ctx)
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Invalidate this filter.
|
||||
f.state.Caches.DB.Filter.Invalidate("ID", id)
|
||||
|
||||
// Invalidate all keywords and statuses for this filter.
|
||||
f.state.Caches.DB.FilterKeyword.Invalidate("FilterID", id)
|
||||
f.state.Caches.DB.FilterStatus.Invalidate("FilterID", id)
|
||||
// Invalidate the filter itself, and
|
||||
// call invalidate hook in-case not cached.
|
||||
f.state.Caches.DB.Filter.Invalidate("ID", filter.ID)
|
||||
f.state.Caches.OnInvalidateFilter(filter)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,26 +38,30 @@ func (suite *FilterTestSuite) TestFilterCRUD() {
|
|||
|
||||
// Create new example filter with attached keyword.
|
||||
filter := >smodel.Filter{
|
||||
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
|
||||
AccountID: "01HNEJXCPRTJVJY9MV0VVHGD47",
|
||||
Title: "foss jail",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
ContextHome: util.Ptr(true),
|
||||
ContextPublic: util.Ptr(true),
|
||||
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
|
||||
AccountID: "01HNEJXCPRTJVJY9MV0VVHGD47",
|
||||
Title: "foss jail",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome | gtsmodel.FilterContextPublic),
|
||||
}
|
||||
filterKeyword := >smodel.FilterKeyword{
|
||||
ID: "01HNEK4RW5QEAMG9Y4ET6ST0J4",
|
||||
AccountID: filter.AccountID,
|
||||
FilterID: filter.ID,
|
||||
Keyword: "GNU/Linux",
|
||||
ID: "01HNEK4RW5QEAMG9Y4ET6ST0J4",
|
||||
FilterID: filter.ID,
|
||||
Keyword: "GNU/Linux",
|
||||
}
|
||||
filter.Keywords = []*gtsmodel.FilterKeyword{filterKeyword}
|
||||
filter.KeywordIDs = []string{filterKeyword.ID}
|
||||
|
||||
// Create new cancellable test context.
|
||||
ctx := suite.T().Context()
|
||||
ctx, cncl := context.WithCancel(ctx)
|
||||
defer cncl()
|
||||
|
||||
// Insert the example filter keyword into db.
|
||||
if err := suite.db.PutFilterKeyword(ctx, filterKeyword); err != nil {
|
||||
t.Fatalf("error inserting filter keyword: %v", err)
|
||||
}
|
||||
|
||||
// Insert the example filter into db.
|
||||
if err := suite.db.PutFilter(ctx, filter); err != nil {
|
||||
t.Fatalf("error inserting filter: %v", err)
|
||||
|
|
@ -74,27 +78,18 @@ func (suite *FilterTestSuite) TestFilterCRUD() {
|
|||
suite.Equal(filter.AccountID, check.AccountID)
|
||||
suite.Equal(filter.Title, check.Title)
|
||||
suite.Equal(filter.Action, check.Action)
|
||||
suite.Equal(filter.ContextHome, check.ContextHome)
|
||||
suite.Equal(filter.ContextNotifications, check.ContextNotifications)
|
||||
suite.Equal(filter.ContextPublic, check.ContextPublic)
|
||||
suite.Equal(filter.ContextThread, check.ContextThread)
|
||||
suite.Equal(filter.ContextAccount, check.ContextAccount)
|
||||
suite.NotZero(check.CreatedAt)
|
||||
suite.NotZero(check.UpdatedAt)
|
||||
suite.Equal(filter.Contexts, check.Contexts)
|
||||
|
||||
suite.Equal(len(filter.Keywords), len(check.Keywords))
|
||||
suite.Equal(filter.Keywords[0].ID, check.Keywords[0].ID)
|
||||
suite.Equal(filter.Keywords[0].AccountID, check.Keywords[0].AccountID)
|
||||
suite.Equal(filter.Keywords[0].FilterID, check.Keywords[0].FilterID)
|
||||
suite.Equal(filter.Keywords[0].Keyword, check.Keywords[0].Keyword)
|
||||
suite.Equal(filter.Keywords[0].FilterID, check.Keywords[0].FilterID)
|
||||
suite.NotZero(check.Keywords[0].CreatedAt)
|
||||
suite.NotZero(check.Keywords[0].UpdatedAt)
|
||||
|
||||
suite.Equal(len(filter.Statuses), len(check.Statuses))
|
||||
|
||||
// Fetch all filters.
|
||||
all, err := suite.db.GetFiltersForAccountID(ctx, filter.AccountID)
|
||||
all, err := suite.db.GetFiltersByAccountID(ctx, filter.AccountID)
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching filters: %v", err)
|
||||
}
|
||||
|
|
@ -108,28 +103,39 @@ func (suite *FilterTestSuite) TestFilterCRUD() {
|
|||
|
||||
suite.Empty(all[0].Statuses)
|
||||
|
||||
// Update the filter context and add another keyword and a status.
|
||||
check.ContextNotifications = util.Ptr(true)
|
||||
|
||||
// Update the filter context and
|
||||
// add another keyword and a status.
|
||||
check.Contexts.SetNotifications()
|
||||
newKeyword := >smodel.FilterKeyword{
|
||||
ID: "01HNEMY810E5XKWDDMN5ZRE749",
|
||||
FilterID: filter.ID,
|
||||
AccountID: filter.AccountID,
|
||||
Keyword: "tux",
|
||||
ID: "01HNEMY810E5XKWDDMN5ZRE749",
|
||||
FilterID: filter.ID,
|
||||
Keyword: "tux",
|
||||
}
|
||||
check.Keywords = append(check.Keywords, newKeyword)
|
||||
|
||||
check.KeywordIDs = append(check.KeywordIDs, newKeyword.ID)
|
||||
newStatus := >smodel.FilterStatus{
|
||||
ID: "01HNEMYD5XE7C8HH8TNCZ76FN2",
|
||||
FilterID: filter.ID,
|
||||
AccountID: filter.AccountID,
|
||||
StatusID: "01HNEKZW34SQZ8PSDQ0Z10NZES",
|
||||
ID: "01HNEMYD5XE7C8HH8TNCZ76FN2",
|
||||
FilterID: filter.ID,
|
||||
StatusID: "01HNEKZW34SQZ8PSDQ0Z10NZES",
|
||||
}
|
||||
check.Statuses = append(check.Statuses, newStatus)
|
||||
check.StatusIDs = append(check.StatusIDs, newStatus.ID)
|
||||
|
||||
if err := suite.db.UpdateFilter(ctx, check, nil, [][]string{nil, nil}, nil, nil); err != nil {
|
||||
// Insert the new filter keyword.
|
||||
if err := suite.db.PutFilterKeyword(ctx, newKeyword); err != nil {
|
||||
t.Fatalf("error inserting filter keyword: %v", err)
|
||||
}
|
||||
|
||||
// Insert the new filter status.
|
||||
if err := suite.db.PutFilterStatus(ctx, newStatus); err != nil {
|
||||
t.Fatalf("error inserting filter status: %v", err)
|
||||
}
|
||||
|
||||
// Now update the filter with new keyword and status.
|
||||
if err := suite.db.UpdateFilter(ctx, check); err != nil {
|
||||
t.Fatalf("error updating filter: %v", err)
|
||||
}
|
||||
|
||||
// Now fetch newly updated filter.
|
||||
check, err = suite.db.GetFilterByID(ctx, filter.ID)
|
||||
if err != nil {
|
||||
|
|
@ -137,22 +143,11 @@ func (suite *FilterTestSuite) TestFilterCRUD() {
|
|||
}
|
||||
|
||||
// Ensure expected fields were modified on check filter.
|
||||
suite.True(check.UpdatedAt.After(filter.UpdatedAt))
|
||||
if suite.NotNil(check.ContextHome) {
|
||||
suite.True(*check.ContextHome)
|
||||
}
|
||||
if suite.NotNil(check.ContextNotifications) {
|
||||
suite.True(*check.ContextNotifications)
|
||||
}
|
||||
if suite.NotNil(check.ContextPublic) {
|
||||
suite.True(*check.ContextPublic)
|
||||
}
|
||||
if suite.NotNil(check.ContextThread) {
|
||||
suite.False(*check.ContextThread)
|
||||
}
|
||||
if suite.NotNil(check.ContextAccount) {
|
||||
suite.False(*check.ContextAccount)
|
||||
}
|
||||
suite.True(check.Contexts.Home())
|
||||
suite.True(check.Contexts.Notifications())
|
||||
suite.True(check.Contexts.Public())
|
||||
suite.False(check.Contexts.Thread())
|
||||
suite.False(check.Contexts.Account())
|
||||
|
||||
// Ensure keyword entries were added.
|
||||
suite.Len(check.Keywords, 2)
|
||||
|
|
@ -175,9 +170,19 @@ func (suite *FilterTestSuite) TestFilterCRUD() {
|
|||
check.Keywords = []*gtsmodel.FilterKeyword{filterKeyword}
|
||||
check.Statuses = nil
|
||||
|
||||
if err := suite.db.UpdateFilter(ctx, check, nil, [][]string{{"whole_word"}}, []string{newKeyword.ID}, nil); err != nil {
|
||||
// Update the original filter keyword.
|
||||
filterKeyword.WholeWord = util.Ptr(true)
|
||||
if err := suite.db.UpdateFilterKeyword(ctx, filterKeyword); err != nil {
|
||||
t.Fatalf("error updating filter keyword: %v", err)
|
||||
}
|
||||
|
||||
// Drop most recently added filter keyword from filter.
|
||||
check.Keywords = []*gtsmodel.FilterKeyword{filterKeyword}
|
||||
check.KeywordIDs = []string{filterKeyword.ID}
|
||||
if err := suite.db.UpdateFilter(ctx, check); err != nil {
|
||||
t.Fatalf("error updating filter: %v", err)
|
||||
}
|
||||
|
||||
check, err = suite.db.GetFilterByID(ctx, filter.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching updated filter: %v", err)
|
||||
|
|
@ -186,23 +191,14 @@ func (suite *FilterTestSuite) TestFilterCRUD() {
|
|||
// Ensure expected fields were not modified.
|
||||
suite.Equal(filter.Title, check.Title)
|
||||
suite.Equal(gtsmodel.FilterActionWarn, check.Action)
|
||||
if suite.NotNil(check.ContextHome) {
|
||||
suite.True(*check.ContextHome)
|
||||
}
|
||||
if suite.NotNil(check.ContextNotifications) {
|
||||
suite.True(*check.ContextNotifications)
|
||||
}
|
||||
if suite.NotNil(check.ContextPublic) {
|
||||
suite.True(*check.ContextPublic)
|
||||
}
|
||||
if suite.NotNil(check.ContextThread) {
|
||||
suite.False(*check.ContextThread)
|
||||
}
|
||||
if suite.NotNil(check.ContextAccount) {
|
||||
suite.False(*check.ContextAccount)
|
||||
}
|
||||
suite.True(check.Contexts.Home())
|
||||
suite.True(check.Contexts.Notifications())
|
||||
suite.True(check.Contexts.Public())
|
||||
suite.False(check.Contexts.Thread())
|
||||
suite.False(check.Contexts.Account())
|
||||
|
||||
// Ensure only changed field of keyword was modified, and other keyword was deleted.
|
||||
// Ensure only changed field of keyword was
|
||||
// modified, and other keyword was deleted.
|
||||
suite.Len(check.Keywords, 1)
|
||||
suite.Equal(filterKeyword.ID, check.Keywords[0].ID)
|
||||
suite.Equal("GNU/Linux", check.Keywords[0].Keyword)
|
||||
|
|
@ -214,29 +210,8 @@ func (suite *FilterTestSuite) TestFilterCRUD() {
|
|||
suite.Len(check.Statuses, 1)
|
||||
suite.Equal(newStatus.ID, check.Statuses[0].ID)
|
||||
|
||||
// Add another status entry for the same status ID. It should be ignored without problems.
|
||||
redundantStatus := >smodel.FilterStatus{
|
||||
ID: "01HQXJ5Y405XZSQ67C2BSQ6HJ0",
|
||||
FilterID: filter.ID,
|
||||
AccountID: filter.AccountID,
|
||||
StatusID: newStatus.StatusID,
|
||||
}
|
||||
check.Statuses = []*gtsmodel.FilterStatus{redundantStatus}
|
||||
if err := suite.db.UpdateFilter(ctx, check, nil, [][]string{nil}, nil, nil); err != nil {
|
||||
t.Fatalf("error updating filter: %v", err)
|
||||
}
|
||||
check, err = suite.db.GetFilterByID(ctx, filter.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching updated filter: %v", err)
|
||||
}
|
||||
|
||||
// Ensure status entry was not deleted, updated, or duplicated.
|
||||
suite.Len(check.Statuses, 1)
|
||||
suite.Equal(newStatus.ID, check.Statuses[0].ID)
|
||||
suite.Equal(newStatus.StatusID, check.Statuses[0].StatusID)
|
||||
|
||||
// Now delete the filter from the DB.
|
||||
if err := suite.db.DeleteFilterByID(ctx, filter.ID); err != nil {
|
||||
if err := suite.db.DeleteFilter(ctx, filter); err != nil {
|
||||
t.Fatalf("error deleting filter: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -256,11 +231,11 @@ func (suite *FilterTestSuite) TestFilterTitleOverlap() {
|
|||
|
||||
// Create an empty filter for account 1.
|
||||
account1filter1 := >smodel.Filter{
|
||||
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
|
||||
AccountID: account1,
|
||||
Title: "my filter",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
ContextHome: util.Ptr(true),
|
||||
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
|
||||
AccountID: account1,
|
||||
Title: "my filter",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome),
|
||||
}
|
||||
if err := suite.db.PutFilter(ctx, account1filter1); err != nil {
|
||||
suite.FailNow("", "error putting account1filter1: %s", err)
|
||||
|
|
@ -269,11 +244,11 @@ func (suite *FilterTestSuite) TestFilterTitleOverlap() {
|
|||
// Create a filter for account 2 with
|
||||
// the same title, should be no issue.
|
||||
account2filter1 := >smodel.Filter{
|
||||
ID: "01JAG5GPXG7H5Y4ZP78GV1F2ET",
|
||||
AccountID: account2,
|
||||
Title: "my filter",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
ContextHome: util.Ptr(true),
|
||||
ID: "01JAG5GPXG7H5Y4ZP78GV1F2ET",
|
||||
AccountID: account2,
|
||||
Title: "my filter",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome),
|
||||
}
|
||||
if err := suite.db.PutFilter(ctx, account2filter1); err != nil {
|
||||
suite.FailNow("", "error putting account2filter1: %s", err)
|
||||
|
|
@ -283,11 +258,11 @@ func (suite *FilterTestSuite) TestFilterTitleOverlap() {
|
|||
// account 1 with the same name as
|
||||
// an existing filter of theirs.
|
||||
account1filter2 := >smodel.Filter{
|
||||
ID: "01JAG5J8NYKQE2KYCD28Y4P05V",
|
||||
AccountID: account1,
|
||||
Title: "my filter",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
ContextHome: util.Ptr(true),
|
||||
ID: "01JAG5J8NYKQE2KYCD28Y4P05V",
|
||||
AccountID: account1,
|
||||
Title: "my filter",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome),
|
||||
}
|
||||
err := suite.db.PutFilter(ctx, account1filter2)
|
||||
if !errors.Is(err, db.ErrAlreadyExists) {
|
||||
|
|
|
|||
|
|
@ -20,9 +20,7 @@ package bundb
|
|||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtscontext"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/log"
|
||||
|
|
@ -31,7 +29,7 @@ import (
|
|||
)
|
||||
|
||||
func (f *filterDB) GetFilterKeywordByID(ctx context.Context, id string) (*gtsmodel.FilterKeyword, error) {
|
||||
filterKeyword, err := f.state.Caches.DB.FilterKeyword.LoadOne(
|
||||
return f.state.Caches.DB.FilterKeyword.LoadOne(
|
||||
"ID",
|
||||
func() (*gtsmodel.FilterKeyword, error) {
|
||||
var filterKeyword gtsmodel.FilterKeyword
|
||||
|
|
@ -54,64 +52,16 @@ func (f *filterDB) GetFilterKeywordByID(ctx context.Context, id string) (*gtsmod
|
|||
},
|
||||
id,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !gtscontext.Barebones(ctx) {
|
||||
err = f.populateFilterKeyword(ctx, filterKeyword)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return filterKeyword, nil
|
||||
}
|
||||
|
||||
func (f *filterDB) populateFilterKeyword(ctx context.Context, filterKeyword *gtsmodel.FilterKeyword) (err error) {
|
||||
if filterKeyword.Filter == nil {
|
||||
// Filter is not set, fetch from the cache or database.
|
||||
filterKeyword.Filter, err = f.state.DB.GetFilterByID(
|
||||
|
||||
// Don't populate the filter with all of its keywords
|
||||
// and statuses or we'll just end up back here.
|
||||
gtscontext.SetBarebones(ctx),
|
||||
filterKeyword.FilterID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *filterDB) GetFilterKeywordsForFilterID(ctx context.Context, filterID string) ([]*gtsmodel.FilterKeyword, error) {
|
||||
return f.getFilterKeywords(ctx, "filter_id", filterID)
|
||||
}
|
||||
|
||||
func (f *filterDB) GetFilterKeywordsForAccountID(ctx context.Context, accountID string) ([]*gtsmodel.FilterKeyword, error) {
|
||||
return f.getFilterKeywords(ctx, "account_id", accountID)
|
||||
}
|
||||
|
||||
func (f *filterDB) getFilterKeywords(ctx context.Context, idColumn string, id string) ([]*gtsmodel.FilterKeyword, error) {
|
||||
var filterKeywordIDs []string
|
||||
|
||||
if err := f.db.
|
||||
NewSelect().
|
||||
Model((*gtsmodel.FilterKeyword)(nil)).
|
||||
Column("id").
|
||||
Where("? = ?", bun.Ident(idColumn), id).
|
||||
Scan(ctx, &filterKeywordIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(filterKeywordIDs) == 0 {
|
||||
func (f *filterDB) GetFilterKeywordsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.FilterKeyword, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Get each filter keyword by ID from the cache or DB.
|
||||
filterKeywords, err := f.state.Caches.DB.FilterKeyword.LoadIDs("ID",
|
||||
filterKeywordIDs,
|
||||
ids,
|
||||
func(uncached []string) ([]*gtsmodel.FilterKeyword, error) {
|
||||
filterKeywords := make([]*gtsmodel.FilterKeyword, 0, len(uncached))
|
||||
|
||||
|
|
@ -140,23 +90,10 @@ func (f *filterDB) getFilterKeywords(ctx context.Context, idColumn string, id st
|
|||
}
|
||||
|
||||
// Put the filter keyword structs in the same order as the filter keyword IDs.
|
||||
xslices.OrderBy(filterKeywords, filterKeywordIDs, func(filterKeyword *gtsmodel.FilterKeyword) string {
|
||||
xslices.OrderBy(filterKeywords, ids, func(filterKeyword *gtsmodel.FilterKeyword) string {
|
||||
return filterKeyword.ID
|
||||
})
|
||||
|
||||
if gtscontext.Barebones(ctx) {
|
||||
return filterKeywords, nil
|
||||
}
|
||||
|
||||
// Populate the filter keywords. Remove any that we can't populate from the return slice.
|
||||
filterKeywords = slices.DeleteFunc(filterKeywords, func(filterKeyword *gtsmodel.FilterKeyword) bool {
|
||||
if err := f.populateFilterKeyword(ctx, filterKeyword); err != nil {
|
||||
log.Errorf(ctx, "error populating filter keyword: %v", err)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
return filterKeywords, nil
|
||||
}
|
||||
|
||||
|
|
@ -178,11 +115,7 @@ func (f *filterDB) PutFilterKeyword(ctx context.Context, filterKeyword *gtsmodel
|
|||
})
|
||||
}
|
||||
|
||||
func (f *filterDB) UpdateFilterKeyword(ctx context.Context, filterKeyword *gtsmodel.FilterKeyword, columns ...string) error {
|
||||
filterKeyword.UpdatedAt = time.Now()
|
||||
if len(columns) > 0 {
|
||||
columns = append(columns, "updated_at")
|
||||
}
|
||||
func (f *filterDB) UpdateFilterKeyword(ctx context.Context, filterKeyword *gtsmodel.FilterKeyword, cols ...string) error {
|
||||
if filterKeyword.Regexp == nil {
|
||||
// Ensure regexp is compiled
|
||||
// before attempted caching.
|
||||
|
|
@ -196,22 +129,20 @@ func (f *filterDB) UpdateFilterKeyword(ctx context.Context, filterKeyword *gtsmo
|
|||
NewUpdate().
|
||||
Model(filterKeyword).
|
||||
Where("? = ?", bun.Ident("id"), filterKeyword.ID).
|
||||
Column(columns...).
|
||||
Column(cols...).
|
||||
Exec(ctx)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (f *filterDB) DeleteFilterKeywordByID(ctx context.Context, id string) error {
|
||||
func (f *filterDB) DeleteFilterKeywordsByIDs(ctx context.Context, ids ...string) error {
|
||||
if _, err := f.db.
|
||||
NewDelete().
|
||||
Model((*gtsmodel.FilterKeyword)(nil)).
|
||||
Where("? = ?", bun.Ident("id"), id).
|
||||
Where("? IN (?)", bun.Ident("id"), bun.In(ids)).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.state.Caches.DB.FilterKeyword.Invalidate("ID", id)
|
||||
|
||||
f.state.Caches.DB.FilterKeyword.InvalidateIDs("ID", ids)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,12 +32,11 @@ func (suite *FilterTestSuite) TestFilterKeywordCRUD() {
|
|||
|
||||
// Create new filter.
|
||||
filter := >smodel.Filter{
|
||||
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
|
||||
AccountID: "01HNEJXCPRTJVJY9MV0VVHGD47",
|
||||
Title: "foss jail",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
ContextHome: util.Ptr(true),
|
||||
ContextPublic: util.Ptr(true),
|
||||
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
|
||||
AccountID: "01HNEJXCPRTJVJY9MV0VVHGD47",
|
||||
Title: "foss jail",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome | gtsmodel.FilterContextPublic),
|
||||
}
|
||||
|
||||
// Create new cancellable test context.
|
||||
|
|
@ -51,19 +50,11 @@ func (suite *FilterTestSuite) TestFilterKeywordCRUD() {
|
|||
t.Fatalf("error inserting filter: %v", err)
|
||||
}
|
||||
|
||||
// There should be no filter keywords yet.
|
||||
all, err := suite.db.GetFilterKeywordsForAccountID(ctx, filter.AccountID)
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching filter keywords: %v", err)
|
||||
}
|
||||
suite.Empty(all)
|
||||
|
||||
// Add a filter keyword to it.
|
||||
filterKeyword := >smodel.FilterKeyword{
|
||||
ID: "01HNEK4RW5QEAMG9Y4ET6ST0J4",
|
||||
AccountID: filter.AccountID,
|
||||
FilterID: filter.ID,
|
||||
Keyword: "GNU/Linux",
|
||||
ID: "01HNEK4RW5QEAMG9Y4ET6ST0J4",
|
||||
FilterID: filter.ID,
|
||||
Keyword: "GNU/Linux",
|
||||
}
|
||||
|
||||
// Insert the new filter keyword into the DB.
|
||||
|
|
@ -78,28 +69,17 @@ func (suite *FilterTestSuite) TestFilterKeywordCRUD() {
|
|||
t.Fatalf("error fetching filter keyword: %v", err)
|
||||
}
|
||||
suite.Equal(filterKeyword.ID, check.ID)
|
||||
suite.NotZero(check.CreatedAt)
|
||||
suite.NotZero(check.UpdatedAt)
|
||||
suite.Equal(filterKeyword.AccountID, check.AccountID)
|
||||
suite.Equal(filterKeyword.FilterID, check.FilterID)
|
||||
suite.Equal(filterKeyword.Keyword, check.Keyword)
|
||||
suite.Equal(filterKeyword.WholeWord, check.WholeWord)
|
||||
|
||||
// Loading filter keywords by account ID should find the one we inserted.
|
||||
all, err = suite.db.GetFilterKeywordsForAccountID(ctx, filter.AccountID)
|
||||
// Check that fetching multiple filter keywords by IDs works.
|
||||
checks, err := suite.db.GetFilterKeywordsByIDs(ctx, []string{filterKeyword.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching filter keywords: %v", err)
|
||||
}
|
||||
suite.Len(all, 1)
|
||||
suite.Equal(filterKeyword.ID, all[0].ID)
|
||||
|
||||
// Loading filter keywords by filter ID should also find the one we inserted.
|
||||
all, err = suite.db.GetFilterKeywordsForFilterID(ctx, filter.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching filter keywords: %v", err)
|
||||
}
|
||||
suite.Len(all, 1)
|
||||
suite.Equal(filterKeyword.ID, all[0].ID)
|
||||
suite.Len(checks, 1)
|
||||
suite.Equal(filterKeyword.ID, checks[0].ID)
|
||||
|
||||
// Modify the filter keyword.
|
||||
filterKeyword.WholeWord = util.Ptr(true)
|
||||
|
|
@ -114,15 +94,12 @@ func (suite *FilterTestSuite) TestFilterKeywordCRUD() {
|
|||
t.Fatalf("error fetching filter keyword: %v", err)
|
||||
}
|
||||
suite.Equal(filterKeyword.ID, check.ID)
|
||||
suite.NotZero(check.CreatedAt)
|
||||
suite.True(check.UpdatedAt.After(check.CreatedAt))
|
||||
suite.Equal(filterKeyword.AccountID, check.AccountID)
|
||||
suite.Equal(filterKeyword.FilterID, check.FilterID)
|
||||
suite.Equal(filterKeyword.Keyword, check.Keyword)
|
||||
suite.Equal(filterKeyword.WholeWord, check.WholeWord)
|
||||
|
||||
// Delete the filter keyword from the DB.
|
||||
err = suite.db.DeleteFilterKeywordByID(ctx, filter.ID)
|
||||
err = suite.db.DeleteFilterKeywordsByIDs(ctx, filter.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("error deleting filter keyword: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,86 +19,38 @@ package bundb
|
|||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtscontext"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/util/xslices"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
func (f *filterDB) GetFilterStatusByID(ctx context.Context, id string) (*gtsmodel.FilterStatus, error) {
|
||||
filterStatus, err := f.state.Caches.DB.FilterStatus.LoadOne(
|
||||
return f.state.Caches.DB.FilterStatus.LoadOne(
|
||||
"ID",
|
||||
func() (*gtsmodel.FilterStatus, error) {
|
||||
var filterStatus gtsmodel.FilterStatus
|
||||
err := f.db.
|
||||
if err := f.db.
|
||||
NewSelect().
|
||||
Model(&filterStatus).
|
||||
Where("? = ?", bun.Ident("id"), id).
|
||||
Scan(ctx)
|
||||
return &filterStatus, err
|
||||
Scan(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &filterStatus, nil
|
||||
},
|
||||
id,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !gtscontext.Barebones(ctx) {
|
||||
err = f.populateFilterStatus(ctx, filterStatus)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return filterStatus, nil
|
||||
}
|
||||
|
||||
func (f *filterDB) populateFilterStatus(ctx context.Context, filterStatus *gtsmodel.FilterStatus) error {
|
||||
if filterStatus.Filter == nil {
|
||||
// Filter is not set, fetch from the cache or database.
|
||||
filter, err := f.state.DB.GetFilterByID(
|
||||
// Don't populate the filter with all of its keywords and statuses or we'll just end up back here.
|
||||
gtscontext.SetBarebones(ctx),
|
||||
filterStatus.FilterID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filterStatus.Filter = filter
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *filterDB) GetFilterStatusesForFilterID(ctx context.Context, filterID string) ([]*gtsmodel.FilterStatus, error) {
|
||||
return f.getFilterStatuses(ctx, "filter_id", filterID)
|
||||
}
|
||||
|
||||
func (f *filterDB) GetFilterStatusesForAccountID(ctx context.Context, accountID string) ([]*gtsmodel.FilterStatus, error) {
|
||||
return f.getFilterStatuses(ctx, "account_id", accountID)
|
||||
}
|
||||
|
||||
func (f *filterDB) getFilterStatuses(ctx context.Context, idColumn string, id string) ([]*gtsmodel.FilterStatus, error) {
|
||||
var filterStatusIDs []string
|
||||
if err := f.db.
|
||||
NewSelect().
|
||||
Model((*gtsmodel.FilterStatus)(nil)).
|
||||
Column("id").
|
||||
Where("? = ?", bun.Ident(idColumn), id).
|
||||
Scan(ctx, &filterStatusIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(filterStatusIDs) == 0 {
|
||||
func (f *filterDB) GetFilterStatusesByIDs(ctx context.Context, ids []string) ([]*gtsmodel.FilterStatus, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Get each filter status by ID from the cache or DB.
|
||||
filterStatuses, err := f.state.Caches.DB.FilterStatus.LoadIDs("ID",
|
||||
filterStatusIDs,
|
||||
ids,
|
||||
func(uncached []string) ([]*gtsmodel.FilterStatus, error) {
|
||||
filterStatuses := make([]*gtsmodel.FilterStatus, 0, len(uncached))
|
||||
if err := f.db.
|
||||
|
|
@ -116,29 +68,11 @@ func (f *filterDB) getFilterStatuses(ctx context.Context, idColumn string, id st
|
|||
}
|
||||
|
||||
// Put the filter status structs in the same order as the filter status IDs.
|
||||
xslices.OrderBy(filterStatuses, filterStatusIDs, func(filterStatus *gtsmodel.FilterStatus) string {
|
||||
xslices.OrderBy(filterStatuses, ids, func(filterStatus *gtsmodel.FilterStatus) string {
|
||||
return filterStatus.ID
|
||||
})
|
||||
|
||||
if gtscontext.Barebones(ctx) {
|
||||
return filterStatuses, nil
|
||||
}
|
||||
|
||||
// Populate the filter statuses. Remove any that we can't populate from the return slice.
|
||||
errs := gtserror.NewMultiError(len(filterStatuses))
|
||||
filterStatuses = slices.DeleteFunc(filterStatuses, func(filterStatus *gtsmodel.FilterStatus) bool {
|
||||
if err := f.populateFilterStatus(ctx, filterStatus); err != nil {
|
||||
errs.Appendf(
|
||||
"error populating filter status %s: %w",
|
||||
filterStatus.ID,
|
||||
err,
|
||||
)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
return filterStatuses, errs.Combine()
|
||||
return filterStatuses, nil
|
||||
}
|
||||
|
||||
func (f *filterDB) PutFilterStatus(ctx context.Context, filterStatus *gtsmodel.FilterStatus) error {
|
||||
|
|
@ -152,11 +86,6 @@ func (f *filterDB) PutFilterStatus(ctx context.Context, filterStatus *gtsmodel.F
|
|||
}
|
||||
|
||||
func (f *filterDB) UpdateFilterStatus(ctx context.Context, filterStatus *gtsmodel.FilterStatus, columns ...string) error {
|
||||
filterStatus.UpdatedAt = time.Now()
|
||||
if len(columns) > 0 {
|
||||
columns = append(columns, "updated_at")
|
||||
}
|
||||
|
||||
return f.state.Caches.DB.FilterStatus.Store(filterStatus, func() error {
|
||||
_, err := f.db.
|
||||
NewUpdate().
|
||||
|
|
@ -168,16 +97,14 @@ func (f *filterDB) UpdateFilterStatus(ctx context.Context, filterStatus *gtsmode
|
|||
})
|
||||
}
|
||||
|
||||
func (f *filterDB) DeleteFilterStatusByID(ctx context.Context, id string) error {
|
||||
func (f *filterDB) DeleteFilterStatusesByIDs(ctx context.Context, ids ...string) error {
|
||||
if _, err := f.db.
|
||||
NewDelete().
|
||||
Model((*gtsmodel.FilterStatus)(nil)).
|
||||
Where("? = ?", bun.Ident("id"), id).
|
||||
Where("? IN (?)", bun.Ident("id"), bun.In(ids)).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.state.Caches.DB.FilterStatus.Invalidate("ID", id)
|
||||
|
||||
f.state.Caches.DB.FilterStatus.InvalidateIDs("ID", ids)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import (
|
|||
|
||||
"code.superseriousbusiness.org/gotosocial/internal/db"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/util"
|
||||
)
|
||||
|
||||
// TestFilterStatusCRD tests CRD (no U) and read-all operations on filter statuses.
|
||||
|
|
@ -32,12 +31,11 @@ func (suite *FilterTestSuite) TestFilterStatusCRD() {
|
|||
|
||||
// Create new filter.
|
||||
filter := >smodel.Filter{
|
||||
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
|
||||
AccountID: "01HNEJXCPRTJVJY9MV0VVHGD47",
|
||||
Title: "foss jail",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
ContextHome: util.Ptr(true),
|
||||
ContextPublic: util.Ptr(true),
|
||||
ID: "01HNEJNVZZVXJTRB3FX3K2B1YF",
|
||||
AccountID: "01HNEJXCPRTJVJY9MV0VVHGD47",
|
||||
Title: "foss jail",
|
||||
Action: gtsmodel.FilterActionWarn,
|
||||
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome | gtsmodel.FilterContextPublic),
|
||||
}
|
||||
|
||||
// Create new cancellable test context.
|
||||
|
|
@ -51,19 +49,11 @@ func (suite *FilterTestSuite) TestFilterStatusCRD() {
|
|||
t.Fatalf("error inserting filter: %v", err)
|
||||
}
|
||||
|
||||
// There should be no filter statuses yet.
|
||||
all, err := suite.db.GetFilterStatusesForAccountID(ctx, filter.AccountID)
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching filter statuses: %v", err)
|
||||
}
|
||||
suite.Empty(all)
|
||||
|
||||
// Add a filter status to it.
|
||||
filterStatus := >smodel.FilterStatus{
|
||||
ID: "01HNEK4RW5QEAMG9Y4ET6ST0J4",
|
||||
AccountID: filter.AccountID,
|
||||
FilterID: filter.ID,
|
||||
StatusID: "01HQXGMQ3QFXRT4GX9WNQ8KC0X",
|
||||
ID: "01HNEK4RW5QEAMG9Y4ET6ST0J4",
|
||||
FilterID: filter.ID,
|
||||
StatusID: "01HQXGMQ3QFXRT4GX9WNQ8KC0X",
|
||||
}
|
||||
|
||||
// Insert the new filter status into the DB.
|
||||
|
|
@ -78,30 +68,19 @@ func (suite *FilterTestSuite) TestFilterStatusCRD() {
|
|||
t.Fatalf("error fetching filter status: %v", err)
|
||||
}
|
||||
suite.Equal(filterStatus.ID, check.ID)
|
||||
suite.NotZero(check.CreatedAt)
|
||||
suite.NotZero(check.UpdatedAt)
|
||||
suite.Equal(filterStatus.AccountID, check.AccountID)
|
||||
suite.Equal(filterStatus.FilterID, check.FilterID)
|
||||
suite.Equal(filterStatus.StatusID, check.StatusID)
|
||||
|
||||
// Loading filter statuses by account ID should find the one we inserted.
|
||||
all, err = suite.db.GetFilterStatusesForAccountID(ctx, filter.AccountID)
|
||||
// Check that fetching multiple filter statuses by IDs works.
|
||||
checks, err := suite.db.GetFilterStatusesByIDs(ctx, []string{filterStatus.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching filter statuses: %v", err)
|
||||
}
|
||||
suite.Len(all, 1)
|
||||
suite.Equal(filterStatus.ID, all[0].ID)
|
||||
|
||||
// Loading filter statuses by filter ID should also find the one we inserted.
|
||||
all, err = suite.db.GetFilterStatusesForFilterID(ctx, filter.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching filter statuses: %v", err)
|
||||
}
|
||||
suite.Len(all, 1)
|
||||
suite.Equal(filterStatus.ID, all[0].ID)
|
||||
suite.Len(checks, 1)
|
||||
suite.Equal(filterStatus.ID, checks[0].ID)
|
||||
|
||||
// Delete the filter status from the DB.
|
||||
err = suite.db.DeleteFilterStatusByID(ctx, filter.ID)
|
||||
err = suite.db.DeleteFilterStatusesByIDs(ctx, filter.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("error deleting filter status: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ package migrations
|
|||
import (
|
||||
"context"
|
||||
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
|
||||
gtsmodel "code.superseriousbusiness.org/gotosocial/internal/db/bundb/migrations/20241018151036_filter_unique_fix"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/dialect"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package gtsmodel
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Filter stores a filter created by a local account.
|
||||
type Filter struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
|
||||
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
|
||||
ExpiresAt time.Time `bun:"type:timestamptz,nullzero"` // Time filter should expire. If null, should not expire.
|
||||
AccountID string `bun:"type:CHAR(26),notnull,nullzero,unique:filters_account_id_title_uniq"` // ID of the local account that created the filter.
|
||||
Title string `bun:",nullzero,notnull,unique:filters_account_id_title_uniq"` // The name of the filter.
|
||||
Action FilterAction `bun:",nullzero,notnull"` // The action to take.
|
||||
Keywords []*FilterKeyword `bun:"-"` // Keywords for this filter.
|
||||
Statuses []*FilterStatus `bun:"-"` // Statuses for this filter.
|
||||
ContextHome *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
|
||||
ContextNotifications *bool `bun:",nullzero,notnull,default:false"` // Apply filter to notifications.
|
||||
ContextPublic *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
|
||||
ContextThread *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing a status's associated thread.
|
||||
ContextAccount *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing an account profile.
|
||||
}
|
||||
|
||||
// FilterKeyword stores a single keyword to filter statuses against.
|
||||
type FilterKeyword struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
|
||||
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
|
||||
AccountID string `bun:"type:CHAR(26),notnull,nullzero"` // ID of the local account that created the filter keyword.
|
||||
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_keywords_filter_id_keyword_uniq"` // ID of the filter that this keyword belongs to.
|
||||
Filter *Filter `bun:"-"` // Filter corresponding to FilterID
|
||||
Keyword string `bun:",nullzero,notnull,unique:filter_keywords_filter_id_keyword_uniq"` // The keyword or phrase to filter against.
|
||||
WholeWord *bool `bun:",nullzero,notnull,default:false"` // Should the filter consider word boundaries?
|
||||
Regexp *regexp.Regexp `bun:"-"` // pre-prepared regular expression
|
||||
}
|
||||
|
||||
// FilterStatus stores a single status to filter.
|
||||
type FilterStatus struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
|
||||
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
|
||||
AccountID string `bun:"type:CHAR(26),notnull,nullzero"` // ID of the local account that created the filter keyword.
|
||||
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the filter that this keyword belongs to.
|
||||
Filter *Filter `bun:"-"` // Filter corresponding to FilterID
|
||||
StatusID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the status to filter.
|
||||
}
|
||||
|
||||
// FilterAction represents the action to take on a filtered status.
|
||||
type FilterAction string
|
||||
|
||||
const (
|
||||
// FilterActionNone filters should not exist, except internally, for partially constructed or invalid filters.
|
||||
FilterActionNone FilterAction = ""
|
||||
// FilterActionWarn means that the status should be shown behind a warning.
|
||||
FilterActionWarn FilterAction = "warn"
|
||||
// FilterActionHide means that the status should be removed from timeline results entirely.
|
||||
FilterActionHide FilterAction = "hide"
|
||||
)
|
||||
|
|
@ -0,0 +1,303 @@
|
|||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
oldmodel "code.superseriousbusiness.org/gotosocial/internal/db/bundb/migrations/20241018151036_filter_unique_fix"
|
||||
newmodel "code.superseriousbusiness.org/gotosocial/internal/db/bundb/migrations/20250617122055_filter_improvements"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
func init() {
|
||||
up := func(ctx context.Context, db *bun.DB) error {
|
||||
// Replace 'context_*' and 'action' columns with space-saving enum / bitfields.
|
||||
if err := db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
newFilterType := reflect.TypeOf((*newmodel.Filter)(nil))
|
||||
|
||||
// Generate bun definition for new filter table contexts column.
|
||||
newColDef, err := getBunColumnDef(tx, newFilterType, "Contexts")
|
||||
if err != nil {
|
||||
return gtserror.Newf("error getting bun column def: %w", err)
|
||||
}
|
||||
|
||||
// Add new column type to table.
|
||||
if _, err := tx.NewAddColumn().
|
||||
Model((*oldmodel.Filter)(nil)).
|
||||
ColumnExpr(newColDef).
|
||||
Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error adding filter.contexts column: %w", err)
|
||||
}
|
||||
|
||||
// Generate bun definition for new filter table action column.
|
||||
newColDef, err = getBunColumnDef(tx, newFilterType, "Action")
|
||||
if err != nil {
|
||||
return gtserror.Newf("error getting bun column def: %w", err)
|
||||
}
|
||||
|
||||
// For now, name it as '_new'.
|
||||
newColDef = strings.ReplaceAll(
|
||||
newColDef,
|
||||
"action",
|
||||
"action_new",
|
||||
)
|
||||
|
||||
// Add new column type to table.
|
||||
if _, err := tx.NewAddColumn().
|
||||
Model((*oldmodel.Filter)(nil)).
|
||||
ColumnExpr(newColDef).
|
||||
Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error adding filter.contexts column: %w", err)
|
||||
}
|
||||
|
||||
var oldFilters []*oldmodel.Filter
|
||||
|
||||
// Select all filters.
|
||||
if err := tx.NewSelect().
|
||||
Model(&oldFilters).
|
||||
Column("id",
|
||||
"context_home",
|
||||
"context_notifications",
|
||||
"context_public",
|
||||
"context_thread",
|
||||
"context_account",
|
||||
"action").
|
||||
Scan(ctx); err != nil {
|
||||
return gtserror.Newf("error selecting filters: %w", err)
|
||||
}
|
||||
|
||||
for _, oldFilter := range oldFilters {
|
||||
var newContexts newmodel.FilterContexts
|
||||
var newAction newmodel.FilterAction
|
||||
|
||||
// Convert old contexts
|
||||
// to new contexts type.
|
||||
if *oldFilter.ContextHome {
|
||||
newContexts.SetHome()
|
||||
}
|
||||
if *oldFilter.ContextNotifications {
|
||||
newContexts.SetNotifications()
|
||||
}
|
||||
if *oldFilter.ContextPublic {
|
||||
newContexts.SetPublic()
|
||||
}
|
||||
if *oldFilter.ContextThread {
|
||||
newContexts.SetThread()
|
||||
}
|
||||
if *oldFilter.ContextAccount {
|
||||
newContexts.SetAccount()
|
||||
}
|
||||
|
||||
// Convert old action
|
||||
// to new action type.
|
||||
switch oldFilter.Action {
|
||||
case oldmodel.FilterActionHide:
|
||||
newAction = newmodel.FilterActionHide
|
||||
case oldmodel.FilterActionWarn:
|
||||
newAction = newmodel.FilterActionWarn
|
||||
default:
|
||||
return gtserror.Newf("invalid filter action %q for %s", oldFilter.Action, oldFilter.ID)
|
||||
}
|
||||
|
||||
// Update filter row with
|
||||
// the new contexts value.
|
||||
if _, err := tx.NewUpdate().
|
||||
Model((*oldmodel.Filter)(nil)).
|
||||
Where("? = ?", bun.Ident("id"), oldFilter.ID).
|
||||
Set("? = ?", bun.Ident("contexts"), newContexts).
|
||||
Set("? = ?", bun.Ident("action_new"), newAction).
|
||||
Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error updating filter.contexts: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Drop the old updated columns.
|
||||
for _, col := range []string{
|
||||
"context_home",
|
||||
"context_notifications",
|
||||
"context_public",
|
||||
"context_thread",
|
||||
"context_account",
|
||||
"action",
|
||||
} {
|
||||
if _, err := tx.NewDropColumn().
|
||||
Model((*oldmodel.Filter)(nil)).
|
||||
Column(col).
|
||||
Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error dropping filter.%s column: %w", col, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Rename the new action
|
||||
// column to correct name.
|
||||
if _, err := tx.NewRaw(
|
||||
"ALTER TABLE ? RENAME COLUMN ? TO ?",
|
||||
bun.Ident("filters"),
|
||||
bun.Ident("action_new"),
|
||||
bun.Ident("action"),
|
||||
).Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error renaming new action column: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// SQLITE: force WAL checkpoint to merge writes.
|
||||
if err := doWALCheckpoint(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Drop a bunch of (now, and more generally) unused columns from filter tables.
|
||||
if err := db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
for model, indices := range map[any][]string{
|
||||
(*oldmodel.FilterKeyword)(nil): {"filter_keywords_account_id_idx"},
|
||||
(*oldmodel.FilterStatus)(nil): {"filter_statuses_account_id_idx"},
|
||||
} {
|
||||
for _, index := range indices {
|
||||
if _, err := tx.NewDropIndex().
|
||||
Model(model).
|
||||
Index(index).
|
||||
Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error dropping %s index: %w", index, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for model, cols := range map[any][]string{
|
||||
(*oldmodel.Filter)(nil): {"created_at", "updated_at"},
|
||||
(*oldmodel.FilterKeyword)(nil): {"created_at", "updated_at", "account_id"},
|
||||
(*oldmodel.FilterStatus)(nil): {"created_at", "updated_at", "account_id"},
|
||||
} {
|
||||
for _, col := range cols {
|
||||
if _, err := tx.NewDropColumn().
|
||||
Model(model).
|
||||
Column(col).
|
||||
Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error dropping %T.%s column: %w", model, col, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// SQLITE: force WAL checkpoint to merge writes.
|
||||
if err := doWALCheckpoint(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create links from 'filters' table to 'filter_{keywords,statuses}' tables.
|
||||
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
newFilterType := reflect.TypeOf((*newmodel.Filter)(nil))
|
||||
|
||||
var filterIDs string
|
||||
|
||||
// Select all filter IDs.
|
||||
if err := tx.NewSelect().
|
||||
Model((*newmodel.Filter)(nil)).
|
||||
Column("id").
|
||||
Scan(ctx, &filterIDs); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return gtserror.Newf("error selecting filter ids: %w", err)
|
||||
}
|
||||
|
||||
for _, data := range []struct {
|
||||
Field string
|
||||
Model any
|
||||
}{
|
||||
{
|
||||
Field: "KeywordIDs",
|
||||
Model: (*newmodel.FilterKeyword)(nil),
|
||||
},
|
||||
{
|
||||
Field: "StatusIDs",
|
||||
Model: (*newmodel.FilterStatus)(nil),
|
||||
},
|
||||
} {
|
||||
// Generate bun definition for new filter table field column.
|
||||
newColDef, err := getBunColumnDef(tx, newFilterType, data.Field)
|
||||
if err != nil {
|
||||
return gtserror.Newf("error getting bun column def: %w", err)
|
||||
}
|
||||
|
||||
// Add new column type to table.
|
||||
if _, err := tx.NewAddColumn().
|
||||
Model((*oldmodel.Filter)(nil)).
|
||||
ColumnExpr(newColDef).
|
||||
Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error adding filter.%s column: %w", data.Field, err)
|
||||
}
|
||||
|
||||
// Get the SQL field information from bun for Filter{}.$Field.
|
||||
field, _, err := getModelField(tx, newFilterType, data.Field)
|
||||
if err != nil {
|
||||
return gtserror.Newf("error getting bun model field: %w", err)
|
||||
}
|
||||
|
||||
// Extract column name.
|
||||
col := field.SQLName
|
||||
|
||||
var relatedIDs []string
|
||||
for _, filterID := range filterIDs {
|
||||
// Reset related IDs.
|
||||
clear(relatedIDs)
|
||||
relatedIDs = relatedIDs[:0]
|
||||
|
||||
// Select $Model IDs that
|
||||
// are attached to filterID.
|
||||
if err := tx.NewSelect().
|
||||
Model(data.Model).
|
||||
Column("id").
|
||||
Where("? = ?", bun.Ident("filter_id"), filterID).
|
||||
Scan(ctx, &relatedIDs); err != nil {
|
||||
return gtserror.Newf("error selecting %T ids: %w", data.Model, err)
|
||||
}
|
||||
|
||||
// Now update the relevant filter
|
||||
// row to contain these related IDs.
|
||||
if _, err := tx.NewUpdate().
|
||||
Model((*newmodel.Filter)(nil)).
|
||||
Where("? = ?", bun.Ident("id"), filterID).
|
||||
Set("? = ?", bun.Ident(col), relatedIDs).
|
||||
Exec(ctx); err != nil {
|
||||
return gtserror.Newf("error updating filters.%s ids: %w", col, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
down := func(ctx context.Context, db *bun.DB) error {
|
||||
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := Migrations.Register(up, down); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,243 @@
|
|||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package gtsmodel
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"code.superseriousbusiness.org/gotosocial/internal/util"
|
||||
)
|
||||
|
||||
// smallint is the largest size supported
|
||||
// by a PostgreSQL SMALLINT, since an SQLite
|
||||
// SMALLINT is actually variable in size.
|
||||
type smallint int16
|
||||
|
||||
// enumType is the type we (at least, should) use
|
||||
// for database enum types, as smallest int size.
|
||||
type enumType smallint
|
||||
|
||||
// bitFieldType is the type we use
|
||||
// for database int bit fields, at
|
||||
// least where the smallest int size
|
||||
// will suffice for number of fields.
|
||||
type bitFieldType smallint
|
||||
|
||||
// FilterContext represents the
|
||||
// context in which a Filter applies.
|
||||
//
|
||||
// These are used as bit-field masks to determine
|
||||
// which are enabled in a FilterContexts bit field,
|
||||
// as well as to signify internally any particular
|
||||
// context in which a status should be filtered in.
|
||||
type FilterContext bitFieldType
|
||||
|
||||
const (
|
||||
// FilterContextNone means no filters should
|
||||
// be applied, this is for internal use only.
|
||||
FilterContextNone FilterContext = 0
|
||||
|
||||
// FilterContextHome means this status is being
|
||||
// filtered as part of a home or list timeline.
|
||||
FilterContextHome FilterContext = 1 << 1
|
||||
|
||||
// FilterContextNotifications means this status is
|
||||
// being filtered as part of the notifications timeline.
|
||||
FilterContextNotifications FilterContext = 1 << 2
|
||||
|
||||
// FilterContextPublic means this status is
|
||||
// being filtered as part of a public or tag timeline.
|
||||
FilterContextPublic FilterContext = 1 << 3
|
||||
|
||||
// FilterContextThread means this status is
|
||||
// being filtered as part of a thread's context.
|
||||
FilterContextThread FilterContext = 1 << 4
|
||||
|
||||
// FilterContextAccount means this status is
|
||||
// being filtered as part of an account's statuses.
|
||||
FilterContextAccount FilterContext = 1 << 5
|
||||
)
|
||||
|
||||
// FilterContexts stores multiple contexts
|
||||
// in which a Filter applies as bits in an int.
|
||||
type FilterContexts bitFieldType
|
||||
|
||||
// Applies returns whether receiving FilterContexts applies in FilterContexts.
|
||||
func (ctxs FilterContexts) Applies(ctx FilterContext) bool {
|
||||
switch ctx {
|
||||
case FilterContextHome:
|
||||
return ctxs.Home()
|
||||
case FilterContextNotifications:
|
||||
return ctxs.Notifications()
|
||||
case FilterContextPublic:
|
||||
return ctxs.Public()
|
||||
case FilterContextThread:
|
||||
return ctxs.Thread()
|
||||
case FilterContextAccount:
|
||||
return ctxs.Account()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Home returns whether FilterContextHome is set.
|
||||
func (ctxs FilterContexts) Home() bool {
|
||||
return ctxs&FilterContexts(FilterContextHome) != 0
|
||||
}
|
||||
|
||||
// SetHome will set the FilterContextHome bit.
|
||||
func (ctxs *FilterContexts) SetHome() {
|
||||
*ctxs |= FilterContexts(FilterContextHome)
|
||||
}
|
||||
|
||||
// UnsetHome will unset the FilterContextHome bit.
|
||||
func (ctxs *FilterContexts) UnsetHome() {
|
||||
*ctxs &= ^FilterContexts(FilterContextHome)
|
||||
}
|
||||
|
||||
// Notifications returns whether FilterContextNotifications is set.
|
||||
func (ctxs FilterContexts) Notifications() bool {
|
||||
return ctxs&FilterContexts(FilterContextNotifications) != 0
|
||||
}
|
||||
|
||||
// SetNotifications will set the FilterContextNotifications bit.
|
||||
func (ctxs *FilterContexts) SetNotifications() {
|
||||
*ctxs |= FilterContexts(FilterContextNotifications)
|
||||
}
|
||||
|
||||
// UnsetNotifications will unset the FilterContextNotifications bit.
|
||||
func (ctxs *FilterContexts) UnsetNotifications() {
|
||||
*ctxs &= ^FilterContexts(FilterContextNotifications)
|
||||
}
|
||||
|
||||
// Public returns whether FilterContextPublic is set.
|
||||
func (ctxs FilterContexts) Public() bool {
|
||||
return ctxs&FilterContexts(FilterContextPublic) != 0
|
||||
}
|
||||
|
||||
// SetPublic will set the FilterContextPublic bit.
|
||||
func (ctxs *FilterContexts) SetPublic() {
|
||||
*ctxs |= FilterContexts(FilterContextPublic)
|
||||
}
|
||||
|
||||
// UnsetPublic will unset the FilterContextPublic bit.
|
||||
func (ctxs *FilterContexts) UnsetPublic() {
|
||||
*ctxs &= ^FilterContexts(FilterContextPublic)
|
||||
}
|
||||
|
||||
// Thread returns whether FilterContextThread is set.
|
||||
func (ctxs FilterContexts) Thread() bool {
|
||||
return ctxs&FilterContexts(FilterContextThread) != 0
|
||||
}
|
||||
|
||||
// SetThread will set the FilterContextThread bit.
|
||||
func (ctxs *FilterContexts) SetThread() {
|
||||
*ctxs |= FilterContexts(FilterContextThread)
|
||||
}
|
||||
|
||||
// UnsetThread will unset the FilterContextThread bit.
|
||||
func (ctxs *FilterContexts) UnsetThread() {
|
||||
*ctxs &= ^FilterContexts(FilterContextThread)
|
||||
}
|
||||
|
||||
// Account returns whether FilterContextAccount is set.
|
||||
func (ctxs FilterContexts) Account() bool {
|
||||
return ctxs&FilterContexts(FilterContextAccount) != 0
|
||||
}
|
||||
|
||||
// SetAccount will set / unset the FilterContextAccount bit.
|
||||
func (ctxs *FilterContexts) SetAccount() {
|
||||
*ctxs |= FilterContexts(FilterContextAccount)
|
||||
}
|
||||
|
||||
// UnsetAccount will unset the FilterContextAccount bit.
|
||||
func (ctxs *FilterContexts) UnsetAccount() {
|
||||
*ctxs &= ^FilterContexts(FilterContextAccount)
|
||||
}
|
||||
|
||||
// FilterAction represents the action
|
||||
// to take on a filtered status.
|
||||
type FilterAction enumType
|
||||
|
||||
const (
|
||||
// FilterActionNone filters should not exist, except
|
||||
// internally, for partially constructed or invalid filters.
|
||||
FilterActionNone FilterAction = 0
|
||||
|
||||
// FilterActionWarn means that the
|
||||
// status should be shown behind a warning.
|
||||
FilterActionWarn FilterAction = 1
|
||||
|
||||
// FilterActionHide means that the status should
|
||||
// be removed from timeline results entirely.
|
||||
FilterActionHide FilterAction = 2
|
||||
)
|
||||
|
||||
// Filter stores a filter created by a local account.
|
||||
type Filter struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
ExpiresAt time.Time `bun:"type:timestamptz,nullzero"` // Time filter should expire. If null, should not expire.
|
||||
AccountID string `bun:"type:CHAR(26),notnull,nullzero,unique:filters_account_id_title_uniq"` // ID of the local account that created the filter.
|
||||
Title string `bun:",nullzero,notnull,unique:filters_account_id_title_uniq"` // The name of the filter.
|
||||
Action FilterAction `bun:",nullzero,notnull,default:0"` // The action to take.
|
||||
Keywords []*FilterKeyword `bun:"-"` // Keywords for this filter.
|
||||
KeywordIDs []string `bun:"keywords,array"` //
|
||||
Statuses []*FilterStatus `bun:"-"` // Statuses for this filter.
|
||||
StatusIDs []string `bun:"statuses,array"` //
|
||||
Contexts FilterContexts `bun:",nullzero,notnull,default:0"` // Which contexts does this filter apply in?
|
||||
}
|
||||
|
||||
// FilterKeyword stores a single keyword to filter statuses against.
|
||||
type FilterKeyword struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_keywords_filter_id_keyword_uniq"` // ID of the filter that this keyword belongs to.
|
||||
Keyword string `bun:",nullzero,notnull,unique:filter_keywords_filter_id_keyword_uniq"` // The keyword or phrase to filter against.
|
||||
WholeWord *bool `bun:",nullzero,notnull,default:false"` // Should the filter consider word boundaries?
|
||||
Regexp *regexp.Regexp `bun:"-"` // pre-prepared regular expression
|
||||
}
|
||||
|
||||
// Compile will compile this FilterKeyword as a prepared regular expression.
|
||||
func (k *FilterKeyword) Compile() (err error) {
|
||||
var (
|
||||
wordBreakStart string
|
||||
wordBreakEnd string
|
||||
)
|
||||
|
||||
if util.PtrOrZero(k.WholeWord) {
|
||||
// Either word boundary or
|
||||
// whitespace or start of line.
|
||||
wordBreakStart = `(?:\b|\s|^)`
|
||||
|
||||
// Either word boundary or
|
||||
// whitespace or end of line.
|
||||
wordBreakEnd = `(?:\b|\s|$)`
|
||||
}
|
||||
|
||||
// Compile keyword filter regexp.
|
||||
quoted := regexp.QuoteMeta(k.Keyword)
|
||||
k.Regexp, err = regexp.Compile(`(?i)` + wordBreakStart + quoted + wordBreakEnd)
|
||||
return // caller is expected to wrap this error
|
||||
}
|
||||
|
||||
// FilterStatus stores a single status to filter.
|
||||
type FilterStatus struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the filter that this keyword belongs to.
|
||||
StatusID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the status to filter.
|
||||
}
|
||||
|
|
@ -23,81 +23,51 @@ import (
|
|||
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
|
||||
)
|
||||
|
||||
// Filter contains methods for creating, reading, updating, and deleting filters and their keyword and status entries.
|
||||
// Filter contains methods for creating, reading, updating,
|
||||
// and deleting filters and their keyword and status entries.
|
||||
type Filter interface {
|
||||
//<editor-fold desc="Filter methods">
|
||||
|
||||
// GetFilterByID gets one filter with the given id.
|
||||
GetFilterByID(ctx context.Context, id string) (*gtsmodel.Filter, error)
|
||||
|
||||
// GetFiltersForAccountID gets all filters owned by the given accountID.
|
||||
GetFiltersForAccountID(ctx context.Context, accountID string) ([]*gtsmodel.Filter, error)
|
||||
// GetFiltersByAccountID gets all filters owned by the given accountID.
|
||||
GetFiltersByAccountID(ctx context.Context, accountID string) ([]*gtsmodel.Filter, error)
|
||||
|
||||
// PutFilter puts a new filter in the database, adding any attached keywords or statuses.
|
||||
// It uses a transaction to ensure no partial updates.
|
||||
PutFilter(ctx context.Context, filter *gtsmodel.Filter) error
|
||||
|
||||
// UpdateFilter updates the given filter,
|
||||
// upserts any attached keywords and inserts any new statuses (existing statuses cannot be updated),
|
||||
// and deletes indicated filter keywords and statuses by ID.
|
||||
// It uses a transaction to ensure no partial updates.
|
||||
// The column lists are optional; if not specified, all columns will be updated.
|
||||
// The filter keyword columns list is *per keyword*.
|
||||
// To update all keyword columns, provide a list where every element is an empty list.
|
||||
UpdateFilter(
|
||||
ctx context.Context,
|
||||
filter *gtsmodel.Filter,
|
||||
filterColumns []string,
|
||||
filterKeywordColumns [][]string,
|
||||
deleteFilterKeywordIDs []string,
|
||||
deleteFilterStatusIDs []string,
|
||||
) error
|
||||
// UpdateFilter ...
|
||||
UpdateFilter(ctx context.Context, filter *gtsmodel.Filter, cols ...string) error
|
||||
|
||||
// DeleteFilterByID deletes one filter with the given ID.
|
||||
// It uses a transaction to ensure no partial updates.
|
||||
DeleteFilterByID(ctx context.Context, id string) error
|
||||
|
||||
//</editor-fold>
|
||||
|
||||
//<editor-fold desc="Filter keyword methods">
|
||||
// DeleteFilter deletes the given filter and all associated FilterKeyword{}
|
||||
// and FilterStatus{} models from the database in a single transaction.
|
||||
DeleteFilter(ctx context.Context, filter *gtsmodel.Filter) error
|
||||
|
||||
// GetFilterKeywordByID gets one filter keyword with the given ID.
|
||||
GetFilterKeywordByID(ctx context.Context, id string) (*gtsmodel.FilterKeyword, error)
|
||||
|
||||
// GetFilterKeywordsForFilterID gets filter keywords from the given filterID.
|
||||
GetFilterKeywordsForFilterID(ctx context.Context, filterID string) ([]*gtsmodel.FilterKeyword, error)
|
||||
|
||||
// GetFilterKeywordsForAccountID gets filter keywords from the given accountID.
|
||||
GetFilterKeywordsForAccountID(ctx context.Context, accountID string) ([]*gtsmodel.FilterKeyword, error)
|
||||
// GetFilterKeywordsByIDs ...
|
||||
GetFilterKeywordsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.FilterKeyword, error)
|
||||
|
||||
// PutFilterKeyword inserts a single filter keyword into the database.
|
||||
PutFilterKeyword(ctx context.Context, filterKeyword *gtsmodel.FilterKeyword) error
|
||||
|
||||
// UpdateFilterKeyword updates the given filter keyword.
|
||||
// Columns is optional, if not specified all will be updated.
|
||||
UpdateFilterKeyword(ctx context.Context, filterKeyword *gtsmodel.FilterKeyword, columns ...string) error
|
||||
UpdateFilterKeyword(ctx context.Context, filterKeyword *gtsmodel.FilterKeyword, cols ...string) error
|
||||
|
||||
// DeleteFilterKeywordByID deletes one filter keyword with the given id.
|
||||
DeleteFilterKeywordByID(ctx context.Context, id string) error
|
||||
|
||||
//</editor-fold>
|
||||
|
||||
//<editor-fold desc="Filter status methods">
|
||||
// DeleteFilterKeywordsByIDs deletes filter keywords with the given ids.
|
||||
DeleteFilterKeywordsByIDs(ctx context.Context, ids ...string) error
|
||||
|
||||
// GetFilterStatusByID gets one filter status with the given ID.
|
||||
GetFilterStatusByID(ctx context.Context, id string) (*gtsmodel.FilterStatus, error)
|
||||
|
||||
// GetFilterStatusesForFilterID gets filter statuses from the given filterID.
|
||||
GetFilterStatusesForFilterID(ctx context.Context, filterID string) ([]*gtsmodel.FilterStatus, error)
|
||||
|
||||
// GetFilterStatusesForAccountID gets filter keywords from the given accountID.
|
||||
GetFilterStatusesForAccountID(ctx context.Context, accountID string) ([]*gtsmodel.FilterStatus, error)
|
||||
// GetFilterStatusesByIDs ...
|
||||
GetFilterStatusesByIDs(ctx context.Context, ids []string) ([]*gtsmodel.FilterStatus, error)
|
||||
|
||||
// PutFilterStatus inserts a single filter status into the database.
|
||||
PutFilterStatus(ctx context.Context, filterStatus *gtsmodel.FilterStatus) error
|
||||
|
||||
// DeleteFilterStatusByID deletes one filter status with the given id.
|
||||
DeleteFilterStatusByID(ctx context.Context, id string) error
|
||||
|
||||
//</editor-fold>
|
||||
// DeleteFilterStatusesByIDs deletes filter statuses with the given ids.
|
||||
DeleteFilterStatusesByIDs(ctx context.Context, ids ...string) error
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue