mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-10-29 07:22:24 -05:00
[performance] filter model and database table improvements (#4277)
- removes unnecessary fields / columns (created_at, updated_at)
- replaces filter.context_* columns with singular filter.contexts bit field which should save both struct memory and database space
- replaces filter.action string with integer enum type which should save both struct memory and database space
- adds links from filter to filter_* tables with Filter{}.KeywordIDs and Filter{}.StatusIDs fields (this also means we now have those ID slices cached, which reduces some lookups)
- removes account_id fields from filter_* tables, since there's a more direct connection between filter and filter_* tables, and filter.account_id already exists
- refactors a bunch of the filter processor logic to save on code repetition, factor in the above changes, fix a few bugs with missed error returns and bring it more in-line with some of our newer code
Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4277
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
tobi
parent
9d5af6c3dc
commit
996da6e029
82 changed files with 2440 additions and 1722 deletions
|
|
@ -17,8 +17,17 @@
|
|||
|
||||
package gtsmodel
|
||||
|
||||
// smallint is the largest size supported
|
||||
// by a PostgreSQL SMALLINT, since an SQLite
|
||||
// SMALLINT is actually variable in size.
|
||||
type smallint int16
|
||||
|
||||
// enumType is the type we (at least, should) use
|
||||
// for database enum types. it is the largest size
|
||||
// supported by a PostgreSQL SMALLINT, since an
|
||||
// SQLite SMALLINT is actually variable in size.
|
||||
type enumType int16
|
||||
// for database enum types, as smallest int size.
|
||||
type enumType smallint
|
||||
|
||||
// bitFieldType is the type we use
|
||||
// for database int bit fields, at
|
||||
// least where the smallest int size
|
||||
// will suffice for number of fields.
|
||||
type bitFieldType smallint
|
||||
|
|
|
|||
|
|
@ -18,28 +18,251 @@
|
|||
package gtsmodel
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"code.superseriousbusiness.org/gotosocial/internal/util"
|
||||
"codeberg.org/gruf/go-byteutil"
|
||||
)
|
||||
|
||||
// FilterContext represents the
|
||||
// context in which a Filter applies.
|
||||
//
|
||||
// These are used as bit-field masks to determine
|
||||
// which are enabled in a FilterContexts bit field,
|
||||
// as well as to signify internally any particular
|
||||
// context in which a status should be filtered in.
|
||||
type FilterContext bitFieldType
|
||||
|
||||
const (
|
||||
// FilterContextNone means no filters should
|
||||
// be applied, this is for internal use only.
|
||||
FilterContextNone FilterContext = 0
|
||||
|
||||
// FilterContextHome means this status is being
|
||||
// filtered as part of a home or list timeline.
|
||||
FilterContextHome FilterContext = 1 << 1
|
||||
|
||||
// FilterContextNotifications means this status is
|
||||
// being filtered as part of the notifications timeline.
|
||||
FilterContextNotifications FilterContext = 1 << 2
|
||||
|
||||
// FilterContextPublic means this status is
|
||||
// being filtered as part of a public or tag timeline.
|
||||
FilterContextPublic FilterContext = 1 << 3
|
||||
|
||||
// FilterContextThread means this status is
|
||||
// being filtered as part of a thread's context.
|
||||
FilterContextThread FilterContext = 1 << 4
|
||||
|
||||
// FilterContextAccount means this status is
|
||||
// being filtered as part of an account's statuses.
|
||||
FilterContextAccount FilterContext = 1 << 5
|
||||
)
|
||||
|
||||
// String returns human-readable form of FilterContext.
|
||||
func (ctx FilterContext) String() string {
|
||||
switch ctx {
|
||||
case FilterContextNone:
|
||||
return ""
|
||||
case FilterContextHome:
|
||||
return "home"
|
||||
case FilterContextNotifications:
|
||||
return "notifications"
|
||||
case FilterContextPublic:
|
||||
return "public"
|
||||
case FilterContextThread:
|
||||
return "thread"
|
||||
case FilterContextAccount:
|
||||
return "account"
|
||||
default:
|
||||
panic(fmt.Sprintf("invalid filter context: %d", ctx))
|
||||
}
|
||||
}
|
||||
|
||||
// FilterContexts stores multiple contexts
|
||||
// in which a Filter applies as bits in an int.
|
||||
type FilterContexts bitFieldType
|
||||
|
||||
// Applies returns whether receiving FilterContexts applies in FilterContexts.
|
||||
func (ctxs FilterContexts) Applies(ctx FilterContext) bool {
|
||||
return ctxs&FilterContexts(ctx) != 0
|
||||
}
|
||||
|
||||
// Home returns whether FilterContextHome is set.
|
||||
func (ctxs FilterContexts) Home() bool {
|
||||
return ctxs&FilterContexts(FilterContextHome) != 0
|
||||
}
|
||||
|
||||
// SetHome will set the FilterContextHome bit.
|
||||
func (ctxs *FilterContexts) SetHome() {
|
||||
*ctxs |= FilterContexts(FilterContextHome)
|
||||
}
|
||||
|
||||
// UnsetHome will unset the FilterContextHome bit.
|
||||
func (ctxs *FilterContexts) UnsetHome() {
|
||||
*ctxs &= ^FilterContexts(FilterContextHome)
|
||||
}
|
||||
|
||||
// Notifications returns whether FilterContextNotifications is set.
|
||||
func (ctxs FilterContexts) Notifications() bool {
|
||||
return ctxs&FilterContexts(FilterContextNotifications) != 0
|
||||
}
|
||||
|
||||
// SetNotifications will set the FilterContextNotifications bit.
|
||||
func (ctxs *FilterContexts) SetNotifications() {
|
||||
*ctxs |= FilterContexts(FilterContextNotifications)
|
||||
}
|
||||
|
||||
// UnsetNotifications will unset the FilterContextNotifications bit.
|
||||
func (ctxs *FilterContexts) UnsetNotifications() {
|
||||
*ctxs &= ^FilterContexts(FilterContextNotifications)
|
||||
}
|
||||
|
||||
// Public returns whether FilterContextPublic is set.
|
||||
func (ctxs FilterContexts) Public() bool {
|
||||
return ctxs&FilterContexts(FilterContextPublic) != 0
|
||||
}
|
||||
|
||||
// SetPublic will set the FilterContextPublic bit.
|
||||
func (ctxs *FilterContexts) SetPublic() {
|
||||
*ctxs |= FilterContexts(FilterContextPublic)
|
||||
}
|
||||
|
||||
// UnsetPublic will unset the FilterContextPublic bit.
|
||||
func (ctxs *FilterContexts) UnsetPublic() {
|
||||
*ctxs &= ^FilterContexts(FilterContextPublic)
|
||||
}
|
||||
|
||||
// Thread returns whether FilterContextThread is set.
|
||||
func (ctxs FilterContexts) Thread() bool {
|
||||
return ctxs&FilterContexts(FilterContextThread) != 0
|
||||
}
|
||||
|
||||
// SetThread will set the FilterContextThread bit.
|
||||
func (ctxs *FilterContexts) SetThread() {
|
||||
*ctxs |= FilterContexts(FilterContextThread)
|
||||
}
|
||||
|
||||
// UnsetThread will unset the FilterContextThread bit.
|
||||
func (ctxs *FilterContexts) UnsetThread() {
|
||||
*ctxs &= ^FilterContexts(FilterContextThread)
|
||||
}
|
||||
|
||||
// Account returns whether FilterContextAccount is set.
|
||||
func (ctxs FilterContexts) Account() bool {
|
||||
return ctxs&FilterContexts(FilterContextAccount) != 0
|
||||
}
|
||||
|
||||
// SetAccount will set / unset the FilterContextAccount bit.
|
||||
func (ctxs *FilterContexts) SetAccount() {
|
||||
*ctxs |= FilterContexts(FilterContextAccount)
|
||||
}
|
||||
|
||||
// UnsetAccount will unset the FilterContextAccount bit.
|
||||
func (ctxs *FilterContexts) UnsetAccount() {
|
||||
*ctxs &= ^FilterContexts(FilterContextAccount)
|
||||
}
|
||||
|
||||
// String returns a single human-readable form of FilterContexts.
|
||||
func (ctxs FilterContexts) String() string {
|
||||
var buf byteutil.Buffer
|
||||
buf.Guarantee(72) // worst-case estimate
|
||||
buf.B = append(buf.B, '{')
|
||||
buf.B = append(buf.B, "home="...)
|
||||
buf.B = strconv.AppendBool(buf.B, ctxs.Home())
|
||||
buf.B = append(buf.B, ',')
|
||||
buf.B = append(buf.B, "notifications="...)
|
||||
buf.B = strconv.AppendBool(buf.B, ctxs.Notifications())
|
||||
buf.B = append(buf.B, ',')
|
||||
buf.B = append(buf.B, "public="...)
|
||||
buf.B = strconv.AppendBool(buf.B, ctxs.Public())
|
||||
buf.B = append(buf.B, ',')
|
||||
buf.B = append(buf.B, "thread="...)
|
||||
buf.B = strconv.AppendBool(buf.B, ctxs.Thread())
|
||||
buf.B = append(buf.B, ',')
|
||||
buf.B = append(buf.B, "account="...)
|
||||
buf.B = strconv.AppendBool(buf.B, ctxs.Account())
|
||||
buf.B = append(buf.B, '}')
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// FilterAction represents the action
|
||||
// to take on a filtered status.
|
||||
type FilterAction enumType
|
||||
|
||||
const (
|
||||
// FilterActionNone filters should not exist, except
|
||||
// internally, for partially constructed or invalid filters.
|
||||
FilterActionNone FilterAction = 0
|
||||
|
||||
// FilterActionWarn means that the
|
||||
// status should be shown behind a warning.
|
||||
FilterActionWarn FilterAction = 1
|
||||
|
||||
// FilterActionHide means that the status should
|
||||
// be removed from timeline results entirely.
|
||||
FilterActionHide FilterAction = 2
|
||||
)
|
||||
|
||||
// String returns human-readable form of FilterAction.
|
||||
func (act FilterAction) String() string {
|
||||
switch act {
|
||||
case FilterActionNone:
|
||||
return ""
|
||||
case FilterActionWarn:
|
||||
return "warn"
|
||||
case FilterActionHide:
|
||||
return "hide"
|
||||
default:
|
||||
panic(fmt.Sprintf("invalid filter action: %d", act))
|
||||
}
|
||||
}
|
||||
|
||||
// Filter stores a filter created by a local account.
|
||||
type Filter struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
|
||||
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
|
||||
ExpiresAt time.Time `bun:"type:timestamptz,nullzero"` // Time filter should expire. If null, should not expire.
|
||||
AccountID string `bun:"type:CHAR(26),notnull,nullzero,unique:filters_account_id_title_uniq"` // ID of the local account that created the filter.
|
||||
Title string `bun:",nullzero,notnull,unique:filters_account_id_title_uniq"` // The name of the filter.
|
||||
Action FilterAction `bun:",nullzero,notnull"` // The action to take.
|
||||
Keywords []*FilterKeyword `bun:"-"` // Keywords for this filter.
|
||||
Statuses []*FilterStatus `bun:"-"` // Statuses for this filter.
|
||||
ContextHome *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
|
||||
ContextNotifications *bool `bun:",nullzero,notnull,default:false"` // Apply filter to notifications.
|
||||
ContextPublic *bool `bun:",nullzero,notnull,default:false"` // Apply filter to home timeline and lists.
|
||||
ContextThread *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing a status's associated thread.
|
||||
ContextAccount *bool `bun:",nullzero,notnull,default:false"` // Apply filter when viewing an account profile.
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
ExpiresAt time.Time `bun:"type:timestamptz,nullzero"` // Time filter should expire. If null, should not expire.
|
||||
AccountID string `bun:"type:CHAR(26),notnull,nullzero,unique:filters_account_id_title_uniq"` // ID of the local account that created the filter.
|
||||
Title string `bun:",nullzero,notnull,unique:filters_account_id_title_uniq"` // The name of the filter.
|
||||
Action FilterAction `bun:",nullzero,notnull,default:0"` // The action to take.
|
||||
Keywords []*FilterKeyword `bun:"-"` // Keywords for this filter.
|
||||
KeywordIDs []string `bun:"keywords,array"` //
|
||||
Statuses []*FilterStatus `bun:"-"` // Statuses for this filter.
|
||||
StatusIDs []string `bun:"statuses,array"` //
|
||||
Contexts FilterContexts `bun:",nullzero,notnull,default:0"` // Which contexts does this filter apply in?
|
||||
}
|
||||
|
||||
// KeywordsPopulated returns whether keywords
|
||||
// are populated according to current KeywordIDs.
|
||||
func (f *Filter) KeywordsPopulated() bool {
|
||||
if len(f.KeywordIDs) != len(f.Keywords) {
|
||||
// this is the quickest indicator.
|
||||
return false
|
||||
}
|
||||
for i, id := range f.KeywordIDs {
|
||||
if f.Keywords[i].ID != id {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// StatusesPopulated returns whether statuses
|
||||
// are populated according to current StatusIDs.
|
||||
func (f *Filter) StatusesPopulated() bool {
|
||||
if len(f.StatusIDs) != len(f.Statuses) {
|
||||
// this is the quickest indicator.
|
||||
return false
|
||||
}
|
||||
for i, id := range f.StatusIDs {
|
||||
if f.Statuses[i].ID != id {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Expired returns whether the filter has expired at a given time.
|
||||
|
|
@ -51,11 +274,7 @@ func (f *Filter) Expired(now time.Time) bool {
|
|||
// FilterKeyword stores a single keyword to filter statuses against.
|
||||
type FilterKeyword struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
|
||||
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
|
||||
AccountID string `bun:"type:CHAR(26),notnull,nullzero"` // ID of the local account that created the filter keyword.
|
||||
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_keywords_filter_id_keyword_uniq"` // ID of the filter that this keyword belongs to.
|
||||
Filter *Filter `bun:"-"` // Filter corresponding to FilterID
|
||||
Keyword string `bun:",nullzero,notnull,unique:filter_keywords_filter_id_keyword_uniq"` // The keyword or phrase to filter against.
|
||||
WholeWord *bool `bun:",nullzero,notnull,default:false"` // Should the filter consider word boundaries?
|
||||
Regexp *regexp.Regexp `bun:"-"` // pre-prepared regular expression
|
||||
|
|
@ -72,6 +291,7 @@ func (k *FilterKeyword) Compile() (err error) {
|
|||
// Either word boundary or
|
||||
// whitespace or start of line.
|
||||
wordBreakStart = `(?:\b|\s|^)`
|
||||
|
||||
// Either word boundary or
|
||||
// whitespace or end of line.
|
||||
wordBreakEnd = `(?:\b|\s|$)`
|
||||
|
|
@ -85,23 +305,7 @@ func (k *FilterKeyword) Compile() (err error) {
|
|||
|
||||
// FilterStatus stores a single status to filter.
|
||||
type FilterStatus struct {
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
|
||||
UpdatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
|
||||
AccountID string `bun:"type:CHAR(26),notnull,nullzero"` // ID of the local account that created the filter keyword.
|
||||
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the filter that this keyword belongs to.
|
||||
Filter *Filter `bun:"-"` // Filter corresponding to FilterID
|
||||
StatusID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the status to filter.
|
||||
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
|
||||
FilterID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the filter that this keyword belongs to.
|
||||
StatusID string `bun:"type:CHAR(26),notnull,nullzero,unique:filter_statuses_filter_id_status_id_uniq"` // ID of the status to filter.
|
||||
}
|
||||
|
||||
// FilterAction represents the action to take on a filtered status.
|
||||
type FilterAction string
|
||||
|
||||
const (
|
||||
// FilterActionNone filters should not exist, except internally, for partially constructed or invalid filters.
|
||||
FilterActionNone FilterAction = ""
|
||||
// FilterActionWarn means that the status should be shown behind a warning.
|
||||
FilterActionWarn FilterAction = "warn"
|
||||
// FilterActionHide means that the status should be removed from timeline results entirely.
|
||||
FilterActionHide FilterAction = "hide"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue