diff --git a/internal/cache/cache.go b/internal/cache/cache.go index d3d2d5f2b..5611ddec0 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -41,21 +41,22 @@ type Caches struct { // the block []headerfilter.Filter cache. BlockHeaderFilters headerfilter.Cache - // TTL cache of statuses -> filterable text fields. - // To ensure up-to-date fields, cache is keyed as: - // `[status.ID][status.UpdatedAt.Unix()]` - StatusesFilterableFields *ttl.Cache[string, []string] - - // Timelines ... + // Timelines provides access to the + // collection of timeline object caches, + // used in timeline lookups and streaming. Timelines TimelineCaches // Mutes provides access to the item mutes // cache. (used by the item mutes filter). - Mutes MutesCache + Mutes StructCache[*CachedMute] + + // StatusFilter provides access to the status filter + // cache. (used by the status-filter results filter). + StatusFilter StructCache[*CachedStatusFilterResults] // Visibility provides access to the item visibility // cache. (used by the visibility filter). - Visibility VisibilityCache + Visibility StructCache[*CachedVisibility] // Webfinger provides access to the webfinger URL cache. Webfinger *ttl.Cache[string, string] // TTL=24hr, sweep=5min @@ -119,7 +120,6 @@ func (c *Caches) Init() { c.initStatusEdit() c.initStatusFave() c.initStatusFaveIDs() - c.initStatusesFilterableFields() c.initTag() c.initThreadMute() c.initToken() @@ -131,6 +131,7 @@ func (c *Caches) Init() { c.initWebPushSubscription() c.initWebPushSubscriptionIDs() c.initMutes() + c.initStatusFilter() c.initVisibility() } @@ -143,10 +144,6 @@ func (c *Caches) Start() error { return gtserror.New("could not start webfinger cache") } - if !c.StatusesFilterableFields.Start(5 * time.Minute) { - return gtserror.New("could not start statusesFilterableFields cache") - } - return nil } @@ -158,9 +155,6 @@ func (c *Caches) Stop() { if c.Webfinger != nil { _ = c.Webfinger.Stop() } - if c.StatusesFilterableFields != nil { - _ = c.StatusesFilterableFields.Stop() - } } // Sweep will sweep all the available caches to ensure none @@ -183,6 +177,7 @@ func (c *Caches) Sweep(threshold float64) { c.DB.Emoji.Trim(threshold) c.DB.EmojiCategory.Trim(threshold) c.DB.Filter.Trim(threshold) + c.DB.FilterIDs.Trim(threshold) c.DB.FilterKeyword.Trim(threshold) c.DB.FilterStatus.Trim(threshold) c.DB.Follow.Trim(threshold) @@ -218,20 +213,13 @@ func (c *Caches) Sweep(threshold float64) { c.DB.User.Trim(threshold) c.DB.UserMute.Trim(threshold) c.DB.UserMuteIDs.Trim(threshold) + c.Mutes.Trim(threshold) + c.StatusFilter.Trim(threshold) c.Timelines.Home.Trim() c.Timelines.List.Trim() c.Visibility.Trim(threshold) } -func (c *Caches) initStatusesFilterableFields() { - c.StatusesFilterableFields = new(ttl.Cache[string, []string]) - c.StatusesFilterableFields.Init( - 0, - 512, - 1*time.Hour, - ) -} - func (c *Caches) initWebfinger() { // Calculate maximum cache size. cap := calculateCacheMax( diff --git a/internal/cache/invalidate.go b/internal/cache/invalidate.go index 4941b2540..569238e9b 100644 --- a/internal/cache/invalidate.go +++ b/internal/cache/invalidate.go @@ -26,19 +26,23 @@ import ( // as an invalidation indicates a database INSERT / UPDATE / DELETE. // NOTE THEY ARE ONLY CALLED WHEN THE ITEM IS IN THE CACHE, SO FOR // HOOKS TO BE CALLED ON DELETE YOU MUST FIRST POPULATE IT IN THE CACHE. +// +// Also note that while Timelines are a part of the Caches{} object, +// they are generally not modified as part of side-effects here, as +// they often need specific IDs or more information that can only be +// fetched from the database. As such, they are generally handled as +// side-effects in the ./internal/processor/workers/ package. func (c *Caches) OnInvalidateAccount(account *gtsmodel.Account) { - // Invalidate cached stats objects for this account. - c.DB.AccountStats.Invalidate("AccountID", account.ID) - // Invalidate as possible visibility target result. c.Visibility.Invalidate("ItemID", account.ID) // If account is local, invalidate as - // possible mute / visibility result requester. + // possible visibility result requester, + // also, invalidate any cached stats. if account.IsLocal() { + c.DB.AccountStats.Invalidate("AccountID", account.ID) c.Visibility.Invalidate("RequesterID", account.ID) - c.Mutes.Invalidate("RequesterID", account.ID) } // Invalidate this account's @@ -94,9 +98,8 @@ func (c *Caches) OnInvalidateBlock(block *gtsmodel.Block) { localAccountIDs = append(localAccountIDs, block.TargetAccountID) } - // Now perform local mute / visibility result invalidations. + // Now perform local visibility result invalidations. c.Visibility.InvalidateIDs("RequesterID", localAccountIDs) - c.Mutes.InvalidateIDs("RequesterID", localAccountIDs) // Invalidate source account's block lists. c.DB.BlockIDs.Invalidate(block.AccountID) @@ -120,9 +123,8 @@ func (c *Caches) OnInvalidateFilter(filter *gtsmodel.Filter) { c.DB.FilterKeyword.InvalidateIDs("ID", filter.KeywordIDs) c.DB.FilterStatus.InvalidateIDs("ID", filter.StatusIDs) - // Invalidate account's timelines (in case local). - c.Timelines.Home.Unprepare(filter.AccountID) - c.Timelines.List.Unprepare(filter.AccountID) + // Invalidate account's status filter cache. + c.StatusFilter.Invalidate("RequesterID", filter.AccountID) } func (c *Caches) OnInvalidateFilterKeyword(filterKeyword *gtsmodel.FilterKeyword) { @@ -161,9 +163,8 @@ func (c *Caches) OnInvalidateFollow(follow *gtsmodel.Follow) { localAccountIDs = append(localAccountIDs, follow.TargetAccountID) } - // Now perform local mute / visibility result invalidations. + // Now perform local visibility result invalidations. c.Visibility.InvalidateIDs("RequesterID", localAccountIDs) - c.Mutes.InvalidateIDs("RequesterID", localAccountIDs) // Invalidate ID slice cache. c.DB.FollowIDs.Invalidate( @@ -295,6 +296,9 @@ func (c *Caches) OnInvalidateStatus(status *gtsmodel.Status) { // Invalidate cached stats objects for this account. c.DB.AccountStats.Invalidate("AccountID", status.AccountID) + // Invalidate filter results targeting status. + c.StatusFilter.Invalidate("StatusID", status.ID) + // Invalidate status ID cached visibility. c.Visibility.Invalidate("ItemID", status.ID) diff --git a/internal/cache/mutes.go b/internal/cache/mutes.go index 9ad7736a0..bdf7990dc 100644 --- a/internal/cache/mutes.go +++ b/internal/cache/mutes.go @@ -25,10 +25,6 @@ import ( "codeberg.org/gruf/go-structr" ) -type MutesCache struct { - StructCache[*CachedMute] -} - func (c *Caches) initMutes() { // Calculate maximum cache size. cap := calculateResultCacheMax( diff --git a/internal/cache/size.go b/internal/cache/size.go index 8a6c9e9ad..ab54ada87 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -25,6 +25,7 @@ import ( "unsafe" "code.superseriousbusiness.org/gotosocial/internal/ap" + apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" "code.superseriousbusiness.org/gotosocial/internal/config" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/id" @@ -653,6 +654,20 @@ func sizeofStatusFave() uintptr { })) } +func sizeofStatusFilterResults() uintptr { + return uintptr(size.Of(&CachedStatusFilterResults{ + StatusID: exampleID, + RequesterID: exampleID, + Results: [5][]StatusFilterResult{ + {{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}}, + {{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}}, + {{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}}, + {{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}}, + {{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}}, + }, + })) +} + func sizeofTag() uintptr { return uintptr(size.Of(>smodel.Tag{ ID: exampleID, diff --git a/internal/cache/statusfilter.go b/internal/cache/statusfilter.go new file mode 100644 index 000000000..073caa7f0 --- /dev/null +++ b/internal/cache/statusfilter.go @@ -0,0 +1,107 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cache + +import ( + "time" + + apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" + "code.superseriousbusiness.org/gotosocial/internal/config" + "code.superseriousbusiness.org/gotosocial/internal/log" + "codeberg.org/gruf/go-structr" +) + +func (c *Caches) initStatusFilter() { + // Calculate maximum cache size. + cap := calculateResultCacheMax( + sizeofStatusFilterResults(), // model in-mem size. + config.GetCacheStatusFilterMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(r1 *CachedStatusFilterResults) *CachedStatusFilterResults { + r2 := new(CachedStatusFilterResults) + *r2 = *r1 + return r2 + } + + c.StatusFilter.Init(structr.CacheConfig[*CachedStatusFilterResults]{ + Indices: []structr.IndexConfig{ + {Fields: "RequesterID,StatusID"}, + {Fields: "StatusID", Multiple: true}, + {Fields: "RequesterID", Multiple: true}, + }, + MaxSize: cap, + IgnoreErr: func(err error) bool { + // don't cache any errors, + // it gets a little too tricky + // otherwise with ensuring + // errors are cleared out + return true + }, + Copy: copyF, + }) +} + +const ( + KeyContextHome = iota + KeyContextPublic + KeyContextNotifs + KeyContextThread + KeyContextAccount + keysLen // must always be last in list +) + +// CachedStatusFilterResults contains the +// results of a cached status filter lookup. +type CachedStatusFilterResults struct { + + // StatusID is the ID of the + // status this is a result for. + StatusID string + + // RequesterID is the ID of the requesting + // account for this status filter lookup. + RequesterID string + + // Results is a map (int-key-array) of status filter + // result slices in all possible filtering contexts. + Results [keysLen][]StatusFilterResult +} + +// StatusFilterResult stores a single (positive, +// i.e. match) filter result for a status by a filter. +type StatusFilterResult struct { + + // Expiry stores the time at which + // (if any) the filter result expires. + Expiry time.Time + + // Result stores any generated filter result for + // this match intended to be shown at the frontend. + // This can be used to determine the filter action: + // - value => gtsmodel.FilterActionWarn + // - nil => gtsmodel.FilterActionHide + Result *apimodel.FilterResult +} + +// Expired returns whether the filter result has expired. +func (r *StatusFilterResult) Expired(now time.Time) bool { + return !r.Expiry.IsZero() && !r.Expiry.After(now) +} diff --git a/internal/cache/visibility.go b/internal/cache/visibility.go index 3797ab701..bfb72e4f6 100644 --- a/internal/cache/visibility.go +++ b/internal/cache/visibility.go @@ -23,10 +23,6 @@ import ( "codeberg.org/gruf/go-structr" ) -type VisibilityCache struct { - StructCache[*CachedVisibility] -} - func (c *Caches) initVisibility() { // Calculate maximum cache size. cap := calculateResultCacheMax( diff --git a/internal/config/config.go b/internal/config/config.go index 528000478..8139770e0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -267,6 +267,7 @@ type CacheConfiguration struct { WebPushSubscriptionMemRatio float64 `name:"web-push-subscription-mem-ratio"` WebPushSubscriptionIDsMemRatio float64 `name:"web-push-subscription-ids-mem-ratio"` MutesMemRatio float64 `name:"mutes-mem-ratio"` + StatusFilterMemRatio float64 `name:"status-filter-mem-ratio"` VisibilityMemRatio float64 `name:"visibility-mem-ratio"` } diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 82de65bb7..1540cc76b 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -233,8 +233,9 @@ var Defaults = Configuration{ WebfingerMemRatio: 0.1, WebPushSubscriptionMemRatio: 1, WebPushSubscriptionIDsMemRatio: 1, - VisibilityMemRatio: 2, MutesMemRatio: 2, + StatusFilterMemRatio: 7, + VisibilityMemRatio: 2, }, HTTPClient: HTTPClientConfiguration{ diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 7dfe1db23..36dd927f8 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -211,6 +211,7 @@ const ( CacheWebPushSubscriptionMemRatioFlag = "cache-web-push-subscription-mem-ratio" CacheWebPushSubscriptionIDsMemRatioFlag = "cache-web-push-subscription-ids-mem-ratio" CacheMutesMemRatioFlag = "cache-mutes-mem-ratio" + CacheStatusFilterMemRatioFlag = "cache-status-filter-mem-ratio" CacheVisibilityMemRatioFlag = "cache-visibility-mem-ratio" AdminAccountUsernameFlag = "username" AdminAccountEmailFlag = "email" @@ -404,11 +405,12 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) { flags.Float64("cache-web-push-subscription-mem-ratio", cfg.Cache.WebPushSubscriptionMemRatio, "") flags.Float64("cache-web-push-subscription-ids-mem-ratio", cfg.Cache.WebPushSubscriptionIDsMemRatio, "") flags.Float64("cache-mutes-mem-ratio", cfg.Cache.MutesMemRatio, "") + flags.Float64("cache-status-filter-mem-ratio", cfg.Cache.StatusFilterMemRatio, "") flags.Float64("cache-visibility-mem-ratio", cfg.Cache.VisibilityMemRatio, "") } func (cfg *Configuration) MarshalMap() map[string]any { - cfgmap := make(map[string]any, 190) + cfgmap := make(map[string]any, 191) cfgmap["log-level"] = cfg.LogLevel cfgmap["log-timestamp-format"] = cfg.LogTimestampFormat cfgmap["log-db-queries"] = cfg.LogDbQueries @@ -591,6 +593,7 @@ func (cfg *Configuration) MarshalMap() map[string]any { cfgmap["cache-web-push-subscription-mem-ratio"] = cfg.Cache.WebPushSubscriptionMemRatio cfgmap["cache-web-push-subscription-ids-mem-ratio"] = cfg.Cache.WebPushSubscriptionIDsMemRatio cfgmap["cache-mutes-mem-ratio"] = cfg.Cache.MutesMemRatio + cfgmap["cache-status-filter-mem-ratio"] = cfg.Cache.StatusFilterMemRatio cfgmap["cache-visibility-mem-ratio"] = cfg.Cache.VisibilityMemRatio cfgmap["username"] = cfg.AdminAccountUsername cfgmap["email"] = cfg.AdminAccountEmail @@ -2098,6 +2101,14 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error { } } + if ival, ok := cfgmap["cache-status-filter-mem-ratio"]; ok { + var err error + cfg.Cache.StatusFilterMemRatio, err = cast.ToFloat64E(ival) + if err != nil { + return fmt.Errorf("error casting %#v -> float64 for 'cache-status-filter-mem-ratio': %w", ival, err) + } + } + if ival, ok := cfgmap["cache-visibility-mem-ratio"]; ok { var err error cfg.Cache.VisibilityMemRatio, err = cast.ToFloat64E(ival) @@ -6197,6 +6208,28 @@ func GetCacheMutesMemRatio() float64 { return global.GetCacheMutesMemRatio() } // SetCacheMutesMemRatio safely sets the value for global configuration 'Cache.MutesMemRatio' field func SetCacheMutesMemRatio(v float64) { global.SetCacheMutesMemRatio(v) } +// GetCacheStatusFilterMemRatio safely fetches the Configuration value for state's 'Cache.StatusFilterMemRatio' field +func (st *ConfigState) GetCacheStatusFilterMemRatio() (v float64) { + st.mutex.RLock() + v = st.config.Cache.StatusFilterMemRatio + st.mutex.RUnlock() + return +} + +// SetCacheStatusFilterMemRatio safely sets the Configuration value for state's 'Cache.StatusFilterMemRatio' field +func (st *ConfigState) SetCacheStatusFilterMemRatio(v float64) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.StatusFilterMemRatio = v + st.reloadToViper() +} + +// GetCacheStatusFilterMemRatio safely fetches the value for global configuration 'Cache.StatusFilterMemRatio' field +func GetCacheStatusFilterMemRatio() float64 { return global.GetCacheStatusFilterMemRatio() } + +// SetCacheStatusFilterMemRatio safely sets the value for global configuration 'Cache.StatusFilterMemRatio' field +func SetCacheStatusFilterMemRatio(v float64) { global.SetCacheStatusFilterMemRatio(v) } + // GetCacheVisibilityMemRatio safely fetches the Configuration value for state's 'Cache.VisibilityMemRatio' field func (st *ConfigState) GetCacheVisibilityMemRatio() (v float64) { st.mutex.RLock() @@ -6433,6 +6466,7 @@ func (st *ConfigState) GetTotalOfMemRatios() (total float64) { total += st.config.Cache.WebPushSubscriptionMemRatio total += st.config.Cache.WebPushSubscriptionIDsMemRatio total += st.config.Cache.MutesMemRatio + total += st.config.Cache.StatusFilterMemRatio total += st.config.Cache.VisibilityMemRatio st.mutex.RUnlock() return @@ -7395,6 +7429,17 @@ func flattenConfigMap(cfgmap map[string]any) { } } + for _, key := range [][]string{ + {"cache", "status-filter-mem-ratio"}, + } { + ival, ok := mapGet(cfgmap, key...) + if ok { + cfgmap["cache-status-filter-mem-ratio"] = ival + nestedKeys[key[0]] = struct{}{} + break + } + } + for _, key := range [][]string{ {"cache", "visibility-mem-ratio"}, } { diff --git a/internal/filter/mutes/filter.go b/internal/filter/mutes/filter.go index 20adc3daf..fc5dd3362 100644 --- a/internal/filter/mutes/filter.go +++ b/internal/filter/mutes/filter.go @@ -41,5 +41,5 @@ const noauth = "noauth" // given statuses or accounts are muted by a requester (user). type Filter struct{ state *state.State } -// NewFilter returns a new Filter interface that will use the provided database. +// NewFilter returns a new Filter interface that will use the provided state. func NewFilter(state *state.State) *Filter { return &Filter{state: state} } diff --git a/internal/filter/status/api.go b/internal/filter/status/api.go new file mode 100644 index 000000000..1d6684b59 --- /dev/null +++ b/internal/filter/status/api.go @@ -0,0 +1,105 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package status + +import ( + "time" + + apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" + "code.superseriousbusiness.org/gotosocial/internal/gtserror" + "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" + "code.superseriousbusiness.org/gotosocial/internal/util" +) + +// NOTE: the below functions have all been copied +// from typeutils to prevent an import cycle. when +// we move the filtering logic out of the converter +// then we can safely remove these and call necessary +// function without any worry of import cycles. + +func toAPIFilterV2(filter *gtsmodel.Filter) apimodel.FilterV2 { + apiFilterKeywords := make([]apimodel.FilterKeyword, len(filter.Keywords)) + if len(apiFilterKeywords) != len(filter.Keywords) { + // bound check eliminiation compiler-hint + panic(gtserror.New("BCE")) + } + for i, filterKeyword := range filter.Keywords { + apiFilterKeywords[i] = apimodel.FilterKeyword{ + ID: filterKeyword.ID, + Keyword: filterKeyword.Keyword, + WholeWord: util.PtrOrValue(filterKeyword.WholeWord, false), + } + } + apiFilterStatuses := make([]apimodel.FilterStatus, len(filter.Statuses)) + if len(apiFilterStatuses) != len(filter.Statuses) { + // bound check eliminiation compiler-hint + panic(gtserror.New("BCE")) + } + for i, filterStatus := range filter.Statuses { + apiFilterStatuses[i] = apimodel.FilterStatus{ + ID: filterStatus.ID, + StatusID: filterStatus.StatusID, + } + } + return apimodel.FilterV2{ + ID: filter.ID, + Title: filter.Title, + Context: toAPIFilterContexts(filter), + ExpiresAt: toAPIFilterExpiresAt(filter.ExpiresAt), + FilterAction: toAPIFilterAction(filter.Action), + Keywords: apiFilterKeywords, + Statuses: apiFilterStatuses, + } +} + +func toAPIFilterExpiresAt(expiresAt time.Time) *string { + if expiresAt.IsZero() { + return nil + } + return util.Ptr(util.FormatISO8601(expiresAt)) +} + +func toAPIFilterContexts(filter *gtsmodel.Filter) []apimodel.FilterContext { + apiContexts := make([]apimodel.FilterContext, 0, apimodel.FilterContextNumValues) + if filter.Contexts.Home() { + apiContexts = append(apiContexts, apimodel.FilterContextHome) + } + if filter.Contexts.Notifications() { + apiContexts = append(apiContexts, apimodel.FilterContextNotifications) + } + if filter.Contexts.Public() { + apiContexts = append(apiContexts, apimodel.FilterContextPublic) + } + if filter.Contexts.Thread() { + apiContexts = append(apiContexts, apimodel.FilterContextThread) + } + if filter.Contexts.Account() { + apiContexts = append(apiContexts, apimodel.FilterContextAccount) + } + return apiContexts +} + +func toAPIFilterAction(m gtsmodel.FilterAction) apimodel.FilterAction { + switch m { + case gtsmodel.FilterActionWarn: + return apimodel.FilterActionWarn + case gtsmodel.FilterActionHide: + return apimodel.FilterActionHide + } + return apimodel.FilterActionNone +} diff --git a/internal/filter/status/filter.go b/internal/filter/status/filter.go new file mode 100644 index 000000000..d9ec12934 --- /dev/null +++ b/internal/filter/status/filter.go @@ -0,0 +1,33 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package status + +import ( + "code.superseriousbusiness.org/gotosocial/internal/state" +) + +// noauth is a placeholder ID used in cache lookups +// when there is no authorized account ID to use. +const noauth = "noauth" + +// Filter packages up logic for checking whether +// given status is muted by a given requester (user). +type Filter struct{ state *state.State } + +// New returns a new Filter interface that will use the provided state. +func NewFilter(state *state.State) *Filter { return &Filter{state} } diff --git a/internal/filter/status/status.go b/internal/filter/status/status.go index 1a611cdd1..5f997129d 100644 --- a/internal/filter/status/status.go +++ b/internal/filter/status/status.go @@ -15,12 +15,317 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -// Package status represents status filters managed by the user through the API. package status import ( - "errors" + "context" + "regexp" + "slices" + "time" + + apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" + "code.superseriousbusiness.org/gotosocial/internal/cache" + "code.superseriousbusiness.org/gotosocial/internal/gtserror" + "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" ) -// ErrHideStatus indicates that a status has been filtered and should not be returned at all. -var ErrHideStatus = errors.New("hide status") +// StatusFilterResultsInContext returns status filtering results, limited +// to the given filtering context, about the given status for requester. +// The hide flag is immediately returned if any filters match with the +// HIDE action set, else API model filter results for the WARN action. +func (f *Filter) StatusFilterResultsInContext( + ctx context.Context, + requester *gtsmodel.Account, + status *gtsmodel.Status, + context gtsmodel.FilterContext, +) ( + results []apimodel.FilterResult, + hidden bool, + err error, +) { + if context == gtsmodel.FilterContextNone { + // fast-check any context. + return nil, false, nil + } + + // Get cached filter results for status to requester in all contexts. + allResults, now, err := f.StatusFilterResults(ctx, requester, status) + if err != nil { + return nil, false, err + } + + // Get results applicable to current context. + var forContext []cache.StatusFilterResult + switch context { + case gtsmodel.FilterContextHome: + forContext = allResults.Results[cache.KeyContextHome] + case gtsmodel.FilterContextPublic: + forContext = allResults.Results[cache.KeyContextPublic] + case gtsmodel.FilterContextNotifications: + forContext = allResults.Results[cache.KeyContextNotifs] + case gtsmodel.FilterContextThread: + forContext = allResults.Results[cache.KeyContextThread] + case gtsmodel.FilterContextAccount: + forContext = allResults.Results[cache.KeyContextAccount] + } + + // Iterate results in context, gathering prepared API models. + results = make([]apimodel.FilterResult, 0, len(forContext)) + for _, result := range forContext { + + // Check if result expired. + if result.Expired(now) { + continue + } + + // If the result indicates + // status should just be + // hidden then return here. + if result.Result == nil { + return nil, true, nil + } + + // Append pre-prepared API model to slice. + results = append(results, *result.Result) + } + + return +} + +// StatusFilterResults returns status filtering results (in all contexts) about the given status for the given requesting account. +func (f *Filter) StatusFilterResults(ctx context.Context, requester *gtsmodel.Account, status *gtsmodel.Status) (*cache.CachedStatusFilterResults, time.Time, error) { + + // For requester ID use a + // fallback 'noauth' string + // by default for lookups. + requesterID := noauth + if requester != nil { + requesterID = requester.ID + } + + // Get current time. + now := time.Now() + + // Load status filtering results for this requesting account about status from cache, using load callback function if necessary. + results, err := f.state.Caches.StatusFilter.LoadOne("RequesterID,StatusID", func() (*cache.CachedStatusFilterResults, error) { + + // Load status filter results for given status. + results, err := f.getStatusFilterResults(ctx, + requester, + status, + now, + ) + if err != nil { + if err == cache.SentinelError { + // Filter-out our temporary + // race-condition error. + return &cache.CachedStatusFilterResults{}, nil + } + + return nil, err + } + + // Convert to cacheable results type. + return &cache.CachedStatusFilterResults{ + StatusID: status.ID, + RequesterID: requesterID, + Results: results, + }, nil + }, requesterID, status.ID) + if err != nil { + return nil, now, err + } + + return results, now, err +} + +// getStatusFilterResults loads status filtering results for +// the given status, given the current time (checking expiries). +// this will load results for all possible filtering contexts. +func (f *Filter) getStatusFilterResults( + ctx context.Context, + requester *gtsmodel.Account, + status *gtsmodel.Status, + now time.Time, +) ( + [5][]cache.StatusFilterResult, + error, +) { + var results [5][]cache.StatusFilterResult + + if requester == nil { + // Without auth, there will be no possible + // filters to exists, return as 'unfiltered'. + return results, nil + } + + // Get the string fields status is + // filterable on for keyword matching. + fields := getFilterableFields(status) + + // Get all status filters owned by the requesting account. + filters, err := f.state.DB.GetFiltersByAccountID(ctx, requester.ID) + if err != nil { + return results, gtserror.Newf("error getting account filters: %w", err) + } + + // For proper status filtering we need all fields populated. + if err := f.state.DB.PopulateStatus(ctx, status); err != nil { + return results, gtserror.Newf("error populating status: %w", err) + } + + // Generate result for each filter. + for _, filter := range filters { + + // Skip already expired. + if filter.Expired(now) { + continue + } + + // Later stored API result, if any. + // (for the HIDE action, it is unset). + var apiResult *apimodel.FilterResult + + switch filter.Action { + case gtsmodel.FilterActionWarn: + // For filter action WARN get all possible filter matches against status. + keywordMatches, statusMatches := getFilterMatches(filter, status.ID, fields) + if len(keywordMatches) == 0 && len(statusMatches) == 0 { + continue + } + + // Wrap matches in frontend API model. + apiResult = &apimodel.FilterResult{ + Filter: toAPIFilterV2(filter), + + KeywordMatches: keywordMatches, + StatusMatches: statusMatches, + } + + // For filter action HIDE quickly + // look for first possible match + // against this status, or reloop. + case gtsmodel.FilterActionHide: + if !doesFilterMatch(filter, status.ID, fields) { + continue + } + } + + // Wrap the filter result in our cache model. + // This model simply existing implies this + // status has been filtered, defaulting to + // action HIDE, or WARN on a non-nil result. + result := cache.StatusFilterResult{ + Expiry: filter.ExpiresAt, + Result: apiResult, + } + + // Append generated result if + // applies in 'home' context. + if filter.Contexts.Home() { + const key = cache.KeyContextHome + results[key] = append(results[key], result) + } + + // Append generated result if + // applies in 'public' context. + if filter.Contexts.Public() { + const key = cache.KeyContextPublic + results[key] = append(results[key], result) + } + + // Append generated result if + // applies in 'notifs' context. + if filter.Contexts.Notifications() { + const key = cache.KeyContextNotifs + results[key] = append(results[key], result) + } + + // Append generated result if + // applies in 'thread' context. + if filter.Contexts.Thread() { + const key = cache.KeyContextThread + results[key] = append(results[key], result) + } + + // Append generated result if + // applies in 'account' context. + if filter.Contexts.Account() { + const key = cache.KeyContextAccount + results[key] = append(results[key], result) + } + } + + // Iterate all filter results. + for _, key := range [5]int{ + cache.KeyContextHome, + cache.KeyContextPublic, + cache.KeyContextNotifs, + cache.KeyContextThread, + cache.KeyContextAccount, + } { + // Sort the slice of filter results by their expiry, soonest coming first. + slices.SortFunc(results[key], func(a, b cache.StatusFilterResult) int { + const k = +1 + switch { + case a.Expiry.IsZero(): + if b.Expiry.IsZero() { + return 0 + } + return +k + case b.Expiry.IsZero(): + return -k + case a.Expiry.Before(b.Expiry): + return -k + case b.Expiry.Before(a.Expiry): + return +k + default: + return 0 + } + }) + } + + return results, nil +} + +// getFilterMatches returns *all* the keyword and status matches of status ID and fields on given filter. +func getFilterMatches(filter *gtsmodel.Filter, statusID string, fields []string) ([]string, []string) { + keywordMatches := make([]string, 0, len(filter.Keywords)) + for _, keyword := range filter.Keywords { + if doesKeywordMatch(keyword.Regexp, fields) { + keywordMatches = append(keywordMatches, keyword.Keyword) + } + } + statusMatches := make([]string, 0, 1) + for _, status := range filter.Statuses { + if status.StatusID == statusID { + statusMatches = append(statusMatches, statusID) + } + } + return keywordMatches, statusMatches +} + +// doesFilterMatch returns if any of fields or status ID match on the given filter. +func doesFilterMatch(filter *gtsmodel.Filter, statusID string, fields []string) bool { + for _, status := range filter.Statuses { + if status.StatusID == statusID { + return true + } + } + for _, keyword := range filter.Keywords { + if doesKeywordMatch(keyword.Regexp, fields) { + return true + } + } + return false +} + +// doesKeywordMatch returns if any of fields match given keyword regex. +func doesKeywordMatch(rgx *regexp.Regexp, fields []string) bool { + for _, field := range fields { + if rgx.MatchString(field) { + return true + } + } + return false +} diff --git a/internal/filter/status/text.go b/internal/filter/status/text.go new file mode 100644 index 000000000..347e1193c --- /dev/null +++ b/internal/filter/status/text.go @@ -0,0 +1,80 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package status + +import ( + "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" + "code.superseriousbusiness.org/gotosocial/internal/text" +) + +// getFilterableFields returns text fields from +// a status that we might want to filter on: +// +// - content warning +// - content (converted to plaintext from HTML) +// - media descriptions +// - poll options +// +// Each field should be filtered separately. This avoids +// scenarios where false-positive multiple-word matches +// can be made by matching the last word of one field +// combined with the first word of the next field together. +func getFilterableFields(status *gtsmodel.Status) []string { + + // Estimate expected no of status fields. + fieldCount := 2 + len(status.Attachments) + if status.Poll != nil { + fieldCount += len(status.Poll.Options) + } + fields := make([]string, 0, fieldCount) + + // Append content warning / title. + if status.ContentWarning != "" { + fields = append(fields, status.ContentWarning) + } + + // Status content. Though we have raw text + // available for statuses created on our + // instance, use the plaintext version to + // remove markdown-formatting characters + // and ensure more consistent filtering. + if status.Content != "" { + text := text.ParseHTMLToPlain(status.Content) + if text != "" { + fields = append(fields, text) + } + } + + // Media descriptions, only where they are set. + for _, attachment := range status.Attachments { + if attachment.Description != "" { + fields = append(fields, attachment.Description) + } + } + + // Non-empty poll options. + if status.Poll != nil { + for _, opt := range status.Poll.Options { + if opt != "" { + fields = append(fields, opt) + } + } + } + + return fields +} diff --git a/internal/filter/status/text_test.go b/internal/filter/status/text_test.go new file mode 100644 index 000000000..f9283f826 --- /dev/null +++ b/internal/filter/status/text_test.go @@ -0,0 +1,84 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package status + +import ( + "testing" + + "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" + "github.com/stretchr/testify/assert" +) + +func TestFilterableText(t *testing.T) { + type testcase struct { + status *gtsmodel.Status + expectedFields []string + } + + for _, testcase := range []testcase{ + { + status: >smodel.Status{ + ContentWarning: "This is a test status", + Content: `

Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.

`, + }, + expectedFields: []string{ + "This is a test status", + "Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.", + }, + }, + { + status: >smodel.Status{ + Content: `

@zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)

https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863

`, + }, + expectedFields: []string{ + "@zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)\n\nhttps://codeberg.org/superseriousbusiness/gotosocial/pulls/2863 ", + }, + }, + { + status: >smodel.Status{ + ContentWarning: "Nerd stuff", + Content: `

Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg, both running on Wazero, and configured with a 50MiB db cache target. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.

`, + Attachments: []*gtsmodel.MediaAttachment{ + { + Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`, + }, + { + Description: `Another media attachment`, + }, + }, + Poll: >smodel.Poll{ + Options: []string{ + "Poll option 1", + "Poll option 2", + }, + }, + }, + expectedFields: []string{ + "Nerd stuff", + "Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg , both running on Wazero , and configured with a 50MiB db cache target . This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.", + "Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.", + "Another media attachment", + "Poll option 1", + "Poll option 2", + }, + }, + } { + fields := getFilterableFields(testcase.status) + assert.Equal(t, testcase.expectedFields, fields) + } +} diff --git a/internal/processing/account/bookmarks.go b/internal/processing/account/bookmarks.go index e6f0886f9..7a0ff9915 100644 --- a/internal/processing/account/bookmarks.go +++ b/internal/processing/account/bookmarks.go @@ -74,11 +74,12 @@ func (p *Processor) BookmarksGet(ctx context.Context, requestingAccount *gtsmode } // Convert the status. - item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, gtsmodel.FilterContextNone, nil) + item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, gtsmodel.FilterContextNone) if err != nil { log.Errorf(ctx, "error converting bookmarked status to api: %s", err) continue } + items = append(items, item) } diff --git a/internal/processing/account/statuses.go b/internal/processing/account/statuses.go index 3b56750c5..f0024d489 100644 --- a/internal/processing/account/statuses.go +++ b/internal/processing/account/statuses.go @@ -96,15 +96,9 @@ func (p *Processor) StatusesGet( return nil, gtserror.NewErrorInternalError(err) } - filters, err := p.state.DB.GetFiltersByAccountID(ctx, requestingAccount.ID) - if err != nil { - err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err) - return nil, gtserror.NewErrorInternalError(err) - } - for _, s := range filtered { // Convert filtered statuses to API statuses. - item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount, gtsmodel.FilterContextAccount, filters) + item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount, gtsmodel.FilterContextAccount) if err != nil { log.Errorf(ctx, "error convering to api status: %v", err) continue diff --git a/internal/processing/common/status.go b/internal/processing/common/status.go index 83acddc84..f5f230e98 100644 --- a/internal/processing/common/status.go +++ b/internal/processing/common/status.go @@ -24,10 +24,10 @@ import ( apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" "code.superseriousbusiness.org/gotosocial/internal/db" "code.superseriousbusiness.org/gotosocial/internal/federation/dereferencing" - statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/log" + "code.superseriousbusiness.org/gotosocial/internal/typeutils" ) // GetOwnStatus fetches the given status with ID, @@ -214,7 +214,6 @@ func (p *Processor) GetAPIStatus( target, requester, gtsmodel.FilterContextNone, - nil, ) if err != nil { err := gtserror.Newf("error converting: %w", err) @@ -235,7 +234,6 @@ func (p *Processor) GetVisibleAPIStatuses( requester *gtsmodel.Account, statuses []*gtsmodel.Status, filterCtx gtsmodel.FilterContext, - filters []*gtsmodel.Filter, ) []apimodel.Status { // Start new log entry with @@ -278,9 +276,8 @@ func (p *Processor) GetVisibleAPIStatuses( status, requester, filterCtx, - filters, ) - if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { + if err != nil && !errors.Is(err, typeutils.ErrHideStatus) { l.Errorf("error converting: %v", err) continue } diff --git a/internal/processing/conversations/update.go b/internal/processing/conversations/update.go index e4024a24a..cf81d6906 100644 --- a/internal/processing/conversations/update.go +++ b/internal/processing/conversations/update.go @@ -23,11 +23,11 @@ import ( apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" "code.superseriousbusiness.org/gotosocial/internal/db" - statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/id" "code.superseriousbusiness.org/gotosocial/internal/log" + "code.superseriousbusiness.org/gotosocial/internal/typeutils" "code.superseriousbusiness.org/gotosocial/internal/util" ) @@ -167,7 +167,7 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt if err != nil { // If the conversation's last status matched a hide filter, skip it. // If there was another kind of error, log that and skip it anyway. - if !errors.Is(err, statusfilter.ErrHideStatus) { + if !errors.Is(err, typeutils.ErrHideStatus) { log.Errorf(ctx, "error converting conversation %s to API representation for account %s: %v", status.ID, diff --git a/internal/processing/filters/common/common.go b/internal/processing/filters/common/common.go index a119d3bd4..8930b3aaf 100644 --- a/internal/processing/filters/common/common.go +++ b/internal/processing/filters/common/common.go @@ -28,12 +28,19 @@ import ( "code.superseriousbusiness.org/gotosocial/internal/gtscontext" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" + "code.superseriousbusiness.org/gotosocial/internal/log" + "code.superseriousbusiness.org/gotosocial/internal/processing/stream" "code.superseriousbusiness.org/gotosocial/internal/state" ) -type Processor struct{ state *state.State } +type Processor struct { + state *state.State + stream *stream.Processor +} -func New(state *state.State) *Processor { return &Processor{state} } +func New(state *state.State, stream *stream.Processor) *Processor { + return &Processor{state, stream} +} // CheckFilterExists calls .GetFilter() with a barebones context to not // fetch any sub-models, and not returning the result. this functionally @@ -160,6 +167,27 @@ func (p *Processor) GetFilterKeyword( return keyword, filter, nil } +// OnFilterChanged ... +func (p *Processor) OnFilterChanged(ctx context.Context, requester *gtsmodel.Account) { + + // Get list of list IDs created by this requesting account. + listIDs, err := p.state.DB.GetListIDsByAccountID(ctx, requester.ID) + if err != nil { + log.Errorf(ctx, "error getting account '%s' lists: %v", requester.Username, err) + } + + // Unprepare this requester's home timeline. + p.state.Caches.Timelines.Home.Unprepare(requester.ID) + + // Unprepare list timelines. + for _, id := range listIDs { + p.state.Caches.Timelines.List.Unprepare(id) + } + + // Send filter changed event for account. + p.stream.FiltersChanged(ctx, requester) +} + // FromAPIContexts converts a slice of frontend API model FilterContext types to our internal FilterContexts bit field. func FromAPIContexts(apiContexts []apimodel.FilterContext) (gtsmodel.FilterContexts, gtserror.WithCode) { var contexts gtsmodel.FilterContexts diff --git a/internal/processing/filters/v1/create.go b/internal/processing/filters/v1/create.go index b2ec69442..9f3fc17e0 100644 --- a/internal/processing/filters/v1/create.go +++ b/internal/processing/filters/v1/create.go @@ -91,8 +91,8 @@ func (p *Processor) Create(ctx context.Context, requester *gtsmodel.Account, for return nil, gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) // Return as converted frontend filter keyword model. return typeutils.FilterKeywordToAPIFilterV1(filter, filterKeyword), nil diff --git a/internal/processing/filters/v1/delete.go b/internal/processing/filters/v1/delete.go index cab8b185d..65768140a 100644 --- a/internal/processing/filters/v1/delete.go +++ b/internal/processing/filters/v1/delete.go @@ -63,8 +63,8 @@ func (p *Processor) Delete( } } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) return nil } diff --git a/internal/processing/filters/v1/filters.go b/internal/processing/filters/v1/filters.go index bcbbd70c0..4492b4e76 100644 --- a/internal/processing/filters/v1/filters.go +++ b/internal/processing/filters/v1/filters.go @@ -19,7 +19,6 @@ package v1 import ( "code.superseriousbusiness.org/gotosocial/internal/processing/filters/common" - "code.superseriousbusiness.org/gotosocial/internal/processing/stream" "code.superseriousbusiness.org/gotosocial/internal/state" "code.superseriousbusiness.org/gotosocial/internal/typeutils" ) @@ -30,15 +29,13 @@ type Processor struct { state *state.State converter *typeutils.Converter - stream *stream.Processor } -func New(state *state.State, converter *typeutils.Converter, common *common.Processor, stream *stream.Processor) Processor { +func New(state *state.State, converter *typeutils.Converter, common *common.Processor) Processor { return Processor{ c: common, state: state, converter: converter, - stream: stream, } } diff --git a/internal/processing/filters/v1/update.go b/internal/processing/filters/v1/update.go index 7e25e6fde..19699f328 100644 --- a/internal/processing/filters/v1/update.go +++ b/internal/processing/filters/v1/update.go @@ -166,8 +166,8 @@ func (p *Processor) Update( return nil, gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) // Return as converted frontend filter keyword model. return typeutils.FilterKeywordToAPIFilterV1(filter, filterKeyword), nil diff --git a/internal/processing/filters/v2/create.go b/internal/processing/filters/v2/create.go index d77c23424..154d80ee1 100644 --- a/internal/processing/filters/v2/create.go +++ b/internal/processing/filters/v2/create.go @@ -34,13 +34,13 @@ import ( // Create a new filter for the given account, using the provided parameters. // These params should have already been validated by the time they reach this function. -func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.FilterCreateRequestV2) (*apimodel.FilterV2, gtserror.WithCode) { +func (p *Processor) Create(ctx context.Context, requester *gtsmodel.Account, form *apimodel.FilterCreateRequestV2) (*apimodel.FilterV2, gtserror.WithCode) { var errWithCode gtserror.WithCode // Create new filter model. filter := >smodel.Filter{ ID: id.NewULID(), - AccountID: account.ID, + AccountID: requester.ID, Title: form.Title, } @@ -104,8 +104,8 @@ func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, form return nil, gtserror.NewErrorInternalError(err) } - // Send a filters changed event. - p.stream.FiltersChanged(ctx, account) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) // Return as converted frontend filter model. return typeutils.FilterToAPIFilterV2(filter), nil diff --git a/internal/processing/filters/v2/delete.go b/internal/processing/filters/v2/delete.go index ca3ade431..fdd6cca92 100644 --- a/internal/processing/filters/v2/delete.go +++ b/internal/processing/filters/v2/delete.go @@ -44,8 +44,8 @@ func (p *Processor) Delete( return gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) return nil } diff --git a/internal/processing/filters/v2/filters.go b/internal/processing/filters/v2/filters.go index 8c0ade1ca..08725ccde 100644 --- a/internal/processing/filters/v2/filters.go +++ b/internal/processing/filters/v2/filters.go @@ -19,7 +19,6 @@ package v2 import ( "code.superseriousbusiness.org/gotosocial/internal/processing/filters/common" - "code.superseriousbusiness.org/gotosocial/internal/processing/stream" "code.superseriousbusiness.org/gotosocial/internal/state" "code.superseriousbusiness.org/gotosocial/internal/typeutils" ) @@ -30,15 +29,13 @@ type Processor struct { state *state.State converter *typeutils.Converter - stream *stream.Processor } -func New(state *state.State, converter *typeutils.Converter, common *common.Processor, stream *stream.Processor) Processor { +func New(state *state.State, converter *typeutils.Converter, common *common.Processor) Processor { return Processor{ c: common, state: state, converter: converter, - stream: stream, } } diff --git a/internal/processing/filters/v2/keywordcreate.go b/internal/processing/filters/v2/keywordcreate.go index da91d5fd3..7ad7c3bd9 100644 --- a/internal/processing/filters/v2/keywordcreate.go +++ b/internal/processing/filters/v2/keywordcreate.go @@ -72,8 +72,8 @@ func (p *Processor) KeywordCreate(ctx context.Context, requester *gtsmodel.Accou return nil, gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) return typeutils.FilterKeywordToAPIFilterKeyword(filterKeyword), nil } diff --git a/internal/processing/filters/v2/keyworddelete.go b/internal/processing/filters/v2/keyworddelete.go index a0ec887e3..5393ffd53 100644 --- a/internal/processing/filters/v2/keyworddelete.go +++ b/internal/processing/filters/v2/keyworddelete.go @@ -54,8 +54,8 @@ func (p *Processor) KeywordDelete( return gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) return nil } diff --git a/internal/processing/filters/v2/keywordupdate.go b/internal/processing/filters/v2/keywordupdate.go index 9d1e5bd0c..047b079db 100644 --- a/internal/processing/filters/v2/keywordupdate.go +++ b/internal/processing/filters/v2/keywordupdate.go @@ -63,8 +63,8 @@ func (p *Processor) KeywordUpdate( return nil, gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) return typeutils.FilterKeywordToAPIFilterKeyword(filterKeyword), nil } diff --git a/internal/processing/filters/v2/statuscreate.go b/internal/processing/filters/v2/statuscreate.go index 1acab448c..2a3c3d74b 100644 --- a/internal/processing/filters/v2/statuscreate.go +++ b/internal/processing/filters/v2/statuscreate.go @@ -71,8 +71,8 @@ func (p *Processor) StatusCreate(ctx context.Context, requester *gtsmodel.Accoun return nil, gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) return typeutils.FilterStatusToAPIFilterStatus(filterStatus), nil } diff --git a/internal/processing/filters/v2/statusdelete.go b/internal/processing/filters/v2/statusdelete.go index 4309bac1a..321dc88e9 100644 --- a/internal/processing/filters/v2/statusdelete.go +++ b/internal/processing/filters/v2/statusdelete.go @@ -54,8 +54,8 @@ func (p *Processor) StatusDelete( return gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) return nil } diff --git a/internal/processing/filters/v2/update.go b/internal/processing/filters/v2/update.go index 96a43612f..f55f99bd5 100644 --- a/internal/processing/filters/v2/update.go +++ b/internal/processing/filters/v2/update.go @@ -135,8 +135,8 @@ func (p *Processor) Update( return nil, gtserror.NewErrorInternalError(err) } - // Stream a filters changed event to WS. - p.stream.FiltersChanged(ctx, requester) + // Handle filter change side-effects. + p.c.OnFilterChanged(ctx, requester) // Return as converted frontend filter model. return typeutils.FilterToAPIFilterV2(filter), nil diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 22574f1d7..c35c807e0 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -225,7 +225,7 @@ func NewProcessor( processor.account = account.New(&common, state, converter, mediaManager, federator, visFilter, parseMentionFunc) processor.media = media.New(&common, state, converter, federator, mediaManager, federator.TransportController()) processor.stream = stream.New(state, oauthServer) - filterCommon := filterCommon.New(state) + filterCommon := filterCommon.New(state, &processor.stream) // Instantiate the rest of the sub // processors + pin them to this struct. @@ -234,8 +234,8 @@ func NewProcessor( processor.application = application.New(state, converter) processor.conversations = conversations.New(state, converter, visFilter, muteFilter) processor.fedi = fedi.New(state, &common, converter, federator, visFilter) - processor.filtersv1 = filtersv1.New(state, converter, filterCommon, &processor.stream) - processor.filtersv2 = filtersv2.New(state, converter, filterCommon, &processor.stream) + processor.filtersv1 = filtersv1.New(state, converter, filterCommon) + processor.filtersv2 = filtersv2.New(state, converter, filterCommon) processor.interactionRequests = interactionrequests.New(&common, state, converter) processor.list = list.New(state, converter) processor.markers = markers.New(state, converter) diff --git a/internal/processing/search/util.go b/internal/processing/search/util.go index b4568722d..fc105940f 100644 --- a/internal/processing/search/util.go +++ b/internal/processing/search/util.go @@ -113,7 +113,7 @@ func (p *Processor) packageStatuses( continue } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, gtsmodel.FilterContextNone, nil) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, gtsmodel.FilterContextNone) if err != nil { log.Debugf(ctx, "skipping status %s because it couldn't be converted to its api representation: %s", status.ID, err) continue diff --git a/internal/processing/status/context.go b/internal/processing/status/context.go index 531dff1d6..f153b2e3a 100644 --- a/internal/processing/status/context.go +++ b/internal/processing/status/context.go @@ -275,20 +275,6 @@ func (p *Processor) ContextGet( requester *gtsmodel.Account, targetStatusID string, ) (*apimodel.ThreadContext, gtserror.WithCode) { - // Retrieve filters as they affect - // what should be shown to requester. - filters, err := p.state.DB.GetFiltersByAccountID( - ctx, // Populate filters. - requester.ID, - ) - if err != nil { - err = gtserror.Newf( - "couldn't retrieve filters for account %s: %w", - requester.ID, err, - ) - return nil, gtserror.NewErrorInternalError(err) - } - // Retrieve the full thread context. threadContext, errWithCode := p.contextGet(ctx, requester, @@ -305,7 +291,6 @@ func (p *Processor) ContextGet( requester, threadContext.ancestors, gtsmodel.FilterContextThread, - filters, ) // Convert and filter the thread context descendants @@ -313,7 +298,6 @@ func (p *Processor) ContextGet( requester, threadContext.descendants, gtsmodel.FilterContextThread, - filters, ) return &apiContext, nil diff --git a/internal/processing/stream/statusupdate_test.go b/internal/processing/stream/statusupdate_test.go index 74e7a4933..a3ec0415e 100644 --- a/internal/processing/stream/statusupdate_test.go +++ b/internal/processing/stream/statusupdate_test.go @@ -39,7 +39,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() { suite.NoError(errWithCode) editedStatus := suite.testStatuses["remote_account_1_status_1"] - apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(suite.T().Context(), editedStatus, account, gtsmodel.FilterContextNotifications, nil) + apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(suite.T().Context(), editedStatus, account, gtsmodel.FilterContextNotifications) suite.NoError(err) suite.streamProcessor.StatusUpdate(suite.T().Context(), account, apiStatus, stream.TimelineHome) diff --git a/internal/processing/timeline/faved.go b/internal/processing/timeline/faved.go index c1b44fa92..65b23c702 100644 --- a/internal/processing/timeline/faved.go +++ b/internal/processing/timeline/faved.go @@ -56,7 +56,7 @@ func (p *Processor) FavedTimelineGet(ctx context.Context, authed *apiutil.Auth, continue } - apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account, gtsmodel.FilterContextNone, nil) + apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account, gtsmodel.FilterContextNone) if err != nil { log.Errorf(ctx, "error convering to api status: %v", err) continue diff --git a/internal/processing/timeline/notification.go b/internal/processing/timeline/notification.go index 143145bb9..784b2b824 100644 --- a/internal/processing/timeline/notification.go +++ b/internal/processing/timeline/notification.go @@ -27,11 +27,11 @@ import ( apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util" "code.superseriousbusiness.org/gotosocial/internal/db" - "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/log" "code.superseriousbusiness.org/gotosocial/internal/paging" + "code.superseriousbusiness.org/gotosocial/internal/typeutils" "code.superseriousbusiness.org/gotosocial/internal/util" ) @@ -59,12 +59,6 @@ func (p *Processor) NotificationsGet( return util.EmptyPageableResponse(), nil } - filters, err := p.state.DB.GetFiltersByAccountID(ctx, requester.ID) - if err != nil { - err = gtserror.Newf("error getting account %s filters: %w", requester.ID, err) - return nil, gtserror.NewErrorInternalError(err) - } - var ( items = make([]interface{}, 0, count) @@ -115,9 +109,9 @@ func (p *Processor) NotificationsGet( } } - item, err := p.converter.NotificationToAPINotification(ctx, n, filters) + item, err := p.converter.NotificationToAPINotification(ctx, n, true) if err != nil { - if !errors.Is(err, status.ErrHideStatus) { + if !errors.Is(err, typeutils.ErrHideStatus) { log.Debugf(ctx, "skipping notification %s because it couldn't be converted to its api representation: %s", n.ID, err) } continue @@ -160,7 +154,7 @@ func (p *Processor) NotificationGet(ctx context.Context, account *gtsmodel.Accou // or mute checking for a notification directly // fetched by ID. only from timelines etc. - apiNotif, err := p.converter.NotificationToAPINotification(ctx, notif, nil) + apiNotif, err := p.converter.NotificationToAPINotification(ctx, notif, false) if err != nil { err := gtserror.Newf("error converting to api model: %w", err) return nil, gtserror.WrapWithCode(http.StatusInternalServerError, err) diff --git a/internal/processing/timeline/timeline.go b/internal/processing/timeline/timeline.go index a37785879..64d33e430 100644 --- a/internal/processing/timeline/timeline.go +++ b/internal/processing/timeline/timeline.go @@ -25,9 +25,7 @@ import ( apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" timelinepkg "code.superseriousbusiness.org/gotosocial/internal/cache/timeline" - "code.superseriousbusiness.org/gotosocial/internal/db" "code.superseriousbusiness.org/gotosocial/internal/filter/mutes" - statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/filter/visibility" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" @@ -79,18 +77,6 @@ func (p *Processor) getStatusTimeline( gtserror.WithCode, ) { var err error - var filters []*gtsmodel.Filter - - if requester != nil { - // Fetch all filters relevant for requesting account. - filters, err = p.state.DB.GetFiltersByAccountID(ctx, - requester.ID, - ) - if err != nil && !errors.Is(err, db.ErrNoEntries) { - err := gtserror.Newf("error getting account filters: %w", err) - return nil, gtserror.NewErrorInternalError(err) - } - } // Ensure we have valid // input paging cursor. @@ -135,9 +121,8 @@ func (p *Processor) getStatusTimeline( status, requester, filterCtx, - filters, ) - if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { + if err != nil && !errors.Is(err, typeutils.ErrHideStatus) { return nil, err } return apiStatus, nil diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go index 1c30c11be..4453095fd 100644 --- a/internal/processing/workers/fromclientapi_test.go +++ b/internal/processing/workers/fromclientapi_test.go @@ -213,7 +213,6 @@ func (suite *FromClientAPITestSuite) statusJSON( status, requestingAccount, gtsmodel.FilterContextNone, - nil, ) if err != nil { suite.FailNow(err.Error()) @@ -345,7 +344,7 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() { suite.FailNow("timed out waiting for new status notification") } - apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, nil) + apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, false) if err != nil { suite.FailNow(err.Error()) } @@ -2032,7 +2031,7 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithAuthorOnExclusiv suite.FailNow("timed out waiting for new status notification") } - apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, nil) + apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, false) if err != nil { suite.FailNow(err.Error()) } @@ -2217,7 +2216,7 @@ func (suite *FromClientAPITestSuite) TestProcessUpdateStatusInteractedWith() { suite.FailNow("timed out waiting for edited status notification") } - apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, nil) + apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, false) if err != nil { suite.FailNow(err.Error()) } diff --git a/internal/processing/workers/surfacenotify.go b/internal/processing/workers/surfacenotify.go index b11fb103e..de7e3d95a 100644 --- a/internal/processing/workers/surfacenotify.go +++ b/internal/processing/workers/surfacenotify.go @@ -23,11 +23,11 @@ import ( "strings" "code.superseriousbusiness.org/gotosocial/internal/db" - statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/gtscontext" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/id" + "code.superseriousbusiness.org/gotosocial/internal/typeutils" "code.superseriousbusiness.org/gotosocial/internal/util" "code.superseriousbusiness.org/gotosocial/internal/util/xslices" ) @@ -743,14 +743,9 @@ func (s *Surface) Notify( } } - filters, err := s.State.DB.GetFiltersByAccountID(ctx, targetAccount.ID) - if err != nil { - return gtserror.Newf("couldn't retrieve filters for account %s: %w", targetAccount.ID, err) - } - - // Convert the notification to frontend API model for streaming / push. - apiNotif, err := s.Converter.NotificationToAPINotification(ctx, notif, filters) - if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { + // Convert the notification to frontend API model for streaming / web push. + apiNotif, err := s.Converter.NotificationToAPINotification(ctx, notif, true) + if err != nil && !errors.Is(err, typeutils.ErrHideStatus) { return gtserror.Newf("error converting notification to api representation: %w", err) } diff --git a/internal/processing/workers/surfacetimeline.go b/internal/processing/workers/surfacetimeline.go index 7f9bcd596..5e677c626 100644 --- a/internal/processing/workers/surfacetimeline.go +++ b/internal/processing/workers/surfacetimeline.go @@ -22,12 +22,12 @@ import ( "errors" "code.superseriousbusiness.org/gotosocial/internal/cache/timeline" - statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/gtscontext" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/log" "code.superseriousbusiness.org/gotosocial/internal/stream" + "code.superseriousbusiness.org/gotosocial/internal/typeutils" "code.superseriousbusiness.org/gotosocial/internal/util" ) @@ -147,19 +147,10 @@ func (s *Surface) timelineAndNotifyStatusForFollowers( continue } - // Get relevant filters for this follow's account. - // (note the origin account of the follow is receiver of status). - filters, err := s.getFilters(ctx, follow.AccountID) - if err != nil { - log.Error(ctx, err) - continue - } - // Add status to any relevant lists for this follow, if applicable. listTimelined, exclusive, err := s.listTimelineStatusForFollow(ctx, status, follow, - filters, ) if err != nil { log.Errorf(ctx, "error list timelining status: %v", err) @@ -181,7 +172,6 @@ func (s *Surface) timelineAndNotifyStatusForFollowers( status, stream.TimelineHome, gtsmodel.FilterContextHome, - filters, ); homeTimelined { // If hometimelined, add to list of returned account IDs. @@ -239,7 +229,6 @@ func (s *Surface) listTimelineStatusForFollow( ctx context.Context, status *gtsmodel.Status, follow *gtsmodel.Follow, - filters []*gtsmodel.Filter, ) (timelined bool, exclusive bool, err error) { // Get all lists that contain this given follow. @@ -276,7 +265,6 @@ func (s *Surface) listTimelineStatusForFollow( status, stream.TimelineList+":"+list.ID, // key streamType to this specific list gtsmodel.FilterContextHome, - filters, ) // Update flag based on if timelined. @@ -286,15 +274,6 @@ func (s *Surface) listTimelineStatusForFollow( return timelined, exclusive, nil } -// getFiltersAndMutes returns an account's filters and mutes. -func (s *Surface) getFilters(ctx context.Context, accountID string) ([]*gtsmodel.Filter, error) { - filters, err := s.State.DB.GetFiltersByAccountID(ctx, accountID) - if err != nil { - return nil, gtserror.Newf("couldn't retrieve filters for account %s: %w", accountID, err) - } - return filters, err -} - // listEligible checks if the given status is eligible // for inclusion in the list that that the given listEntry // belongs to, based on the replies policy of the list. @@ -370,7 +349,6 @@ func (s *Surface) timelineStatus( status *gtsmodel.Status, streamType string, filterCtx gtsmodel.FilterContext, - filters []*gtsmodel.Filter, ) bool { // Attempt to convert status to frontend API representation, @@ -379,9 +357,8 @@ func (s *Surface) timelineStatus( status, account, filterCtx, - filters, ) - if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { + if err != nil && !errors.Is(err, typeutils.ErrHideStatus) { log.Error(ctx, "error converting status %s to frontend: %v", status.URI, err) } @@ -425,19 +402,12 @@ func (s *Surface) timelineAndNotifyStatusForTagFollowers( // Insert the status into the home timeline of each tag follower. errs := gtserror.MultiError{} for _, tagFollowerAccount := range tagFollowerAccounts { - filters, err := s.getFilters(ctx, tagFollowerAccount.ID) - if err != nil { - errs.Append(err) - continue - } - _ = s.timelineStatus(ctx, s.State.Caches.Timelines.Home.MustGet(tagFollowerAccount.ID), tagFollowerAccount, status, stream.TimelineHome, gtsmodel.FilterContextHome, - filters, ) } @@ -605,19 +575,10 @@ func (s *Surface) timelineStatusUpdateForFollowers( continue } - // Get relevant filters and mutes for this follow's account. - // (note the origin account of the follow is receiver of status). - filters, err := s.getFilters(ctx, follow.AccountID) - if err != nil { - log.Error(ctx, err) - continue - } - // Add status to relevant lists for this follow, if applicable. _, exclusive, err := s.listTimelineStatusUpdateForFollow(ctx, status, follow, - filters, ) if err != nil { log.Errorf(ctx, "error list timelining status: %v", err) @@ -637,7 +598,6 @@ func (s *Surface) timelineStatusUpdateForFollowers( follow.Account, status, stream.TimelineHome, - filters, ) if err != nil { log.Errorf(ctx, "error home timelining status: %v", err) @@ -662,7 +622,6 @@ func (s *Surface) listTimelineStatusUpdateForFollow( ctx context.Context, status *gtsmodel.Status, follow *gtsmodel.Follow, - filters []*gtsmodel.Filter, ) (bool, bool, error) { // Get all lists that contain this given follow. @@ -701,7 +660,6 @@ func (s *Surface) listTimelineStatusUpdateForFollow( follow.Account, status, stream.TimelineList+":"+list.ID, // key streamType to this specific list - filters, ) if err != nil { log.Errorf(ctx, "error adding status to list timeline: %v", err) @@ -724,7 +682,6 @@ func (s *Surface) timelineStreamStatusUpdate( account *gtsmodel.Account, status *gtsmodel.Status, streamType string, - filters []*gtsmodel.Filter, ) (bool, error) { // Convert updated database model to frontend model. @@ -732,14 +689,13 @@ func (s *Surface) timelineStreamStatusUpdate( status, account, gtsmodel.FilterContextHome, - filters, ) switch { case err == nil: // no issue. - case errors.Is(err, statusfilter.ErrHideStatus): + case errors.Is(err, typeutils.ErrHideStatus): // Don't put this status in the stream. return false, nil @@ -774,18 +730,11 @@ func (s *Surface) timelineStatusUpdateForTagFollowers( // Stream the update to the home timeline of each tag follower. errs := gtserror.MultiError{} for _, tagFollowerAccount := range tagFollowerAccounts { - filters, err := s.getFilters(ctx, tagFollowerAccount.ID) - if err != nil { - errs.Append(err) - continue - } - if _, err := s.timelineStreamStatusUpdate( ctx, tagFollowerAccount, status, stream.TimelineHome, - filters, ); err != nil { errs.Appendf( "error updating status %s on home timeline for account %s: %w", diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 789404426..4f3658b0d 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -27,6 +27,7 @@ import ( apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" "code.superseriousbusiness.org/gotosocial/internal/filter/interaction" + "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/filter/visibility" "code.superseriousbusiness.org/gotosocial/internal/log" "code.superseriousbusiness.org/gotosocial/internal/state" @@ -37,6 +38,7 @@ type Converter struct { defaultAvatars []string randAvatars sync.Map visFilter *visibility.Filter + statusFilter *status.Filter intFilter *interaction.Filter randStats atomic.Pointer[apimodel.RandomStats] } @@ -46,6 +48,7 @@ func NewConverter(state *state.State) *Converter { state: state, defaultAvatars: populateDefaultAvatars(), visFilter: visibility.NewFilter(state), + statusFilter: status.NewFilter(state), intFilter: interaction.NewFilter(state), } } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index ed8f3a4cd..a79387c0f 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -24,14 +24,12 @@ import ( "fmt" "math" "slices" - "strconv" "strings" "time" apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" "code.superseriousbusiness.org/gotosocial/internal/config" "code.superseriousbusiness.org/gotosocial/internal/db" - statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/id" @@ -53,6 +51,10 @@ const ( instanceMastodonVersion = "3.5.3" ) +// ErrHideStatus indicates that a status has +// been filtered and should not be returned at all. +var ErrHideStatus = errors.New("hide status") + var instanceStatusesSupportedMimeTypes = []string{ string(apimodel.StatusContentTypePlain), string(apimodel.StatusContentTypeMarkdown), @@ -849,14 +851,12 @@ func (c *Converter) StatusToAPIStatus( status *gtsmodel.Status, requestingAccount *gtsmodel.Account, filterCtx gtsmodel.FilterContext, - filters []*gtsmodel.Filter, ) (*apimodel.Status, error) { return c.statusToAPIStatus( ctx, status, requestingAccount, filterCtx, - filters, true, true, ) @@ -871,7 +871,6 @@ func (c *Converter) statusToAPIStatus( status *gtsmodel.Status, requestingAccount *gtsmodel.Account, filterCtx gtsmodel.FilterContext, - filters []*gtsmodel.Filter, placeholdAttachments bool, addPendingNote bool, ) (*apimodel.Status, error) { @@ -880,7 +879,6 @@ func (c *Converter) statusToAPIStatus( status, requestingAccount, // Can be nil. filterCtx, // Can be empty. - filters, ) if err != nil { return nil, err @@ -938,103 +936,6 @@ func (c *Converter) statusToAPIStatus( return apiStatus, nil } -// statusToAPIFilterResults applies filters and mutes to a status and returns an API filter result object. -// The result may be nil if no filters matched. -// If the status should not be returned at all, it returns the ErrHideStatus error. -func (c *Converter) statusToAPIFilterResults( - ctx context.Context, - s *gtsmodel.Status, - requestingAccount *gtsmodel.Account, - filterCtx gtsmodel.FilterContext, - filters []*gtsmodel.Filter, -) ([]apimodel.FilterResult, error) { - // If there are no filters or mutes, we're done. - // We never hide statuses authored by the requesting account, - // since not being able to see your own posts is confusing. - if filterCtx == 0 || (len(filters) == 0) || s.AccountID == requestingAccount.ID { - return nil, nil - } - - // Both mutes and - // filters can expire. - now := time.Now() - - // Key this status based on ID + last updated time, - // to ensure we always filter on latest version. - statusKey := s.ID + strconv.FormatInt(s.UpdatedAt().Unix(), 10) - - // Check if we have filterable fields cached for this status. - cache := c.state.Caches.StatusesFilterableFields - fields, stored := cache.Get(statusKey) - if !stored { - - // We don't have filterable fields - // cached, calculate + cache now. - fields = filterableFields(s) - cache.Set(statusKey, fields) - } - - // Record all matching warn filters and the reasons they matched. - filterResults := make([]apimodel.FilterResult, 0, len(filters)) - for _, filter := range filters { - if !filter.Contexts.Applies(filterCtx) { - // Filter doesn't apply - // to this context. - continue - } - - if filter.Expired(now) { - // Filter doesn't - // apply anymore. - continue - } - - // Assemble matching keywords (if any) from this filter. - keywordMatches := make([]string, 0, len(filter.Keywords)) - for _, keyword := range filter.Keywords { - // Check if at least one filterable field - // in the status matches on this filter. - if slices.ContainsFunc( - fields, - func(field string) bool { - return keyword.Regexp.MatchString(field) - }, - ) { - // At least one field matched on this filter. - keywordMatches = append(keywordMatches, keyword.Keyword) - } - } - - // A status has only one ID. Not clear - // why this is a list in the Mastodon API. - statusMatches := make([]string, 0, 1) - for _, filterStatus := range filter.Statuses { - if s.ID == filterStatus.StatusID { - statusMatches = append(statusMatches, filterStatus.StatusID) - break - } - } - - if len(keywordMatches) > 0 || len(statusMatches) > 0 { - switch filter.Action { - case gtsmodel.FilterActionWarn: - // Record what matched. - filterResults = append(filterResults, apimodel.FilterResult{ - Filter: *FilterToAPIFilterV2(filter), - KeywordMatches: keywordMatches, - StatusMatches: statusMatches, - }) - - case gtsmodel.FilterActionHide: - // Don't show this status. Immediate return. - return nil, statusfilter.ErrHideStatus - } - } - } - - return filterResults, nil -} - // StatusToWebStatus converts a gts model status into an // api representation suitable for serving into a web template. // @@ -1046,7 +947,6 @@ func (c *Converter) StatusToWebStatus( apiStatus, err := c.statusToFrontend(ctx, s, nil, // No authed requester. gtsmodel.FilterContextNone, // No filters. - nil, // No filters. ) if err != nil { return nil, err @@ -1216,7 +1116,6 @@ func (c *Converter) statusToFrontend( status *gtsmodel.Status, requestingAccount *gtsmodel.Account, filterCtx gtsmodel.FilterContext, - filters []*gtsmodel.Filter, ) ( *apimodel.Status, error, @@ -1225,7 +1124,6 @@ func (c *Converter) statusToFrontend( status, requestingAccount, filterCtx, - filters, ) if err != nil { return nil, err @@ -1236,9 +1134,8 @@ func (c *Converter) statusToFrontend( status.BoostOf, requestingAccount, filterCtx, - filters, ) - if errors.Is(err, statusfilter.ErrHideStatus) { + if errors.Is(err, ErrHideStatus) { // If we'd hide the original status, hide the boost. return nil, err } else if err != nil { @@ -1266,10 +1163,9 @@ func (c *Converter) statusToFrontend( // account to api/web model -- the caller must do that. func (c *Converter) baseStatusToFrontend( ctx context.Context, - s *gtsmodel.Status, - requestingAccount *gtsmodel.Account, + status *gtsmodel.Status, + requester *gtsmodel.Account, filterCtx gtsmodel.FilterContext, - filters []*gtsmodel.Filter, ) ( *apimodel.Status, error, @@ -1277,12 +1173,12 @@ func (c *Converter) baseStatusToFrontend( // Try to populate status struct pointer fields. // We can continue in many cases of partial failure, // but there are some fields we actually need. - if err := c.state.DB.PopulateStatus(ctx, s); err != nil { + if err := c.state.DB.PopulateStatus(ctx, status); err != nil { switch { - case s.Account == nil: + case status.Account == nil: return nil, gtserror.Newf("error(s) populating status, required account not set: %w", err) - case s.BoostOfID != "" && s.BoostOf == nil: + case status.BoostOfID != "" && status.BoostOf == nil: return nil, gtserror.Newf("error(s) populating status, required boost not set: %w", err) default: @@ -1290,37 +1186,37 @@ func (c *Converter) baseStatusToFrontend( } } - repliesCount, err := c.state.DB.CountStatusReplies(ctx, s.ID) + repliesCount, err := c.state.DB.CountStatusReplies(ctx, status.ID) if err != nil { return nil, gtserror.Newf("error counting replies: %w", err) } - reblogsCount, err := c.state.DB.CountStatusBoosts(ctx, s.ID) + reblogsCount, err := c.state.DB.CountStatusBoosts(ctx, status.ID) if err != nil { return nil, gtserror.Newf("error counting reblogs: %w", err) } - favesCount, err := c.state.DB.CountStatusFaves(ctx, s.ID) + favesCount, err := c.state.DB.CountStatusFaves(ctx, status.ID) if err != nil { return nil, gtserror.Newf("error counting faves: %w", err) } - apiAttachments, err := c.convertAttachmentsToAPIAttachments(ctx, s.Attachments, s.AttachmentIDs) + apiAttachments, err := c.convertAttachmentsToAPIAttachments(ctx, status.Attachments, status.AttachmentIDs) if err != nil { log.Errorf(ctx, "error converting status attachments: %v", err) } - apiMentions, err := c.convertMentionsToAPIMentions(ctx, s.Mentions, s.MentionIDs) + apiMentions, err := c.convertMentionsToAPIMentions(ctx, status.Mentions, status.MentionIDs) if err != nil { log.Errorf(ctx, "error converting status mentions: %v", err) } - apiTags, err := c.convertTagsToAPITags(ctx, s.Tags, s.TagIDs) + apiTags, err := c.convertTagsToAPITags(ctx, status.Tags, status.TagIDs) if err != nil { log.Errorf(ctx, "error converting status tags: %v", err) } - apiEmojis, err := c.convertEmojisToAPIEmojis(ctx, s.Emojis, s.EmojiIDs) + apiEmojis, err := c.convertEmojisToAPIEmojis(ctx, status.Emojis, status.EmojiIDs) if err != nil { log.Errorf(ctx, "error converting status emojis: %v", err) } @@ -1328,32 +1224,30 @@ func (c *Converter) baseStatusToFrontend( // Take status's interaction policy, or // fall back to default for its visibility. var p *gtsmodel.InteractionPolicy - if s.InteractionPolicy != nil { - p = s.InteractionPolicy - } else { - p = gtsmodel.DefaultInteractionPolicyFor(s.Visibility) + if p = status.InteractionPolicy; p == nil { + p = gtsmodel.DefaultInteractionPolicyFor(status.Visibility) } - apiInteractionPolicy, err := c.InteractionPolicyToAPIInteractionPolicy(ctx, p, s, requestingAccount) + apiInteractionPolicy, err := c.InteractionPolicyToAPIInteractionPolicy(ctx, p, status, requester) if err != nil { return nil, gtserror.Newf("error converting interaction policy: %w", err) } apiStatus := &apimodel.Status{ - ID: s.ID, - CreatedAt: util.FormatISO8601(s.CreatedAt), + ID: status.ID, + CreatedAt: util.FormatISO8601(status.CreatedAt), InReplyToID: nil, // Set below. InReplyToAccountID: nil, // Set below. - Sensitive: *s.Sensitive, - Visibility: VisToAPIVis(s.Visibility), - LocalOnly: s.IsLocalOnly(), + Sensitive: *status.Sensitive, + Visibility: VisToAPIVis(status.Visibility), + LocalOnly: status.IsLocalOnly(), Language: nil, // Set below. - URI: s.URI, - URL: s.URL, + URI: status.URI, + URL: status.URL, RepliesCount: repliesCount, ReblogsCount: reblogsCount, FavouritesCount: favesCount, - Content: s.Content, + Content: status.Content, Reblog: nil, // Set below. Application: nil, // Set below. Account: nil, // Caller must do this. @@ -1362,37 +1256,37 @@ func (c *Converter) baseStatusToFrontend( Tags: apiTags, Emojis: apiEmojis, Card: nil, // TODO: implement cards - Text: s.Text, - ContentType: ContentTypeToAPIContentType(s.ContentType), + Text: status.Text, + ContentType: ContentTypeToAPIContentType(status.ContentType), InteractionPolicy: *apiInteractionPolicy, // Mastodon API says spoiler_text should be *text*, not HTML, so // parse any HTML back to plaintext when serializing via the API, // attempting to preserve semantic intent to keep it readable. - SpoilerText: text.ParseHTMLToPlain(s.ContentWarning), + SpoilerText: text.ParseHTMLToPlain(status.ContentWarning), } - if at := s.EditedAt; !at.IsZero() { + if at := status.EditedAt; !at.IsZero() { timestamp := util.FormatISO8601(at) apiStatus.EditedAt = util.Ptr(timestamp) } - apiStatus.InReplyToID = util.PtrIf(s.InReplyToID) - apiStatus.InReplyToAccountID = util.PtrIf(s.InReplyToAccountID) - apiStatus.Language = util.PtrIf(s.Language) + apiStatus.InReplyToID = util.PtrIf(status.InReplyToID) + apiStatus.InReplyToAccountID = util.PtrIf(status.InReplyToAccountID) + apiStatus.Language = util.PtrIf(status.Language) switch { - case s.CreatedWithApplication != nil: + case status.CreatedWithApplication != nil: // App exists for this status and is set. - apiStatus.Application, err = c.AppToAPIAppPublic(ctx, s.CreatedWithApplication) + apiStatus.Application, err = c.AppToAPIAppPublic(ctx, status.CreatedWithApplication) if err != nil { return nil, gtserror.Newf( "error converting application %s: %w", - s.CreatedWithApplicationID, err, + status.CreatedWithApplicationID, err, ) } - case s.CreatedWithApplicationID != "": + case status.CreatedWithApplicationID != "": // App existed for this status but not // anymore, it's probably been cleaned up. // Set a dummy application. @@ -1405,13 +1299,13 @@ func (c *Converter) baseStatusToFrontend( // status, so nothing to do (app is optional). } - if s.Poll != nil { + if status.Poll != nil { // Set originating // status on the poll. - poll := s.Poll - poll.Status = s + poll := status.Poll + poll.Status = status - apiStatus.Poll, err = c.PollToAPIPoll(ctx, requestingAccount, poll) + apiStatus.Poll, err = c.PollToAPIPoll(ctx, requester, poll) if err != nil { return nil, fmt.Errorf("error converting poll: %w", err) } @@ -1419,15 +1313,15 @@ func (c *Converter) baseStatusToFrontend( // Status interactions. // - if s.BoostOf != nil { //nolint + if status.BoostOf != nil { //nolint // populated *outside* this // function to prevent recursion. } else { - interacts, err := c.interactionsWithStatusForAccount(ctx, s, requestingAccount) + interacts, err := c.interactionsWithStatusForAccount(ctx, status, requester) if err != nil { log.Errorf(ctx, "error getting interactions for status %s for account %s: %v", - s.ID, requestingAccount.ID, err, + status.URI, requester.URI, err, ) // Ensure non-nil object. @@ -1442,21 +1336,24 @@ func (c *Converter) baseStatusToFrontend( // If web URL is empty for whatever // reason, provide AP URI as fallback. - if s.URL == "" { - s.URL = s.URI + if apiStatus.URL == "" { + apiStatus.URL = apiStatus.URI } - // Apply filters. - filterResults, err := c.statusToAPIFilterResults(ctx, s, requestingAccount, filterCtx, filters) + var hide bool + + // Pass the status through any stored filters of requesting account's, in context. + apiStatus.Filtered, hide, err = c.statusFilter.StatusFilterResultsInContext(ctx, + requester, + status, + filterCtx, + ) if err != nil { - if errors.Is(err, statusfilter.ErrHideStatus) { - return nil, err - } - return nil, fmt.Errorf("error applying filters: %w", err) + return nil, gtserror.Newf("error filtering status %s: %w", status.URI, err) + } else if hide { + return nil, ErrHideStatus } - apiStatus.Filtered = filterResults - return apiStatus, nil } @@ -1968,30 +1865,35 @@ func (c *Converter) RelationshipToAPIRelationship(ctx context.Context, r *gtsmod // NotificationToAPINotification converts a gts notification into a api notification func (c *Converter) NotificationToAPINotification( ctx context.Context, - n *gtsmodel.Notification, - filters []*gtsmodel.Filter, + notif *gtsmodel.Notification, + filter bool, ) (*apimodel.Notification, error) { // Ensure notif populated. - if err := c.state.DB.PopulateNotification(ctx, n); err != nil { + if err := c.state.DB.PopulateNotification(ctx, notif); err != nil { return nil, gtserror.Newf("error populating notification: %w", err) } // Get account that triggered this notif. - apiAccount, err := c.AccountToAPIAccountPublic(ctx, n.OriginAccount) + apiAccount, err := c.AccountToAPIAccountPublic(ctx, notif.OriginAccount) if err != nil { return nil, gtserror.Newf("error converting account to api: %w", err) } // Get status that triggered this notif, if set. var apiStatus *apimodel.Status - if n.Status != nil { - apiStatus, err = c.StatusToAPIStatus( - ctx, n.Status, - n.TargetAccount, - gtsmodel.FilterContextNotifications, - filters, + if notif.Status != nil { + var filterCtx gtsmodel.FilterContext + + if filter { + filterCtx = gtsmodel.FilterContextNotifications + } + + apiStatus, err = c.StatusToAPIStatus(ctx, + notif.Status, + notif.TargetAccount, + filterCtx, ) - if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { + if err != nil && !errors.Is(err, ErrHideStatus) { return nil, gtserror.Newf("error converting status to api: %w", err) } @@ -2009,9 +1911,9 @@ func (c *Converter) NotificationToAPINotification( } return &apimodel.Notification{ - ID: n.ID, - Type: n.NotificationType.String(), - CreatedAt: util.FormatISO8601(n.CreatedAt), + ID: notif.ID, + Type: notif.NotificationType.String(), + CreatedAt: util.FormatISO8601(notif.CreatedAt), Account: apiAccount, Status: apiStatus, }, nil @@ -2040,9 +1942,8 @@ func (c *Converter) ConversationToAPIConversation( conversation.LastStatus, requester, gtsmodel.FilterContextNotifications, - filters, ) - if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) { + if err != nil && !errors.Is(err, ErrHideStatus) { return nil, gtserror.Newf( "error converting status %s to API representation: %w", conversation.LastStatus.ID, @@ -2309,7 +2210,6 @@ func (c *Converter) ReportToAdminAPIReport(ctx context.Context, r *gtsmodel.Repo s, requestingAccount, gtsmodel.FilterContextNone, - nil, // No filters. true, // Placehold unknown attachments. // Don't add note about @@ -3014,7 +2914,6 @@ func (c *Converter) InteractionReqToAPIInteractionReq( req.Status, requestingAcct, gtsmodel.FilterContextNone, - nil, // No filters. ) if err != nil { err := gtserror.Newf("error converting interacted status: %w", err) @@ -3028,7 +2927,6 @@ func (c *Converter) InteractionReqToAPIInteractionReq( req.Reply, requestingAcct, gtsmodel.FilterContextNone, - nil, // No filters. true, // Placehold unknown attachments. // Don't add note about pending; diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go index 1795180e9..1fc55acca 100644 --- a/internal/typeutils/internaltofrontend_test.go +++ b/internal/typeutils/internaltofrontend_test.go @@ -27,8 +27,9 @@ import ( apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model" "code.superseriousbusiness.org/gotosocial/internal/config" "code.superseriousbusiness.org/gotosocial/internal/db" - statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" + "code.superseriousbusiness.org/gotosocial/internal/id" + "code.superseriousbusiness.org/gotosocial/internal/typeutils" "code.superseriousbusiness.org/gotosocial/internal/util" "code.superseriousbusiness.org/gotosocial/testrig" "github.com/stretchr/testify/suite" @@ -465,7 +466,7 @@ func (suite *InternalToFrontendTestSuite) TestLocalInstanceAccountToFrontendBloc func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() { testStatus := suite.testStatuses["admin_account_status_1"] requestingAccount := suite.testAccounts["local_account_1"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -628,7 +629,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendHTMLContentWarning testStatus.ContentWarning = `

First paragraph of content warning

Here's the title!

Big boobs
Tee hee!

Some more text
And a bunch more

Hasta la victoria siempre!

` requestingAccount := suite.testAccounts["local_account_1"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -794,7 +795,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted } requestingAccount := suite.testAccounts["local_account_1"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(ctx, testStatus, requestingAccount, gtsmodel.FilterContextNone, nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(ctx, testStatus, requestingAccount, gtsmodel.FilterContextNone) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -952,6 +953,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted // Modify a fixture status into a status that should be filtered, // and then filter it, returning the API status or any error from converting it. func (suite *InternalToFrontendTestSuite) filteredStatusToFrontend(action gtsmodel.FilterAction, boost bool) (*apimodel.Status, error) { + ctx := suite.T().Context() + testStatus := suite.testStatuses["admin_account_status_1"] testStatus.Content += " fnord" testStatus.Text += " fnord" @@ -969,19 +972,14 @@ func (suite *InternalToFrontendTestSuite) filteredStatusToFrontend(action gtsmod expectedMatchingFilter := suite.testFilters["local_account_1_filter_1"] expectedMatchingFilter.Action = action - expectedMatchingFilterKeyword := suite.testFilterKeywords["local_account_1_filter_1_keyword_1"] - suite.NoError(expectedMatchingFilterKeyword.Compile()) - - expectedMatchingFilter.Keywords = []*gtsmodel.FilterKeyword{expectedMatchingFilterKeyword} - - requestingAccountFilters := []*gtsmodel.Filter{expectedMatchingFilter} + err := suite.state.DB.UpdateFilter(ctx, expectedMatchingFilter, "action") + suite.NoError(err) return suite.typeconverter.StatusToAPIStatus( suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextHome, - requestingAccountFilters, ) } @@ -1480,17 +1478,19 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() { // Test that a status which is filtered with a hide filter by the requesting user results in the ErrHideStatus error. func (suite *InternalToFrontendTestSuite) TestHideFilteredStatusToFrontend() { _, err := suite.filteredStatusToFrontend(gtsmodel.FilterActionHide, false) - suite.ErrorIs(err, statusfilter.ErrHideStatus) + suite.ErrorIs(err, typeutils.ErrHideStatus) } // Test that a status which is filtered with a hide filter by the requesting user results in the ErrHideStatus error for a boost of that status. func (suite *InternalToFrontendTestSuite) TestHideFilteredBoostToFrontend() { _, err := suite.filteredStatusToFrontend(gtsmodel.FilterActionHide, true) - suite.ErrorIs(err, statusfilter.ErrHideStatus) + suite.ErrorIs(err, typeutils.ErrHideStatus) } // Test that a hashtag filter for a hashtag in Mastodon HTML content works the way most users would expect. func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wholeWord bool, boost bool) { + ctx := suite.T().Context() + testStatus := new(gtsmodel.Status) *testStatus = *suite.testStatuses["admin_account_status_1"] testStatus.Content = `

doggo doggin' it

#dogsofmastodon

` @@ -1508,29 +1508,38 @@ func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wh testStatus = boost } + var err error + requestingAccount := suite.testAccounts["local_account_1"] - filterKeyword := >smodel.FilterKeyword{ - Keyword: "#dogsofmastodon", - WholeWord: &wholeWord, - Regexp: nil, - } - if err := filterKeyword.Compile(); err != nil { - suite.FailNow(err.Error()) + filter := >smodel.Filter{ + ID: id.NewULID(), + Title: id.NewULID(), + AccountID: requestingAccount.ID, + Action: gtsmodel.FilterActionWarn, + Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome), } - filter := >smodel.Filter{ - Action: gtsmodel.FilterActionWarn, - Keywords: []*gtsmodel.FilterKeyword{filterKeyword}, - Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome), + filterKeyword := >smodel.FilterKeyword{ + ID: id.NewULID(), + FilterID: filter.ID, + Keyword: "#dogsofmastodon", + WholeWord: &wholeWord, } + filter.KeywordIDs = []string{filterKeyword.ID} + + err = suite.state.DB.PutFilterKeyword(ctx, filterKeyword) + suite.NoError(err) + + err = suite.state.DB.PutFilter(ctx, filter) + suite.NoError(err) + apiStatus, err := suite.typeconverter.StatusToAPIStatus( suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextHome, - []*gtsmodel.Filter{filter}, ) if err != nil { suite.FailNow(err.Error()) @@ -1559,7 +1568,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments testStatus := suite.testStatuses["remote_account_2_status_1"] requestingAccount := suite.testAccounts["admin_account"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -1886,7 +1895,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() *testStatus = *suite.testStatuses["admin_account_status_1"] testStatus.Language = "" requestingAccount := suite.testAccounts["local_account_1"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -2047,7 +2056,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendPartialInteraction *testStatus = *suite.testStatuses["local_account_1_status_3"] testStatus.Language = "" requestingAccount := suite.testAccounts["admin_account"] - apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil) + apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone) suite.NoError(err) b, err := json.MarshalIndent(apiStatus, "", " ") @@ -2161,7 +2170,6 @@ func (suite *InternalToFrontendTestSuite) TestStatusToAPIStatusPendingApproval() testStatus, requestingAccount, gtsmodel.FilterContextNone, - nil, ) if err != nil { suite.FailNow(err.Error()) diff --git a/internal/typeutils/util.go b/internal/typeutils/util.go index 2a0293f65..ecd2cecb0 100644 --- a/internal/typeutils/util.go +++ b/internal/typeutils/util.go @@ -350,60 +350,3 @@ func ContentToContentLanguage( return contentStr, langTagStr } - -// filterableFields returns text fields from -// a status that we might want to filter on: -// -// - content warning -// - content (converted to plaintext from HTML) -// - media descriptions -// - poll options -// -// Each field should be filtered separately. -// This avoids scenarios where false-positive -// multiple-word matches can be made by matching -// the last word of one field + the first word -// of the next field together. -func filterableFields(s *gtsmodel.Status) []string { - // Estimate length of fields. - fieldCount := 2 + len(s.Attachments) - if s.Poll != nil { - fieldCount += len(s.Poll.Options) - } - fields := make([]string, 0, fieldCount) - - // Content warning / title. - if s.ContentWarning != "" { - fields = append(fields, s.ContentWarning) - } - - // Status content. Though we have raw text - // available for statuses created on our - // instance, use the plaintext version to - // remove markdown-formatting characters - // and ensure more consistent filtering. - if s.Content != "" { - text := text.ParseHTMLToPlain(s.Content) - if text != "" { - fields = append(fields, text) - } - } - - // Media descriptions. - for _, attachment := range s.Attachments { - if attachment.Description != "" { - fields = append(fields, attachment.Description) - } - } - - // Poll options. - if s.Poll != nil { - for _, opt := range s.Poll.Options { - if opt != "" { - fields = append(fields, opt) - } - } - } - - return fields -} diff --git a/internal/typeutils/util_test.go b/internal/typeutils/util_test.go index 42a86372f..7ebecd232 100644 --- a/internal/typeutils/util_test.go +++ b/internal/typeutils/util_test.go @@ -23,7 +23,6 @@ import ( "code.superseriousbusiness.org/gotosocial/internal/config" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/language" - "github.com/stretchr/testify/assert" ) func TestMisskeyReportContentURLs1(t *testing.T) { @@ -157,62 +156,3 @@ func TestContentToContentLanguage(t *testing.T) { } } } - -func TestFilterableText(t *testing.T) { - type testcase struct { - status *gtsmodel.Status - expectedFields []string - } - - for _, testcase := range []testcase{ - { - status: >smodel.Status{ - ContentWarning: "This is a test status", - Content: `

Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.

`, - }, - expectedFields: []string{ - "This is a test status", - "Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial instance.", - }, - }, - { - status: >smodel.Status{ - Content: `

@zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)

https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863

`, - }, - expectedFields: []string{ - "@zlatko currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)\n\nhttps://codeberg.org/superseriousbusiness/gotosocial/pulls/2863 ", - }, - }, - { - status: >smodel.Status{ - ContentWarning: "Nerd stuff", - Content: `

Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg, both running on Wazero, and configured with a 50MiB db cache target. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.

`, - Attachments: []*gtsmodel.MediaAttachment{ - { - Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`, - }, - { - Description: `Another media attachment`, - }, - }, - Poll: >smodel.Poll{ - Options: []string{ - "Poll option 1", - "Poll option 2", - }, - }, - }, - expectedFields: []string{ - "Nerd stuff", - "Latest graphs for #GoToSocial on Wasm sqlite3 with embedded Wasm ffmpeg , both running on Wazero , and configured with a 50MiB db cache target . This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.", - "Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.", - "Another media attachment", - "Poll option 1", - "Poll option 2", - }, - }, - } { - fields := filterableFields(testcase.status) - assert.Equal(t, testcase.expectedFields, fields) - } -} diff --git a/internal/webpush/realsender_test.go b/internal/webpush/realsender_test.go index a404c166f..e11067e1d 100644 --- a/internal/webpush/realsender_test.go +++ b/internal/webpush/realsender_test.go @@ -190,7 +190,7 @@ func (suite *RealSenderStandardTestSuite) simulatePushNotification( }, nil } - apiNotif, err := suite.typeconverter.NotificationToAPINotification(ctx, notification, nil) + apiNotif, err := suite.typeconverter.NotificationToAPINotification(ctx, notification, false) suite.NoError(err) // Send the push notification. diff --git a/test/envparsing.sh b/test/envparsing.sh index 0c737c1d9..a6247ece5 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -74,6 +74,7 @@ EXPECT=$(cat << "EOF" "cache-status-edit-mem-ratio": 2, "cache-status-fave-ids-mem-ratio": 3, "cache-status-fave-mem-ratio": 2, + "cache-status-filter-mem-ratio": 7, "cache-status-mem-ratio": 5, "cache-tag-mem-ratio": 2, "cache-thread-mute-mem-ratio": 0.2,