mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-28 20:02:24 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[performance] add statusfilter cache to cache calculated status filtering results (#4303)

Browse source 
this adds another 'filter' type cache, similar to the visibility and mute caches, to cache the results of status filtering checks. for the moment this keeps all the check calls themselves within the frontend typeconversion code, but i may move this out of the typeconverter in a future PR (also removing the ErrHideStatus means of propagating a hidden status).

also tweaks some of the cache invalidation hooks to not make unnecessary calls.

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4303
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2025-07-01 16:00:04 +02:00 • committed by kim
commit 4f2aa792b3
50 changed files with 1017 additions and 544 deletions
Download patch file Download diff file Expand all files Collapse all files

38
internal/cache/cache.go vendored
View file

@ -41,21 +41,22 @@ type Caches struct {
// the block []headerfilter.Filter cache.
BlockHeaderFilters headerfilter.Cache
// TTL cache of statuses -> filterable text fields.
// To ensure up-to-date fields, cache is keyed as:
// `[status.ID][status.UpdatedAt.Unix()]`
StatusesFilterableFields *ttl.Cache[string, []string]
// Timelines ...
// Timelines provides access to the
// collection of timeline object caches,
// used in timeline lookups and streaming.
Timelines TimelineCaches
// Mutes provides access to the item mutes
// cache. (used by the item mutes filter).
Mutes MutesCache
Mutes StructCache[*CachedMute]
// StatusFilter provides access to the status filter
// cache. (used by the status-filter results filter).
StatusFilter StructCache[*CachedStatusFilterResults]
// Visibility provides access to the item visibility
// cache. (used by the visibility filter).
Visibility VisibilityCache
Visibility StructCache[*CachedVisibility]
// Webfinger provides access to the webfinger URL cache.
Webfinger *ttl.Cache[string, string] // TTL=24hr, sweep=5min
@ -119,7 +120,6 @@ func (c *Caches) Init() {
c.initStatusEdit()
c.initStatusFave()
c.initStatusFaveIDs()
c.initStatusesFilterableFields()
c.initTag()
c.initThreadMute()
c.initToken()
@ -131,6 +131,7 @@ func (c *Caches) Init() {
c.initWebPushSubscription()
c.initWebPushSubscriptionIDs()
c.initMutes()
c.initStatusFilter()
c.initVisibility()
}
@ -143,10 +144,6 @@ func (c *Caches) Start() error {
return gtserror.New("could not start webfinger cache")
}
if !c.StatusesFilterableFields.Start(5 * time.Minute) {
return gtserror.New("could not start statusesFilterableFields cache")
}
return nil
}
@ -158,9 +155,6 @@ func (c *Caches) Stop() {
if c.Webfinger != nil {
_ = c.Webfinger.Stop()
}
if c.StatusesFilterableFields != nil {
_ = c.StatusesFilterableFields.Stop()
}
}
// Sweep will sweep all the available caches to ensure none
@ -183,6 +177,7 @@ func (c *Caches) Sweep(threshold float64) {
c.DB.Emoji.Trim(threshold)
c.DB.EmojiCategory.Trim(threshold)
c.DB.Filter.Trim(threshold)
c.DB.FilterIDs.Trim(threshold)
c.DB.FilterKeyword.Trim(threshold)
c.DB.FilterStatus.Trim(threshold)
c.DB.Follow.Trim(threshold)
@ -218,20 +213,13 @@ func (c *Caches) Sweep(threshold float64) {
c.DB.User.Trim(threshold)
c.DB.UserMute.Trim(threshold)
c.DB.UserMuteIDs.Trim(threshold)
c.Mutes.Trim(threshold)
c.StatusFilter.Trim(threshold)
c.Timelines.Home.Trim()
c.Timelines.List.Trim()
c.Visibility.Trim(threshold)
}
func (c *Caches) initStatusesFilterableFields() {
c.StatusesFilterableFields = new(ttl.Cache[string, []string])
c.StatusesFilterableFields.Init(
0,
512,
1*time.Hour,
)
}
func (c *Caches) initWebfinger() {
// Calculate maximum cache size.
cap := calculateCacheMax(

28
internal/cache/invalidate.go vendored
View file

@ -26,19 +26,23 @@ import (
// as an invalidation indicates a database INSERT / UPDATE / DELETE.
// NOTE THEY ARE ONLY CALLED WHEN THE ITEM IS IN THE CACHE, SO FOR
// HOOKS TO BE CALLED ON DELETE YOU MUST FIRST POPULATE IT IN THE CACHE.
//
// Also note that while Timelines are a part of the Caches{} object,
// they are generally not modified as part of side-effects here, as
// they often need specific IDs or more information that can only be
// fetched from the database. As such, they are generally handled as
// side-effects in the ./internal/processor/workers/ package.
func (c *Caches) OnInvalidateAccount(account *gtsmodel.Account) {
// Invalidate cached stats objects for this account.
c.DB.AccountStats.Invalidate("AccountID", account.ID)
// Invalidate as possible visibility target result.
c.Visibility.Invalidate("ItemID", account.ID)
// If account is local, invalidate as
// possible mute / visibility result requester.
// possible visibility result requester,
// also, invalidate any cached stats.
if account.IsLocal() {
c.DB.AccountStats.Invalidate("AccountID", account.ID)
c.Visibility.Invalidate("RequesterID", account.ID)
c.Mutes.Invalidate("RequesterID", account.ID)
}
// Invalidate this account's
@ -94,9 +98,8 @@ func (c *Caches) OnInvalidateBlock(block *gtsmodel.Block) {
localAccountIDs = append(localAccountIDs, block.TargetAccountID)
}
// Now perform local mute / visibility result invalidations.
// Now perform local visibility result invalidations.
c.Visibility.InvalidateIDs("RequesterID", localAccountIDs)
c.Mutes.InvalidateIDs("RequesterID", localAccountIDs)
// Invalidate source account's block lists.
c.DB.BlockIDs.Invalidate(block.AccountID)
@ -120,9 +123,8 @@ func (c *Caches) OnInvalidateFilter(filter *gtsmodel.Filter) {
c.DB.FilterKeyword.InvalidateIDs("ID", filter.KeywordIDs)
c.DB.FilterStatus.InvalidateIDs("ID", filter.StatusIDs)
// Invalidate account's timelines (in case local).
c.Timelines.Home.Unprepare(filter.AccountID)
c.Timelines.List.Unprepare(filter.AccountID)
// Invalidate account's status filter cache.
c.StatusFilter.Invalidate("RequesterID", filter.AccountID)
}
func (c *Caches) OnInvalidateFilterKeyword(filterKeyword *gtsmodel.FilterKeyword) {
@ -161,9 +163,8 @@ func (c *Caches) OnInvalidateFollow(follow *gtsmodel.Follow) {
localAccountIDs = append(localAccountIDs, follow.TargetAccountID)
}
// Now perform local mute / visibility result invalidations.
// Now perform local visibility result invalidations.
c.Visibility.InvalidateIDs("RequesterID", localAccountIDs)
c.Mutes.InvalidateIDs("RequesterID", localAccountIDs)
// Invalidate ID slice cache.
c.DB.FollowIDs.Invalidate(
@ -295,6 +296,9 @@ func (c *Caches) OnInvalidateStatus(status *gtsmodel.Status) {
// Invalidate cached stats objects for this account.
c.DB.AccountStats.Invalidate("AccountID", status.AccountID)
// Invalidate filter results targeting status.
c.StatusFilter.Invalidate("StatusID", status.ID)
// Invalidate status ID cached visibility.
c.Visibility.Invalidate("ItemID", status.ID)

4
internal/cache/mutes.go vendored
View file

@ -25,10 +25,6 @@ import (
"codeberg.org/gruf/go-structr"
)
type MutesCache struct {
StructCache[*CachedMute]
}
func (c *Caches) initMutes() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(

15
internal/cache/size.go vendored
View file

@ -25,6 +25,7 @@ import (
"unsafe"
"code.superseriousbusiness.org/gotosocial/internal/ap"
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/id"
@ -653,6 +654,20 @@ func sizeofStatusFave() uintptr {
}))
}
func sizeofStatusFilterResults() uintptr {
return uintptr(size.Of(&CachedStatusFilterResults{
StatusID: exampleID,
RequesterID: exampleID,
Results: [5][]StatusFilterResult{
{{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}},
{{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}},
{{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}},
{{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}},
{{Result: &apimodel.FilterResult{KeywordMatches: []string{"key", "word"}}}, {Result: &apimodel.FilterResult{StatusMatches: []string{exampleID, exampleID}}}, {}},
},
}))
}
func sizeofTag() uintptr {
return uintptr(size.Of(&gtsmodel.Tag{
ID: exampleID,

107
internal/cache/statusfilter.go vendored Normal file
View file

@ -0,0 +1,107 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cache
import (
"time"
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/log"
"codeberg.org/gruf/go-structr"
)
func (c *Caches) initStatusFilter() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(
sizeofStatusFilterResults(), // model in-mem size.
config.GetCacheStatusFilterMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
copyF := func(r1 *CachedStatusFilterResults) *CachedStatusFilterResults {
r2 := new(CachedStatusFilterResults)
*r2 = *r1
return r2
}
c.StatusFilter.Init(structr.CacheConfig[*CachedStatusFilterResults]{
Indices: []structr.IndexConfig{
{Fields: "RequesterID,StatusID"},
{Fields: "StatusID", Multiple: true},
{Fields: "RequesterID", Multiple: true},
},
MaxSize: cap,
IgnoreErr: func(err error) bool {
// don't cache any errors,
// it gets a little too tricky
// otherwise with ensuring
// errors are cleared out
return true
},
Copy: copyF,
})
}
const (
KeyContextHome = iota
KeyContextPublic
KeyContextNotifs
KeyContextThread
KeyContextAccount
keysLen // must always be last in list
)
// CachedStatusFilterResults contains the
// results of a cached status filter lookup.
type CachedStatusFilterResults struct {
// StatusID is the ID of the
// status this is a result for.
StatusID string
// RequesterID is the ID of the requesting
// account for this status filter lookup.
RequesterID string
// Results is a map (int-key-array) of status filter
// result slices in all possible filtering contexts.
Results [keysLen][]StatusFilterResult
}
// StatusFilterResult stores a single (positive,
// i.e. match) filter result for a status by a filter.
type StatusFilterResult struct {
// Expiry stores the time at which
// (if any) the filter result expires.
Expiry time.Time
// Result stores any generated filter result for
// this match intended to be shown at the frontend.
// This can be used to determine the filter action:
// - value => gtsmodel.FilterActionWarn
// - nil => gtsmodel.FilterActionHide
Result *apimodel.FilterResult
}
// Expired returns whether the filter result has expired.
func (r *StatusFilterResult) Expired(now time.Time) bool {
return !r.Expiry.IsZero() && !r.Expiry.After(now)
}

4
internal/cache/visibility.go vendored
View file

@ -23,10 +23,6 @@ import (
"codeberg.org/gruf/go-structr"
)
type VisibilityCache struct {
StructCache[*CachedVisibility]
}
func (c *Caches) initVisibility() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(

1
internal/config/config.go
View file

@ -267,6 +267,7 @@ type CacheConfiguration struct {
WebPushSubscriptionMemRatio float64 `name:"web-push-subscription-mem-ratio"`
WebPushSubscriptionIDsMemRatio float64 `name:"web-push-subscription-ids-mem-ratio"`
MutesMemRatio float64 `name:"mutes-mem-ratio"`
StatusFilterMemRatio float64 `name:"status-filter-mem-ratio"`
VisibilityMemRatio float64 `name:"visibility-mem-ratio"`
}

3
internal/config/defaults.go
View file

@ -233,8 +233,9 @@ var Defaults = Configuration{
WebfingerMemRatio: 0.1,
WebPushSubscriptionMemRatio: 1,
WebPushSubscriptionIDsMemRatio: 1,
VisibilityMemRatio: 2,
MutesMemRatio: 2,
StatusFilterMemRatio: 7,
VisibilityMemRatio: 2,
},
HTTPClient: HTTPClientConfiguration{

47
internal/config/helpers.gen.go
View file

@ -211,6 +211,7 @@ const (
CacheWebPushSubscriptionMemRatioFlag = "cache-web-push-subscription-mem-ratio"
CacheWebPushSubscriptionIDsMemRatioFlag = "cache-web-push-subscription-ids-mem-ratio"
CacheMutesMemRatioFlag = "cache-mutes-mem-ratio"
CacheStatusFilterMemRatioFlag = "cache-status-filter-mem-ratio"
CacheVisibilityMemRatioFlag = "cache-visibility-mem-ratio"
AdminAccountUsernameFlag = "username"
AdminAccountEmailFlag = "email"
@ -404,11 +405,12 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
flags.Float64("cache-web-push-subscription-mem-ratio", cfg.Cache.WebPushSubscriptionMemRatio, "")
flags.Float64("cache-web-push-subscription-ids-mem-ratio", cfg.Cache.WebPushSubscriptionIDsMemRatio, "")
flags.Float64("cache-mutes-mem-ratio", cfg.Cache.MutesMemRatio, "")
flags.Float64("cache-status-filter-mem-ratio", cfg.Cache.StatusFilterMemRatio, "")
flags.Float64("cache-visibility-mem-ratio", cfg.Cache.VisibilityMemRatio, "")
}
func (cfg *Configuration) MarshalMap() map[string]any {
cfgmap := make(map[string]any, 190)
cfgmap := make(map[string]any, 191)
cfgmap["log-level"] = cfg.LogLevel
cfgmap["log-timestamp-format"] = cfg.LogTimestampFormat
cfgmap["log-db-queries"] = cfg.LogDbQueries
@ -591,6 +593,7 @@ func (cfg *Configuration) MarshalMap() map[string]any {
cfgmap["cache-web-push-subscription-mem-ratio"] = cfg.Cache.WebPushSubscriptionMemRatio
cfgmap["cache-web-push-subscription-ids-mem-ratio"] = cfg.Cache.WebPushSubscriptionIDsMemRatio
cfgmap["cache-mutes-mem-ratio"] = cfg.Cache.MutesMemRatio
cfgmap["cache-status-filter-mem-ratio"] = cfg.Cache.StatusFilterMemRatio
cfgmap["cache-visibility-mem-ratio"] = cfg.Cache.VisibilityMemRatio
cfgmap["username"] = cfg.AdminAccountUsername
cfgmap["email"] = cfg.AdminAccountEmail
@ -2098,6 +2101,14 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error {
}
}
if ival, ok := cfgmap["cache-status-filter-mem-ratio"]; ok {
var err error
cfg.Cache.StatusFilterMemRatio, err = cast.ToFloat64E(ival)
if err != nil {
return fmt.Errorf("error casting %#v -> float64 for 'cache-status-filter-mem-ratio': %w", ival, err)
}
}
if ival, ok := cfgmap["cache-visibility-mem-ratio"]; ok {
var err error
cfg.Cache.VisibilityMemRatio, err = cast.ToFloat64E(ival)
@ -6197,6 +6208,28 @@ func GetCacheMutesMemRatio() float64 { return global.GetCacheMutesMemRatio() }
// SetCacheMutesMemRatio safely sets the value for global configuration 'Cache.MutesMemRatio' field
func SetCacheMutesMemRatio(v float64) { global.SetCacheMutesMemRatio(v) }
// GetCacheStatusFilterMemRatio safely fetches the Configuration value for state's 'Cache.StatusFilterMemRatio' field
func (st *ConfigState) GetCacheStatusFilterMemRatio() (v float64) {
st.mutex.RLock()
v = st.config.Cache.StatusFilterMemRatio
st.mutex.RUnlock()
return
}
// SetCacheStatusFilterMemRatio safely sets the Configuration value for state's 'Cache.StatusFilterMemRatio' field
func (st *ConfigState) SetCacheStatusFilterMemRatio(v float64) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.StatusFilterMemRatio = v
st.reloadToViper()
}
// GetCacheStatusFilterMemRatio safely fetches the value for global configuration 'Cache.StatusFilterMemRatio' field
func GetCacheStatusFilterMemRatio() float64 { return global.GetCacheStatusFilterMemRatio() }
// SetCacheStatusFilterMemRatio safely sets the value for global configuration 'Cache.StatusFilterMemRatio' field
func SetCacheStatusFilterMemRatio(v float64) { global.SetCacheStatusFilterMemRatio(v) }
// GetCacheVisibilityMemRatio safely fetches the Configuration value for state's 'Cache.VisibilityMemRatio' field
func (st *ConfigState) GetCacheVisibilityMemRatio() (v float64) {
st.mutex.RLock()
@ -6433,6 +6466,7 @@ func (st *ConfigState) GetTotalOfMemRatios() (total float64) {
total += st.config.Cache.WebPushSubscriptionMemRatio
total += st.config.Cache.WebPushSubscriptionIDsMemRatio
total += st.config.Cache.MutesMemRatio
total += st.config.Cache.StatusFilterMemRatio
total += st.config.Cache.VisibilityMemRatio
st.mutex.RUnlock()
return
@ -7395,6 +7429,17 @@ func flattenConfigMap(cfgmap map[string]any) {
}
}
for _, key := range [][]string{
{"cache", "status-filter-mem-ratio"},
} {
ival, ok := mapGet(cfgmap, key...)
if ok {
cfgmap["cache-status-filter-mem-ratio"] = ival
nestedKeys[key[0]] = struct{}{}
break
}
}
for _, key := range [][]string{
{"cache", "visibility-mem-ratio"},
} {

2
internal/filter/mutes/filter.go
View file

@ -41,5 +41,5 @@ const noauth = "noauth"
// given statuses or accounts are muted by a requester (user).
type Filter struct{ state *state.State }
// NewFilter returns a new Filter interface that will use the provided database.
// NewFilter returns a new Filter interface that will use the provided state.
func NewFilter(state *state.State) *Filter { return &Filter{state: state} }

105
internal/filter/status/api.go Normal file
View file

@ -0,0 +1,105 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package status
import (
"time"
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/util"
)
// NOTE: the below functions have all been copied
// from typeutils to prevent an import cycle. when
// we move the filtering logic out of the converter
// then we can safely remove these and call necessary
// function without any worry of import cycles.
func toAPIFilterV2(filter *gtsmodel.Filter) apimodel.FilterV2 {
apiFilterKeywords := make([]apimodel.FilterKeyword, len(filter.Keywords))
if len(apiFilterKeywords) != len(filter.Keywords) {
// bound check eliminiation compiler-hint
panic(gtserror.New("BCE"))
}
for i, filterKeyword := range filter.Keywords {
apiFilterKeywords[i] = apimodel.FilterKeyword{
ID: filterKeyword.ID,
Keyword: filterKeyword.Keyword,
WholeWord: util.PtrOrValue(filterKeyword.WholeWord, false),
}
}
apiFilterStatuses := make([]apimodel.FilterStatus, len(filter.Statuses))
if len(apiFilterStatuses) != len(filter.Statuses) {
// bound check eliminiation compiler-hint
panic(gtserror.New("BCE"))
}
for i, filterStatus := range filter.Statuses {
apiFilterStatuses[i] = apimodel.FilterStatus{
ID: filterStatus.ID,
StatusID: filterStatus.StatusID,
}
}
return apimodel.FilterV2{
ID: filter.ID,
Title: filter.Title,
Context: toAPIFilterContexts(filter),
ExpiresAt: toAPIFilterExpiresAt(filter.ExpiresAt),
FilterAction: toAPIFilterAction(filter.Action),
Keywords: apiFilterKeywords,
Statuses: apiFilterStatuses,
}
}
func toAPIFilterExpiresAt(expiresAt time.Time) *string {
if expiresAt.IsZero() {
return nil
}
return util.Ptr(util.FormatISO8601(expiresAt))
}
func toAPIFilterContexts(filter *gtsmodel.Filter) []apimodel.FilterContext {
apiContexts := make([]apimodel.FilterContext, 0, apimodel.FilterContextNumValues)
if filter.Contexts.Home() {
apiContexts = append(apiContexts, apimodel.FilterContextHome)
}
if filter.Contexts.Notifications() {
apiContexts = append(apiContexts, apimodel.FilterContextNotifications)
}
if filter.Contexts.Public() {
apiContexts = append(apiContexts, apimodel.FilterContextPublic)
}
if filter.Contexts.Thread() {
apiContexts = append(apiContexts, apimodel.FilterContextThread)
}
if filter.Contexts.Account() {
apiContexts = append(apiContexts, apimodel.FilterContextAccount)
}
return apiContexts
}
func toAPIFilterAction(m gtsmodel.FilterAction) apimodel.FilterAction {
switch m {
case gtsmodel.FilterActionWarn:
return apimodel.FilterActionWarn
case gtsmodel.FilterActionHide:
return apimodel.FilterActionHide
}
return apimodel.FilterActionNone
}

33
internal/filter/status/filter.go Normal file
View file

@ -0,0 +1,33 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package status
import (
"code.superseriousbusiness.org/gotosocial/internal/state"
)
// noauth is a placeholder ID used in cache lookups
// when there is no authorized account ID to use.
const noauth = "noauth"
// Filter packages up logic for checking whether
// given status is muted by a given requester (user).
type Filter struct{ state *state.State }
// New returns a new Filter interface that will use the provided state.
func NewFilter(state *state.State) *Filter { return &Filter{state} }

313
internal/filter/status/status.go
View file

@ -15,12 +15,317 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package status represents status filters managed by the user through the API.
package status
import (
"errors"
"context"
"regexp"
"slices"
"time"
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/cache"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
)
// ErrHideStatus indicates that a status has been filtered and should not be returned at all.
var ErrHideStatus = errors.New("hide status")
// StatusFilterResultsInContext returns status filtering results, limited
// to the given filtering context, about the given status for requester.
// The hide flag is immediately returned if any filters match with the
// HIDE action set, else API model filter results for the WARN action.
func (f *Filter) StatusFilterResultsInContext(
ctx context.Context,
requester *gtsmodel.Account,
status *gtsmodel.Status,
context gtsmodel.FilterContext,
) (
results []apimodel.FilterResult,
hidden bool,
err error,
) {
if context == gtsmodel.FilterContextNone {
// fast-check any context.
return nil, false, nil
}
// Get cached filter results for status to requester in all contexts.
allResults, now, err := f.StatusFilterResults(ctx, requester, status)
if err != nil {
return nil, false, err
}
// Get results applicable to current context.
var forContext []cache.StatusFilterResult
switch context {
case gtsmodel.FilterContextHome:
forContext = allResults.Results[cache.KeyContextHome]
case gtsmodel.FilterContextPublic:
forContext = allResults.Results[cache.KeyContextPublic]
case gtsmodel.FilterContextNotifications:
forContext = allResults.Results[cache.KeyContextNotifs]
case gtsmodel.FilterContextThread:
forContext = allResults.Results[cache.KeyContextThread]
case gtsmodel.FilterContextAccount:
forContext = allResults.Results[cache.KeyContextAccount]
}
// Iterate results in context, gathering prepared API models.
results = make([]apimodel.FilterResult, 0, len(forContext))
for _, result := range forContext {
// Check if result expired.
if result.Expired(now) {
continue
}
// If the result indicates
// status should just be
// hidden then return here.
if result.Result == nil {
return nil, true, nil
}
// Append pre-prepared API model to slice.
results = append(results, *result.Result)
}
return
}
// StatusFilterResults returns status filtering results (in all contexts) about the given status for the given requesting account.
func (f *Filter) StatusFilterResults(ctx context.Context, requester *gtsmodel.Account, status *gtsmodel.Status) (*cache.CachedStatusFilterResults, time.Time, error) {
// For requester ID use a
// fallback 'noauth' string
// by default for lookups.
requesterID := noauth
if requester != nil {
requesterID = requester.ID
}
// Get current time.
now := time.Now()
// Load status filtering results for this requesting account about status from cache, using load callback function if necessary.
results, err := f.state.Caches.StatusFilter.LoadOne("RequesterID,StatusID", func() (*cache.CachedStatusFilterResults, error) {
// Load status filter results for given status.
results, err := f.getStatusFilterResults(ctx,
requester,
status,
now,
)
if err != nil {
if err == cache.SentinelError {
// Filter-out our temporary
// race-condition error.
return &cache.CachedStatusFilterResults{}, nil
}
return nil, err
}
// Convert to cacheable results type.
return &cache.CachedStatusFilterResults{
StatusID: status.ID,
RequesterID: requesterID,
Results: results,
}, nil
}, requesterID, status.ID)
if err != nil {
return nil, now, err
}
return results, now, err
}
// getStatusFilterResults loads status filtering results for
// the given status, given the current time (checking expiries).
// this will load results for all possible filtering contexts.
func (f *Filter) getStatusFilterResults(
ctx context.Context,
requester *gtsmodel.Account,
status *gtsmodel.Status,
now time.Time,
) (
[5][]cache.StatusFilterResult,
error,
) {
var results [5][]cache.StatusFilterResult
if requester == nil {
// Without auth, there will be no possible
// filters to exists, return as 'unfiltered'.
return results, nil
}
// Get the string fields status is
// filterable on for keyword matching.
fields := getFilterableFields(status)
// Get all status filters owned by the requesting account.
filters, err := f.state.DB.GetFiltersByAccountID(ctx, requester.ID)
if err != nil {
return results, gtserror.Newf("error getting account filters: %w", err)
}
// For proper status filtering we need all fields populated.
if err := f.state.DB.PopulateStatus(ctx, status); err != nil {
return results, gtserror.Newf("error populating status: %w", err)
}
// Generate result for each filter.
for _, filter := range filters {
// Skip already expired.
if filter.Expired(now) {
continue
}
// Later stored API result, if any.
// (for the HIDE action, it is unset).
var apiResult *apimodel.FilterResult
switch filter.Action {
case gtsmodel.FilterActionWarn:
// For filter action WARN get all possible filter matches against status.
keywordMatches, statusMatches := getFilterMatches(filter, status.ID, fields)
if len(keywordMatches) == 0 && len(statusMatches) == 0 {
continue
}
// Wrap matches in frontend API model.
apiResult = &apimodel.FilterResult{
Filter: toAPIFilterV2(filter),
KeywordMatches: keywordMatches,
StatusMatches: statusMatches,
}
// For filter action HIDE quickly
// look for first possible match
// against this status, or reloop.
case gtsmodel.FilterActionHide:
if !doesFilterMatch(filter, status.ID, fields) {
continue
}
}
// Wrap the filter result in our cache model.
// This model simply existing implies this
// status has been filtered, defaulting to
// action HIDE, or WARN on a non-nil result.
result := cache.StatusFilterResult{
Expiry: filter.ExpiresAt,
Result: apiResult,
}
// Append generated result if
// applies in 'home' context.
if filter.Contexts.Home() {
const key = cache.KeyContextHome
results[key] = append(results[key], result)
}
// Append generated result if
// applies in 'public' context.
if filter.Contexts.Public() {
const key = cache.KeyContextPublic
results[key] = append(results[key], result)
}
// Append generated result if
// applies in 'notifs' context.
if filter.Contexts.Notifications() {
const key = cache.KeyContextNotifs
results[key] = append(results[key], result)
}
// Append generated result if
// applies in 'thread' context.
if filter.Contexts.Thread() {
const key = cache.KeyContextThread
results[key] = append(results[key], result)
}
// Append generated result if
// applies in 'account' context.
if filter.Contexts.Account() {
const key = cache.KeyContextAccount
results[key] = append(results[key], result)
}
}
// Iterate all filter results.
for _, key := range [5]int{
cache.KeyContextHome,
cache.KeyContextPublic,
cache.KeyContextNotifs,
cache.KeyContextThread,
cache.KeyContextAccount,
} {
// Sort the slice of filter results by their expiry, soonest coming first.
slices.SortFunc(results[key], func(a, b cache.StatusFilterResult) int {
const k = +1
switch {
case a.Expiry.IsZero():
if b.Expiry.IsZero() {
return 0
}
return +k
case b.Expiry.IsZero():
return -k
case a.Expiry.Before(b.Expiry):
return -k
case b.Expiry.Before(a.Expiry):
return +k
default:
return 0
}
})
}
return results, nil
}
// getFilterMatches returns *all* the keyword and status matches of status ID and fields on given filter.
func getFilterMatches(filter *gtsmodel.Filter, statusID string, fields []string) ([]string, []string) {
keywordMatches := make([]string, 0, len(filter.Keywords))
for _, keyword := range filter.Keywords {
if doesKeywordMatch(keyword.Regexp, fields) {
keywordMatches = append(keywordMatches, keyword.Keyword)
}
}
statusMatches := make([]string, 0, 1)
for _, status := range filter.Statuses {
if status.StatusID == statusID {
statusMatches = append(statusMatches, statusID)
}
}
return keywordMatches, statusMatches
}
// doesFilterMatch returns if any of fields or status ID match on the given filter.
func doesFilterMatch(filter *gtsmodel.Filter, statusID string, fields []string) bool {
for _, status := range filter.Statuses {
if status.StatusID == statusID {
return true
}
}
for _, keyword := range filter.Keywords {
if doesKeywordMatch(keyword.Regexp, fields) {
return true
}
}
return false
}
// doesKeywordMatch returns if any of fields match given keyword regex.
func doesKeywordMatch(rgx *regexp.Regexp, fields []string) bool {
for _, field := range fields {
if rgx.MatchString(field) {
return true
}
}
return false
}

80
internal/filter/status/text.go Normal file
View file

@ -0,0 +1,80 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package status
import (
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/text"
)
// getFilterableFields returns text fields from
// a status that we might want to filter on:
//
// - content warning
// - content (converted to plaintext from HTML)
// - media descriptions
// - poll options
//
// Each field should be filtered separately. This avoids
// scenarios where false-positive multiple-word matches
// can be made by matching the last word of one field
// combined with the first word of the next field together.
func getFilterableFields(status *gtsmodel.Status) []string {
// Estimate expected no of status fields.
fieldCount := 2 + len(status.Attachments)
if status.Poll != nil {
fieldCount += len(status.Poll.Options)
}
fields := make([]string, 0, fieldCount)
// Append content warning / title.
if status.ContentWarning != "" {
fields = append(fields, status.ContentWarning)
}
// Status content. Though we have raw text
// available for statuses created on our
// instance, use the plaintext version to
// remove markdown-formatting characters
// and ensure more consistent filtering.
if status.Content != "" {
text := text.ParseHTMLToPlain(status.Content)
if text != "" {
fields = append(fields, text)
}
}
// Media descriptions, only where they are set.
for _, attachment := range status.Attachments {
if attachment.Description != "" {
fields = append(fields, attachment.Description)
}
}
// Non-empty poll options.
if status.Poll != nil {
for _, opt := range status.Poll.Options {
if opt != "" {
fields = append(fields, opt)
}
}
}
return fields
}

84
internal/filter/status/text_test.go Normal file
View file

@ -0,0 +1,84 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package status
import (
"testing"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"github.com/stretchr/testify/assert"
)
func TestFilterableText(t *testing.T) {
type testcase struct {
status *gtsmodel.Status
expectedFields []string
}
for _, testcase := range []testcase{
{
status: &gtsmodel.Status{
ContentWarning: "This is a test status",
Content: `<p>Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> instance.</p>`,
},
expectedFields: []string{
"This is a test status",
"Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> instance.",
},
},
{
status: &gtsmodel.Status{
Content: `<p><span class="h-card"><a href="https://example.org/@zlatko" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>zlatko</span></a></span> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)</p><p><a href="https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863" rel="nofollow noreferrer noopener" target="_blank">https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863</a></p>`,
},
expectedFields: []string{
"@zlatko <https://example.org/@zlatko> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)\n\nhttps://codeberg.org/superseriousbusiness/gotosocial/pulls/2863 <https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863>",
},
},
{
status: &gtsmodel.Status{
ContentWarning: "Nerd stuff",
Content: `<p>Latest graphs for <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> on <a href="https://github.com/ncruces/go-sqlite3" rel="nofollow noreferrer noopener" target="_blank">Wasm sqlite3</a> with <a href="https://codeberg.org/gruf/go-ffmpreg" rel="nofollow noreferrer noopener" target="_blank">embedded Wasm ffmpeg</a>, both running on <a href="https://wazero.io/" rel="nofollow noreferrer noopener" target="_blank">Wazero</a>, and configured with a <a href="https://codeberg.org/superseriousbusiness/gotosocial/src/commit/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266" rel="nofollow noreferrer noopener" target="_blank">50MiB db cache target</a>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.</p>`,
Attachments: []*gtsmodel.MediaAttachment{
{
Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`,
},
{
Description: `Another media attachment`,
},
},
Poll: &gtsmodel.Poll{
Options: []string{
"Poll option 1",
"Poll option 2",
},
},
},
expectedFields: []string{
"Nerd stuff",
"Latest graphs for #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> on Wasm sqlite3 <https://github.com/ncruces/go-sqlite3> with embedded Wasm ffmpeg <https://codeberg.org/gruf/go-ffmpreg>, both running on Wazero <https://wazero.io/>, and configured with a 50MiB db cache target <https://codeberg.org/superseriousbusiness/gotosocial/src/commit/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.",
"Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.",
"Another media attachment",
"Poll option 1",
"Poll option 2",
},
},
} {
fields := getFilterableFields(testcase.status)
assert.Equal(t, testcase.expectedFields, fields)
}
}

3
internal/processing/account/bookmarks.go
View file

@ -74,11 +74,12 @@ func (p *Processor) BookmarksGet(ctx context.Context, requestingAccount *gtsmode
}
// Convert the status.
item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, gtsmodel.FilterContextNone, nil)
item, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, gtsmodel.FilterContextNone)
if err != nil {
log.Errorf(ctx, "error converting bookmarked status to api: %s", err)
continue
}
items = append(items, item)
}

8
internal/processing/account/statuses.go
View file

@ -96,15 +96,9 @@ func (p *Processor) StatusesGet(
return nil, gtserror.NewErrorInternalError(err)
}
filters, err := p.state.DB.GetFiltersByAccountID(ctx, requestingAccount.ID)
if err != nil {
err = gtserror.Newf("couldn't retrieve filters for account %s: %w", requestingAccount.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
for _, s := range filtered {
// Convert filtered statuses to API statuses.
item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount, gtsmodel.FilterContextAccount, filters)
item, err := p.converter.StatusToAPIStatus(ctx, s, requestingAccount, gtsmodel.FilterContextAccount)
if err != nil {
log.Errorf(ctx, "error convering to api status: %v", err)
continue

7
internal/processing/common/status.go
View file

@ -24,10 +24,10 @@ import (
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/db"
"code.superseriousbusiness.org/gotosocial/internal/federation/dereferencing"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
)
// GetOwnStatus fetches the given status with ID,
@ -214,7 +214,6 @@ func (p *Processor) GetAPIStatus(
target,
requester,
gtsmodel.FilterContextNone,
nil,
)
if err != nil {
err := gtserror.Newf("error converting: %w", err)
@ -235,7 +234,6 @@ func (p *Processor) GetVisibleAPIStatuses(
requester *gtsmodel.Account,
statuses []*gtsmodel.Status,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) []apimodel.Status {
// Start new log entry with
@ -278,9 +276,8 @@ func (p *Processor) GetVisibleAPIStatuses(
status,
requester,
filterCtx,
filters,
)
if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
if err != nil && !errors.Is(err, typeutils.ErrHideStatus) {
l.Errorf("error converting: %v", err)
continue
}

4
internal/processing/conversations/update.go
View file

@ -23,11 +23,11 @@ import (
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/db"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/id"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
"code.superseriousbusiness.org/gotosocial/internal/util"
)
@ -167,7 +167,7 @@ func (p *Processor) UpdateConversationsForStatus(ctx context.Context, status *gt
if err != nil {
// If the conversation's last status matched a hide filter, skip it.
// If there was another kind of error, log that and skip it anyway.
if !errors.Is(err, statusfilter.ErrHideStatus) {
if !errors.Is(err, typeutils.ErrHideStatus) {
log.Errorf(ctx,
"error converting conversation %s to API representation for account %s: %v",
status.ID,

32
internal/processing/filters/common/common.go
View file

@ -28,12 +28,19 @@ import (
"code.superseriousbusiness.org/gotosocial/internal/gtscontext"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/processing/stream"
"code.superseriousbusiness.org/gotosocial/internal/state"
)
type Processor struct{ state *state.State }
type Processor struct {
state *state.State
stream *stream.Processor
}
func New(state *state.State) *Processor { return &Processor{state} }
func New(state *state.State, stream *stream.Processor) *Processor {
return &Processor{state, stream}
}
// CheckFilterExists calls .GetFilter() with a barebones context to not
// fetch any sub-models, and not returning the result. this functionally
@ -160,6 +167,27 @@ func (p *Processor) GetFilterKeyword(
return keyword, filter, nil
}
// OnFilterChanged ...
func (p *Processor) OnFilterChanged(ctx context.Context, requester *gtsmodel.Account) {
// Get list of list IDs created by this requesting account.
listIDs, err := p.state.DB.GetListIDsByAccountID(ctx, requester.ID)
if err != nil {
log.Errorf(ctx, "error getting account '%s' lists: %v", requester.Username, err)
}
// Unprepare this requester's home timeline.
p.state.Caches.Timelines.Home.Unprepare(requester.ID)
// Unprepare list timelines.
for _, id := range listIDs {
p.state.Caches.Timelines.List.Unprepare(id)
}
// Send filter changed event for account.
p.stream.FiltersChanged(ctx, requester)
}
// FromAPIContexts converts a slice of frontend API model FilterContext types to our internal FilterContexts bit field.
func FromAPIContexts(apiContexts []apimodel.FilterContext) (gtsmodel.FilterContexts, gtserror.WithCode) {
var contexts gtsmodel.FilterContexts

4
internal/processing/filters/v1/create.go
View file

@ -91,8 +91,8 @@ func (p *Processor) Create(ctx context.Context, requester *gtsmodel.Account, for
return nil, gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
// Return as converted frontend filter keyword model.
return typeutils.FilterKeywordToAPIFilterV1(filter, filterKeyword), nil

4
internal/processing/filters/v1/delete.go
View file

@ -63,8 +63,8 @@ func (p *Processor) Delete(
}
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
return nil
}

5
internal/processing/filters/v1/filters.go
View file

@ -19,7 +19,6 @@ package v1
import (
"code.superseriousbusiness.org/gotosocial/internal/processing/filters/common"
"code.superseriousbusiness.org/gotosocial/internal/processing/stream"
"code.superseriousbusiness.org/gotosocial/internal/state"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
)
@ -30,15 +29,13 @@ type Processor struct {
state *state.State
converter *typeutils.Converter
stream *stream.Processor
}
func New(state *state.State, converter *typeutils.Converter, common *common.Processor, stream *stream.Processor) Processor {
func New(state *state.State, converter *typeutils.Converter, common *common.Processor) Processor {
return Processor{
c: common,
state: state,
converter: converter,
stream: stream,
}
}

4
internal/processing/filters/v1/update.go
View file

@ -166,8 +166,8 @@ func (p *Processor) Update(
return nil, gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
// Return as converted frontend filter keyword model.
return typeutils.FilterKeywordToAPIFilterV1(filter, filterKeyword), nil

8
internal/processing/filters/v2/create.go
View file

@ -34,13 +34,13 @@ import (
// Create a new filter for the given account, using the provided parameters.
// These params should have already been validated by the time they reach this function.
func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.FilterCreateRequestV2) (*apimodel.FilterV2, gtserror.WithCode) {
func (p *Processor) Create(ctx context.Context, requester *gtsmodel.Account, form *apimodel.FilterCreateRequestV2) (*apimodel.FilterV2, gtserror.WithCode) {
var errWithCode gtserror.WithCode
// Create new filter model.
filter := &gtsmodel.Filter{
ID: id.NewULID(),
AccountID: account.ID,
AccountID: requester.ID,
Title: form.Title,
}
@ -104,8 +104,8 @@ func (p *Processor) Create(ctx context.Context, account *gtsmodel.Account, form
return nil, gtserror.NewErrorInternalError(err)
}
// Send a filters changed event.
p.stream.FiltersChanged(ctx, account)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
// Return as converted frontend filter model.
return typeutils.FilterToAPIFilterV2(filter), nil

4
internal/processing/filters/v2/delete.go
View file

@ -44,8 +44,8 @@ func (p *Processor) Delete(
return gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
return nil
}

5
internal/processing/filters/v2/filters.go
View file

@ -19,7 +19,6 @@ package v2
import (
"code.superseriousbusiness.org/gotosocial/internal/processing/filters/common"
"code.superseriousbusiness.org/gotosocial/internal/processing/stream"
"code.superseriousbusiness.org/gotosocial/internal/state"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
)
@ -30,15 +29,13 @@ type Processor struct {
state *state.State
converter *typeutils.Converter
stream *stream.Processor
}
func New(state *state.State, converter *typeutils.Converter, common *common.Processor, stream *stream.Processor) Processor {
func New(state *state.State, converter *typeutils.Converter, common *common.Processor) Processor {
return Processor{
c: common,
state: state,
converter: converter,
stream: stream,
}
}

4
internal/processing/filters/v2/keywordcreate.go
View file

@ -72,8 +72,8 @@ func (p *Processor) KeywordCreate(ctx context.Context, requester *gtsmodel.Accou
return nil, gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
return typeutils.FilterKeywordToAPIFilterKeyword(filterKeyword), nil
}

4
internal/processing/filters/v2/keyworddelete.go
View file

@ -54,8 +54,8 @@ func (p *Processor) KeywordDelete(
return gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
return nil
}

4
internal/processing/filters/v2/keywordupdate.go
View file

@ -63,8 +63,8 @@ func (p *Processor) KeywordUpdate(
return nil, gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
return typeutils.FilterKeywordToAPIFilterKeyword(filterKeyword), nil
}

4
internal/processing/filters/v2/statuscreate.go
View file

@ -71,8 +71,8 @@ func (p *Processor) StatusCreate(ctx context.Context, requester *gtsmodel.Accoun
return nil, gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
return typeutils.FilterStatusToAPIFilterStatus(filterStatus), nil
}

4
internal/processing/filters/v2/statusdelete.go
View file

@ -54,8 +54,8 @@ func (p *Processor) StatusDelete(
return gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
return nil
}

4
internal/processing/filters/v2/update.go
View file

@ -135,8 +135,8 @@ func (p *Processor) Update(
return nil, gtserror.NewErrorInternalError(err)
}
// Stream a filters changed event to WS.
p.stream.FiltersChanged(ctx, requester)
// Handle filter change side-effects.
p.c.OnFilterChanged(ctx, requester)
// Return as converted frontend filter model.
return typeutils.FilterToAPIFilterV2(filter), nil

6
internal/processing/processor.go
View file

@ -225,7 +225,7 @@ func NewProcessor(
processor.account = account.New(&common, state, converter, mediaManager, federator, visFilter, parseMentionFunc)
processor.media = media.New(&common, state, converter, federator, mediaManager, federator.TransportController())
processor.stream = stream.New(state, oauthServer)
filterCommon := filterCommon.New(state)
filterCommon := filterCommon.New(state, &processor.stream)
// Instantiate the rest of the sub
// processors + pin them to this struct.
@ -234,8 +234,8 @@ func NewProcessor(
processor.application = application.New(state, converter)
processor.conversations = conversations.New(state, converter, visFilter, muteFilter)
processor.fedi = fedi.New(state, &common, converter, federator, visFilter)
processor.filtersv1 = filtersv1.New(state, converter, filterCommon, &processor.stream)
processor.filtersv2 = filtersv2.New(state, converter, filterCommon, &processor.stream)
processor.filtersv1 = filtersv1.New(state, converter, filterCommon)
processor.filtersv2 = filtersv2.New(state, converter, filterCommon)
processor.interactionRequests = interactionrequests.New(&common, state, converter)
processor.list = list.New(state, converter)
processor.markers = markers.New(state, converter)

2
internal/processing/search/util.go
View file

@ -113,7 +113,7 @@ func (p *Processor) packageStatuses(
continue
}
apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := p.converter.StatusToAPIStatus(ctx, status, requestingAccount, gtsmodel.FilterContextNone)
if err != nil {
log.Debugf(ctx, "skipping status %s because it couldn't be converted to its api representation: %s", status.ID, err)
continue

16
internal/processing/status/context.go
View file

@ -275,20 +275,6 @@ func (p *Processor) ContextGet(
requester *gtsmodel.Account,
targetStatusID string,
) (*apimodel.ThreadContext, gtserror.WithCode) {
// Retrieve filters as they affect
// what should be shown to requester.
filters, err := p.state.DB.GetFiltersByAccountID(
ctx, // Populate filters.
requester.ID,
)
if err != nil {
err = gtserror.Newf(
"couldn't retrieve filters for account %s: %w",
requester.ID, err,
)
return nil, gtserror.NewErrorInternalError(err)
}
// Retrieve the full thread context.
threadContext, errWithCode := p.contextGet(ctx,
requester,
@ -305,7 +291,6 @@ func (p *Processor) ContextGet(
requester,
threadContext.ancestors,
gtsmodel.FilterContextThread,
filters,
)
// Convert and filter the thread context descendants
@ -313,7 +298,6 @@ func (p *Processor) ContextGet(
requester,
threadContext.descendants,
gtsmodel.FilterContextThread,
filters,
)
return &apiContext, nil

2
internal/processing/stream/statusupdate_test.go
View file

@ -39,7 +39,7 @@ func (suite *StatusUpdateTestSuite) TestStreamNotification() {
suite.NoError(errWithCode)
editedStatus := suite.testStatuses["remote_account_1_status_1"]
apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(suite.T().Context(), editedStatus, account, gtsmodel.FilterContextNotifications, nil)
apiStatus, err := typeutils.NewConverter(&suite.state).StatusToAPIStatus(suite.T().Context(), editedStatus, account, gtsmodel.FilterContextNotifications)
suite.NoError(err)
suite.streamProcessor.StatusUpdate(suite.T().Context(), account, apiStatus, stream.TimelineHome)

2
internal/processing/timeline/faved.go
View file

@ -56,7 +56,7 @@ func (p *Processor) FavedTimelineGet(ctx context.Context, authed *apiutil.Auth,
continue
}
apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account, gtsmodel.FilterContextNone, nil)
apiStatus, err := p.converter.StatusToAPIStatus(ctx, s, authed.Account, gtsmodel.FilterContextNone)
if err != nil {
log.Errorf(ctx, "error convering to api status: %v", err)
continue

14
internal/processing/timeline/notification.go
View file

@ -27,11 +27,11 @@ import (
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util"
"code.superseriousbusiness.org/gotosocial/internal/db"
"code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
"code.superseriousbusiness.org/gotosocial/internal/util"
)
@ -59,12 +59,6 @@ func (p *Processor) NotificationsGet(
return util.EmptyPageableResponse(), nil
}
filters, err := p.state.DB.GetFiltersByAccountID(ctx, requester.ID)
if err != nil {
err = gtserror.Newf("error getting account %s filters: %w", requester.ID, err)
return nil, gtserror.NewErrorInternalError(err)
}
var (
items = make([]interface{}, 0, count)
@ -115,9 +109,9 @@ func (p *Processor) NotificationsGet(
}
}
item, err := p.converter.NotificationToAPINotification(ctx, n, filters)
item, err := p.converter.NotificationToAPINotification(ctx, n, true)
if err != nil {
if !errors.Is(err, status.ErrHideStatus) {
if !errors.Is(err, typeutils.ErrHideStatus) {
log.Debugf(ctx, "skipping notification %s because it couldn't be converted to its api representation: %s", n.ID, err)
}
continue
@ -160,7 +154,7 @@ func (p *Processor) NotificationGet(ctx context.Context, account *gtsmodel.Accou
// or mute checking for a notification directly
// fetched by ID. only from timelines etc.
apiNotif, err := p.converter.NotificationToAPINotification(ctx, notif, nil)
apiNotif, err := p.converter.NotificationToAPINotification(ctx, notif, false)
if err != nil {
err := gtserror.Newf("error converting to api model: %w", err)
return nil, gtserror.WrapWithCode(http.StatusInternalServerError, err)

17
internal/processing/timeline/timeline.go
View file

@ -25,9 +25,7 @@ import (
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
timelinepkg "code.superseriousbusiness.org/gotosocial/internal/cache/timeline"
"code.superseriousbusiness.org/gotosocial/internal/db"
"code.superseriousbusiness.org/gotosocial/internal/filter/mutes"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/filter/visibility"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
@ -79,18 +77,6 @@ func (p *Processor) getStatusTimeline(
gtserror.WithCode,
) {
var err error
var filters []*gtsmodel.Filter
if requester != nil {
// Fetch all filters relevant for requesting account.
filters, err = p.state.DB.GetFiltersByAccountID(ctx,
requester.ID,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err := gtserror.Newf("error getting account filters: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
}
// Ensure we have valid
// input paging cursor.
@ -135,9 +121,8 @@ func (p *Processor) getStatusTimeline(
status,
requester,
filterCtx,
filters,
)
if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
if err != nil && !errors.Is(err, typeutils.ErrHideStatus) {
return nil, err
}
return apiStatus, nil

7
internal/processing/workers/fromclientapi_test.go
View file

@ -213,7 +213,6 @@ func (suite *FromClientAPITestSuite) statusJSON(
status,
requestingAccount,
gtsmodel.FilterContextNone,
nil,
)
if err != nil {
suite.FailNow(err.Error())
@ -345,7 +344,7 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() {
suite.FailNow("timed out waiting for new status notification")
}
apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, nil)
apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, false)
if err != nil {
suite.FailNow(err.Error())
}
@ -2032,7 +2031,7 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithAuthorOnExclusiv
suite.FailNow("timed out waiting for new status notification")
}
apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, nil)
apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, false)
if err != nil {
suite.FailNow(err.Error())
}
@ -2217,7 +2216,7 @@ func (suite *FromClientAPITestSuite) TestProcessUpdateStatusInteractedWith() {
suite.FailNow("timed out waiting for edited status notification")
}
apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, nil)
apiNotif, err := testStructs.TypeConverter.NotificationToAPINotification(ctx, notif, false)
if err != nil {
suite.FailNow(err.Error())
}

13
internal/processing/workers/surfacenotify.go
View file

@ -23,11 +23,11 @@ import (
"strings"
"code.superseriousbusiness.org/gotosocial/internal/db"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtscontext"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/id"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
"code.superseriousbusiness.org/gotosocial/internal/util"
"code.superseriousbusiness.org/gotosocial/internal/util/xslices"
)
@ -743,14 +743,9 @@ func (s *Surface) Notify(
}
}
filters, err := s.State.DB.GetFiltersByAccountID(ctx, targetAccount.ID)
if err != nil {
return gtserror.Newf("couldn't retrieve filters for account %s: %w", targetAccount.ID, err)
}
// Convert the notification to frontend API model for streaming / push.
apiNotif, err := s.Converter.NotificationToAPINotification(ctx, notif, filters)
if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
// Convert the notification to frontend API model for streaming / web push.
apiNotif, err := s.Converter.NotificationToAPINotification(ctx, notif, true)
if err != nil && !errors.Is(err, typeutils.ErrHideStatus) {
return gtserror.Newf("error converting notification to api representation: %w", err)
}

57
internal/processing/workers/surfacetimeline.go
View file

@ -22,12 +22,12 @@ import (
"errors"
"code.superseriousbusiness.org/gotosocial/internal/cache/timeline"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtscontext"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/stream"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
"code.superseriousbusiness.org/gotosocial/internal/util"
)
@ -147,19 +147,10 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
continue
}
// Get relevant filters for this follow's account.
// (note the origin account of the follow is receiver of status).
filters, err := s.getFilters(ctx, follow.AccountID)
if err != nil {
log.Error(ctx, err)
continue
}
// Add status to any relevant lists for this follow, if applicable.
listTimelined, exclusive, err := s.listTimelineStatusForFollow(ctx,
status,
follow,
filters,
)
if err != nil {
log.Errorf(ctx, "error list timelining status: %v", err)
@ -181,7 +172,6 @@ func (s *Surface) timelineAndNotifyStatusForFollowers(
status,
stream.TimelineHome,
gtsmodel.FilterContextHome,
filters,
); homeTimelined {
// If hometimelined, add to list of returned account IDs.
@ -239,7 +229,6 @@ func (s *Surface) listTimelineStatusForFollow(
ctx context.Context,
status *gtsmodel.Status,
follow *gtsmodel.Follow,
filters []*gtsmodel.Filter,
) (timelined bool, exclusive bool, err error) {
// Get all lists that contain this given follow.
@ -276,7 +265,6 @@ func (s *Surface) listTimelineStatusForFollow(
status,
stream.TimelineList+":"+list.ID, // key streamType to this specific list
gtsmodel.FilterContextHome,
filters,
)
// Update flag based on if timelined.
@ -286,15 +274,6 @@ func (s *Surface) listTimelineStatusForFollow(
return timelined, exclusive, nil
}
// getFiltersAndMutes returns an account's filters and mutes.
func (s *Surface) getFilters(ctx context.Context, accountID string) ([]*gtsmodel.Filter, error) {
filters, err := s.State.DB.GetFiltersByAccountID(ctx, accountID)
if err != nil {
return nil, gtserror.Newf("couldn't retrieve filters for account %s: %w", accountID, err)
}
return filters, err
}
// listEligible checks if the given status is eligible
// for inclusion in the list that that the given listEntry
// belongs to, based on the replies policy of the list.
@ -370,7 +349,6 @@ func (s *Surface) timelineStatus(
status *gtsmodel.Status,
streamType string,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) bool {
// Attempt to convert status to frontend API representation,
@ -379,9 +357,8 @@ func (s *Surface) timelineStatus(
status,
account,
filterCtx,
filters,
)
if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
if err != nil && !errors.Is(err, typeutils.ErrHideStatus) {
log.Error(ctx, "error converting status %s to frontend: %v", status.URI, err)
}
@ -425,19 +402,12 @@ func (s *Surface) timelineAndNotifyStatusForTagFollowers(
// Insert the status into the home timeline of each tag follower.
errs := gtserror.MultiError{}
for _, tagFollowerAccount := range tagFollowerAccounts {
filters, err := s.getFilters(ctx, tagFollowerAccount.ID)
if err != nil {
errs.Append(err)
continue
}
_ = s.timelineStatus(ctx,
s.State.Caches.Timelines.Home.MustGet(tagFollowerAccount.ID),
tagFollowerAccount,
status,
stream.TimelineHome,
gtsmodel.FilterContextHome,
filters,
)
}
@ -605,19 +575,10 @@ func (s *Surface) timelineStatusUpdateForFollowers(
continue
}
// Get relevant filters and mutes for this follow's account.
// (note the origin account of the follow is receiver of status).
filters, err := s.getFilters(ctx, follow.AccountID)
if err != nil {
log.Error(ctx, err)
continue
}
// Add status to relevant lists for this follow, if applicable.
_, exclusive, err := s.listTimelineStatusUpdateForFollow(ctx,
status,
follow,
filters,
)
if err != nil {
log.Errorf(ctx, "error list timelining status: %v", err)
@ -637,7 +598,6 @@ func (s *Surface) timelineStatusUpdateForFollowers(
follow.Account,
status,
stream.TimelineHome,
filters,
)
if err != nil {
log.Errorf(ctx, "error home timelining status: %v", err)
@ -662,7 +622,6 @@ func (s *Surface) listTimelineStatusUpdateForFollow(
ctx context.Context,
status *gtsmodel.Status,
follow *gtsmodel.Follow,
filters []*gtsmodel.Filter,
) (bool, bool, error) {
// Get all lists that contain this given follow.
@ -701,7 +660,6 @@ func (s *Surface) listTimelineStatusUpdateForFollow(
follow.Account,
status,
stream.TimelineList+":"+list.ID, // key streamType to this specific list
filters,
)
if err != nil {
log.Errorf(ctx, "error adding status to list timeline: %v", err)
@ -724,7 +682,6 @@ func (s *Surface) timelineStreamStatusUpdate(
account *gtsmodel.Account,
status *gtsmodel.Status,
streamType string,
filters []*gtsmodel.Filter,
) (bool, error) {
// Convert updated database model to frontend model.
@ -732,14 +689,13 @@ func (s *Surface) timelineStreamStatusUpdate(
status,
account,
gtsmodel.FilterContextHome,
filters,
)
switch {
case err == nil:
// no issue.
case errors.Is(err, statusfilter.ErrHideStatus):
case errors.Is(err, typeutils.ErrHideStatus):
// Don't put this status in the stream.
return false, nil
@ -774,18 +730,11 @@ func (s *Surface) timelineStatusUpdateForTagFollowers(
// Stream the update to the home timeline of each tag follower.
errs := gtserror.MultiError{}
for _, tagFollowerAccount := range tagFollowerAccounts {
filters, err := s.getFilters(ctx, tagFollowerAccount.ID)
if err != nil {
errs.Append(err)
continue
}
if _, err := s.timelineStreamStatusUpdate(
ctx,
tagFollowerAccount,
status,
stream.TimelineHome,
filters,
); err != nil {
errs.Appendf(
"error updating status %s on home timeline for account %s: %w",

3
internal/typeutils/converter.go
View file

@ -27,6 +27,7 @@ import (
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/filter/interaction"
"code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/filter/visibility"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/state"
@ -37,6 +38,7 @@ type Converter struct {
defaultAvatars []string
randAvatars sync.Map
visFilter *visibility.Filter
statusFilter *status.Filter
intFilter *interaction.Filter
randStats atomic.Pointer[apimodel.RandomStats]
}
@ -46,6 +48,7 @@ func NewConverter(state *state.State) *Converter {
state: state,
defaultAvatars: populateDefaultAvatars(),
visFilter: visibility.NewFilter(state),
statusFilter: status.NewFilter(state),
intFilter: interaction.NewFilter(state),
}
}

260
internal/typeutils/internaltofrontend.go
View file

@ -24,14 +24,12 @@ import (
"fmt"
"math"
"slices"
"strconv"
"strings"
"time"
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/db"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/id"
@ -53,6 +51,10 @@ const (
instanceMastodonVersion = "3.5.3"
)
// ErrHideStatus indicates that a status has
// been filtered and should not be returned at all.
var ErrHideStatus = errors.New("hide status")
var instanceStatusesSupportedMimeTypes = []string{
string(apimodel.StatusContentTypePlain),
string(apimodel.StatusContentTypeMarkdown),
@ -849,14 +851,12 @@ func (c *Converter) StatusToAPIStatus(
status *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) (*apimodel.Status, error) {
return c.statusToAPIStatus(
ctx,
status,
requestingAccount,
filterCtx,
filters,
true,
true,
)
@ -871,7 +871,6 @@ func (c *Converter) statusToAPIStatus(
status *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
placeholdAttachments bool,
addPendingNote bool,
) (*apimodel.Status, error) {
@ -880,7 +879,6 @@ func (c *Converter) statusToAPIStatus(
status,
requestingAccount, // Can be nil.
filterCtx, // Can be empty.
filters,
)
if err != nil {
return nil, err
@ -938,103 +936,6 @@ func (c *Converter) statusToAPIStatus(
return apiStatus, nil
}
// statusToAPIFilterResults applies filters and mutes to a status and returns an API filter result object.
// The result may be nil if no filters matched.
// If the status should not be returned at all, it returns the ErrHideStatus error.
func (c *Converter) statusToAPIFilterResults(
ctx context.Context,
s *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) ([]apimodel.FilterResult, error) {
// If there are no filters or mutes, we're done.
// We never hide statuses authored by the requesting account,
// since not being able to see your own posts is confusing.
if filterCtx == 0 || (len(filters) == 0) || s.AccountID == requestingAccount.ID {
return nil, nil
}
// Both mutes and
// filters can expire.
now := time.Now()
// Key this status based on ID + last updated time,
// to ensure we always filter on latest version.
statusKey := s.ID + strconv.FormatInt(s.UpdatedAt().Unix(), 10)
// Check if we have filterable fields cached for this status.
cache := c.state.Caches.StatusesFilterableFields
fields, stored := cache.Get(statusKey)
if !stored {
// We don't have filterable fields
// cached, calculate + cache now.
fields = filterableFields(s)
cache.Set(statusKey, fields)
}
// Record all matching warn filters and the reasons they matched.
filterResults := make([]apimodel.FilterResult, 0, len(filters))
for _, filter := range filters {
if !filter.Contexts.Applies(filterCtx) {
// Filter doesn't apply
// to this context.
continue
}
if filter.Expired(now) {
// Filter doesn't
// apply anymore.
continue
}
// Assemble matching keywords (if any) from this filter.
keywordMatches := make([]string, 0, len(filter.Keywords))
for _, keyword := range filter.Keywords {
// Check if at least one filterable field
// in the status matches on this filter.
if slices.ContainsFunc(
fields,
func(field string) bool {
return keyword.Regexp.MatchString(field)
},
) {
// At least one field matched on this filter.
keywordMatches = append(keywordMatches, keyword.Keyword)
}
}
// A status has only one ID. Not clear
// why this is a list in the Mastodon API.
statusMatches := make([]string, 0, 1)
for _, filterStatus := range filter.Statuses {
if s.ID == filterStatus.StatusID {
statusMatches = append(statusMatches, filterStatus.StatusID)
break
}
}
if len(keywordMatches) > 0 || len(statusMatches) > 0 {
switch filter.Action {
case gtsmodel.FilterActionWarn:
// Record what matched.
filterResults = append(filterResults, apimodel.FilterResult{
Filter: *FilterToAPIFilterV2(filter),
KeywordMatches: keywordMatches,
StatusMatches: statusMatches,
})
case gtsmodel.FilterActionHide:
// Don't show this status. Immediate return.
return nil, statusfilter.ErrHideStatus
}
}
}
return filterResults, nil
}
// StatusToWebStatus converts a gts model status into an
// api representation suitable for serving into a web template.
//
@ -1046,7 +947,6 @@ func (c *Converter) StatusToWebStatus(
apiStatus, err := c.statusToFrontend(ctx, s,
nil, // No authed requester.
gtsmodel.FilterContextNone, // No filters.
nil, // No filters.
)
if err != nil {
return nil, err
@ -1216,7 +1116,6 @@ func (c *Converter) statusToFrontend(
status *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) (
*apimodel.Status,
error,
@ -1225,7 +1124,6 @@ func (c *Converter) statusToFrontend(
status,
requestingAccount,
filterCtx,
filters,
)
if err != nil {
return nil, err
@ -1236,9 +1134,8 @@ func (c *Converter) statusToFrontend(
status.BoostOf,
requestingAccount,
filterCtx,
filters,
)
if errors.Is(err, statusfilter.ErrHideStatus) {
if errors.Is(err, ErrHideStatus) {
// If we'd hide the original status, hide the boost.
return nil, err
} else if err != nil {
@ -1266,10 +1163,9 @@ func (c *Converter) statusToFrontend(
// account to api/web model -- the caller must do that.
func (c *Converter) baseStatusToFrontend(
ctx context.Context,
s *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
status *gtsmodel.Status,
requester *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) (
*apimodel.Status,
error,
@ -1277,12 +1173,12 @@ func (c *Converter) baseStatusToFrontend(
// Try to populate status struct pointer fields.
// We can continue in many cases of partial failure,
// but there are some fields we actually need.
if err := c.state.DB.PopulateStatus(ctx, s); err != nil {
if err := c.state.DB.PopulateStatus(ctx, status); err != nil {
switch {
case s.Account == nil:
case status.Account == nil:
return nil, gtserror.Newf("error(s) populating status, required account not set: %w", err)
case s.BoostOfID != "" && s.BoostOf == nil:
case status.BoostOfID != "" && status.BoostOf == nil:
return nil, gtserror.Newf("error(s) populating status, required boost not set: %w", err)
default:
@ -1290,37 +1186,37 @@ func (c *Converter) baseStatusToFrontend(
}
}
repliesCount, err := c.state.DB.CountStatusReplies(ctx, s.ID)
repliesCount, err := c.state.DB.CountStatusReplies(ctx, status.ID)
if err != nil {
return nil, gtserror.Newf("error counting replies: %w", err)
}
reblogsCount, err := c.state.DB.CountStatusBoosts(ctx, s.ID)
reblogsCount, err := c.state.DB.CountStatusBoosts(ctx, status.ID)
if err != nil {
return nil, gtserror.Newf("error counting reblogs: %w", err)
}
favesCount, err := c.state.DB.CountStatusFaves(ctx, s.ID)
favesCount, err := c.state.DB.CountStatusFaves(ctx, status.ID)
if err != nil {
return nil, gtserror.Newf("error counting faves: %w", err)
}
apiAttachments, err := c.convertAttachmentsToAPIAttachments(ctx, s.Attachments, s.AttachmentIDs)
apiAttachments, err := c.convertAttachmentsToAPIAttachments(ctx, status.Attachments, status.AttachmentIDs)
if err != nil {
log.Errorf(ctx, "error converting status attachments: %v", err)
}
apiMentions, err := c.convertMentionsToAPIMentions(ctx, s.Mentions, s.MentionIDs)
apiMentions, err := c.convertMentionsToAPIMentions(ctx, status.Mentions, status.MentionIDs)
if err != nil {
log.Errorf(ctx, "error converting status mentions: %v", err)
}
apiTags, err := c.convertTagsToAPITags(ctx, s.Tags, s.TagIDs)
apiTags, err := c.convertTagsToAPITags(ctx, status.Tags, status.TagIDs)
if err != nil {
log.Errorf(ctx, "error converting status tags: %v", err)
}
apiEmojis, err := c.convertEmojisToAPIEmojis(ctx, s.Emojis, s.EmojiIDs)
apiEmojis, err := c.convertEmojisToAPIEmojis(ctx, status.Emojis, status.EmojiIDs)
if err != nil {
log.Errorf(ctx, "error converting status emojis: %v", err)
}
@ -1328,32 +1224,30 @@ func (c *Converter) baseStatusToFrontend(
// Take status's interaction policy, or
// fall back to default for its visibility.
var p *gtsmodel.InteractionPolicy
if s.InteractionPolicy != nil {
p = s.InteractionPolicy
} else {
p = gtsmodel.DefaultInteractionPolicyFor(s.Visibility)
if p = status.InteractionPolicy; p == nil {
p = gtsmodel.DefaultInteractionPolicyFor(status.Visibility)
}
apiInteractionPolicy, err := c.InteractionPolicyToAPIInteractionPolicy(ctx, p, s, requestingAccount)
apiInteractionPolicy, err := c.InteractionPolicyToAPIInteractionPolicy(ctx, p, status, requester)
if err != nil {
return nil, gtserror.Newf("error converting interaction policy: %w", err)
}
apiStatus := &apimodel.Status{
ID: s.ID,
CreatedAt: util.FormatISO8601(s.CreatedAt),
ID: status.ID,
CreatedAt: util.FormatISO8601(status.CreatedAt),
InReplyToID: nil, // Set below.
InReplyToAccountID: nil, // Set below.
Sensitive: *s.Sensitive,
Visibility: VisToAPIVis(s.Visibility),
LocalOnly: s.IsLocalOnly(),
Sensitive: *status.Sensitive,
Visibility: VisToAPIVis(status.Visibility),
LocalOnly: status.IsLocalOnly(),
Language: nil, // Set below.
URI: s.URI,
URL: s.URL,
URI: status.URI,
URL: status.URL,
RepliesCount: repliesCount,
ReblogsCount: reblogsCount,
FavouritesCount: favesCount,
Content: s.Content,
Content: status.Content,
Reblog: nil, // Set below.
Application: nil, // Set below.
Account: nil, // Caller must do this.
@ -1362,37 +1256,37 @@ func (c *Converter) baseStatusToFrontend(
Tags: apiTags,
Emojis: apiEmojis,
Card: nil, // TODO: implement cards
Text: s.Text,
ContentType: ContentTypeToAPIContentType(s.ContentType),
Text: status.Text,
ContentType: ContentTypeToAPIContentType(status.ContentType),
InteractionPolicy: *apiInteractionPolicy,
// Mastodon API says spoiler_text should be *text*, not HTML, so
// parse any HTML back to plaintext when serializing via the API,
// attempting to preserve semantic intent to keep it readable.
SpoilerText: text.ParseHTMLToPlain(s.ContentWarning),
SpoilerText: text.ParseHTMLToPlain(status.ContentWarning),
}
if at := s.EditedAt; !at.IsZero() {
if at := status.EditedAt; !at.IsZero() {
timestamp := util.FormatISO8601(at)
apiStatus.EditedAt = util.Ptr(timestamp)
}
apiStatus.InReplyToID = util.PtrIf(s.InReplyToID)
apiStatus.InReplyToAccountID = util.PtrIf(s.InReplyToAccountID)
apiStatus.Language = util.PtrIf(s.Language)
apiStatus.InReplyToID = util.PtrIf(status.InReplyToID)
apiStatus.InReplyToAccountID = util.PtrIf(status.InReplyToAccountID)
apiStatus.Language = util.PtrIf(status.Language)
switch {
case s.CreatedWithApplication != nil:
case status.CreatedWithApplication != nil:
// App exists for this status and is set.
apiStatus.Application, err = c.AppToAPIAppPublic(ctx, s.CreatedWithApplication)
apiStatus.Application, err = c.AppToAPIAppPublic(ctx, status.CreatedWithApplication)
if err != nil {
return nil, gtserror.Newf(
"error converting application %s: %w",
s.CreatedWithApplicationID, err,
status.CreatedWithApplicationID, err,
)
}
case s.CreatedWithApplicationID != "":
case status.CreatedWithApplicationID != "":
// App existed for this status but not
// anymore, it's probably been cleaned up.
// Set a dummy application.
@ -1405,13 +1299,13 @@ func (c *Converter) baseStatusToFrontend(
// status, so nothing to do (app is optional).
}
if s.Poll != nil {
if status.Poll != nil {
// Set originating
// status on the poll.
poll := s.Poll
poll.Status = s
poll := status.Poll
poll.Status = status
apiStatus.Poll, err = c.PollToAPIPoll(ctx, requestingAccount, poll)
apiStatus.Poll, err = c.PollToAPIPoll(ctx, requester, poll)
if err != nil {
return nil, fmt.Errorf("error converting poll: %w", err)
}
@ -1419,15 +1313,15 @@ func (c *Converter) baseStatusToFrontend(
// Status interactions.
//
if s.BoostOf != nil { //nolint
if status.BoostOf != nil { //nolint
// populated *outside* this
// function to prevent recursion.
} else {
interacts, err := c.interactionsWithStatusForAccount(ctx, s, requestingAccount)
interacts, err := c.interactionsWithStatusForAccount(ctx, status, requester)
if err != nil {
log.Errorf(ctx,
"error getting interactions for status %s for account %s: %v",
s.ID, requestingAccount.ID, err,
status.URI, requester.URI, err,
)
// Ensure non-nil object.
@ -1442,21 +1336,24 @@ func (c *Converter) baseStatusToFrontend(
// If web URL is empty for whatever
// reason, provide AP URI as fallback.
if s.URL == "" {
s.URL = s.URI
if apiStatus.URL == "" {
apiStatus.URL = apiStatus.URI
}
// Apply filters.
filterResults, err := c.statusToAPIFilterResults(ctx, s, requestingAccount, filterCtx, filters)
var hide bool
// Pass the status through any stored filters of requesting account's, in context.
apiStatus.Filtered, hide, err = c.statusFilter.StatusFilterResultsInContext(ctx,
requester,
status,
filterCtx,
)
if err != nil {
if errors.Is(err, statusfilter.ErrHideStatus) {
return nil, err
}
return nil, fmt.Errorf("error applying filters: %w", err)
return nil, gtserror.Newf("error filtering status %s: %w", status.URI, err)
} else if hide {
return nil, ErrHideStatus
}
apiStatus.Filtered = filterResults
return apiStatus, nil
}
@ -1968,30 +1865,35 @@ func (c *Converter) RelationshipToAPIRelationship(ctx context.Context, r *gtsmod
// NotificationToAPINotification converts a gts notification into a api notification
func (c *Converter) NotificationToAPINotification(
ctx context.Context,
n *gtsmodel.Notification,
filters []*gtsmodel.Filter,
notif *gtsmodel.Notification,
filter bool,
) (*apimodel.Notification, error) {
// Ensure notif populated.
if err := c.state.DB.PopulateNotification(ctx, n); err != nil {
if err := c.state.DB.PopulateNotification(ctx, notif); err != nil {
return nil, gtserror.Newf("error populating notification: %w", err)
}
// Get account that triggered this notif.
apiAccount, err := c.AccountToAPIAccountPublic(ctx, n.OriginAccount)
apiAccount, err := c.AccountToAPIAccountPublic(ctx, notif.OriginAccount)
if err != nil {
return nil, gtserror.Newf("error converting account to api: %w", err)
}
// Get status that triggered this notif, if set.
var apiStatus *apimodel.Status
if n.Status != nil {
apiStatus, err = c.StatusToAPIStatus(
ctx, n.Status,
n.TargetAccount,
gtsmodel.FilterContextNotifications,
filters,
if notif.Status != nil {
var filterCtx gtsmodel.FilterContext
if filter {
filterCtx = gtsmodel.FilterContextNotifications
}
apiStatus, err = c.StatusToAPIStatus(ctx,
notif.Status,
notif.TargetAccount,
filterCtx,
)
if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
if err != nil && !errors.Is(err, ErrHideStatus) {
return nil, gtserror.Newf("error converting status to api: %w", err)
}
@ -2009,9 +1911,9 @@ func (c *Converter) NotificationToAPINotification(
}
return &apimodel.Notification{
ID: n.ID,
Type: n.NotificationType.String(),
CreatedAt: util.FormatISO8601(n.CreatedAt),
ID: notif.ID,
Type: notif.NotificationType.String(),
CreatedAt: util.FormatISO8601(notif.CreatedAt),
Account: apiAccount,
Status: apiStatus,
}, nil
@ -2040,9 +1942,8 @@ func (c *Converter) ConversationToAPIConversation(
conversation.LastStatus,
requester,
gtsmodel.FilterContextNotifications,
filters,
)
if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
if err != nil && !errors.Is(err, ErrHideStatus) {
return nil, gtserror.Newf(
"error converting status %s to API representation: %w",
conversation.LastStatus.ID,
@ -2309,7 +2210,6 @@ func (c *Converter) ReportToAdminAPIReport(ctx context.Context, r *gtsmodel.Repo
s,
requestingAccount,
gtsmodel.FilterContextNone,
nil, // No filters.
true, // Placehold unknown attachments.
// Don't add note about
@ -3014,7 +2914,6 @@ func (c *Converter) InteractionReqToAPIInteractionReq(
req.Status,
requestingAcct,
gtsmodel.FilterContextNone,
nil, // No filters.
)
if err != nil {
err := gtserror.Newf("error converting interacted status: %w", err)
@ -3028,7 +2927,6 @@ func (c *Converter) InteractionReqToAPIInteractionReq(
req.Reply,
requestingAcct,
gtsmodel.FilterContextNone,
nil, // No filters.
true, // Placehold unknown attachments.
// Don't add note about pending;

66
internal/typeutils/internaltofrontend_test.go
View file

@ -27,8 +27,9 @@ import (
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/db"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/id"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
"code.superseriousbusiness.org/gotosocial/internal/util"
"code.superseriousbusiness.org/gotosocial/testrig"
"github.com/stretchr/testify/suite"
@ -465,7 +466,7 @@ func (suite *InternalToFrontendTestSuite) TestLocalInstanceAccountToFrontendBloc
func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
testStatus := suite.testStatuses["admin_account_status_1"]
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -628,7 +629,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendHTMLContentWarning
testStatus.ContentWarning = `<p>First paragraph of content warning</p><h4>Here's the title!</h4><p></p><p>Big boobs<br>Tee hee!<br><br>Some more text<br>And a bunch more<br><br>Hasta la victoria siempre!</p>`
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -794,7 +795,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted
}
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(ctx, testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(ctx, testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -952,6 +953,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted
// Modify a fixture status into a status that should be filtered,
// and then filter it, returning the API status or any error from converting it.
func (suite *InternalToFrontendTestSuite) filteredStatusToFrontend(action gtsmodel.FilterAction, boost bool) (*apimodel.Status, error) {
ctx := suite.T().Context()
testStatus := suite.testStatuses["admin_account_status_1"]
testStatus.Content += " fnord"
testStatus.Text += " fnord"
@ -969,19 +972,14 @@ func (suite *InternalToFrontendTestSuite) filteredStatusToFrontend(action gtsmod
expectedMatchingFilter := suite.testFilters["local_account_1_filter_1"]
expectedMatchingFilter.Action = action
expectedMatchingFilterKeyword := suite.testFilterKeywords["local_account_1_filter_1_keyword_1"]
suite.NoError(expectedMatchingFilterKeyword.Compile())
expectedMatchingFilter.Keywords = []*gtsmodel.FilterKeyword{expectedMatchingFilterKeyword}
requestingAccountFilters := []*gtsmodel.Filter{expectedMatchingFilter}
err := suite.state.DB.UpdateFilter(ctx, expectedMatchingFilter, "action")
suite.NoError(err)
return suite.typeconverter.StatusToAPIStatus(
suite.T().Context(),
testStatus,
requestingAccount,
gtsmodel.FilterContextHome,
requestingAccountFilters,
)
}
@ -1480,17 +1478,19 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() {
// Test that a status which is filtered with a hide filter by the requesting user results in the ErrHideStatus error.
func (suite *InternalToFrontendTestSuite) TestHideFilteredStatusToFrontend() {
_, err := suite.filteredStatusToFrontend(gtsmodel.FilterActionHide, false)
suite.ErrorIs(err, statusfilter.ErrHideStatus)
suite.ErrorIs(err, typeutils.ErrHideStatus)
}
// Test that a status which is filtered with a hide filter by the requesting user results in the ErrHideStatus error for a boost of that status.
func (suite *InternalToFrontendTestSuite) TestHideFilteredBoostToFrontend() {
_, err := suite.filteredStatusToFrontend(gtsmodel.FilterActionHide, true)
suite.ErrorIs(err, statusfilter.ErrHideStatus)
suite.ErrorIs(err, typeutils.ErrHideStatus)
}
// Test that a hashtag filter for a hashtag in Mastodon HTML content works the way most users would expect.
func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wholeWord bool, boost bool) {
ctx := suite.T().Context()
testStatus := new(gtsmodel.Status)
*testStatus = *suite.testStatuses["admin_account_status_1"]
testStatus.Content = `<p>doggo doggin' it</p><p><a href="https://example.test/tags/dogsofmastodon" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>dogsofmastodon</span></a></p>`
@ -1508,29 +1508,38 @@ func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wh
testStatus = boost
}
var err error
requestingAccount := suite.testAccounts["local_account_1"]
filterKeyword := &gtsmodel.FilterKeyword{
Keyword: "#dogsofmastodon",
WholeWord: &wholeWord,
Regexp: nil,
}
if err := filterKeyword.Compile(); err != nil {
suite.FailNow(err.Error())
filter := &gtsmodel.Filter{
ID: id.NewULID(),
Title: id.NewULID(),
AccountID: requestingAccount.ID,
Action: gtsmodel.FilterActionWarn,
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome),
}
filter := &gtsmodel.Filter{
Action: gtsmodel.FilterActionWarn,
Keywords: []*gtsmodel.FilterKeyword{filterKeyword},
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome),
filterKeyword := &gtsmodel.FilterKeyword{
ID: id.NewULID(),
FilterID: filter.ID,
Keyword: "#dogsofmastodon",
WholeWord: &wholeWord,
}
filter.KeywordIDs = []string{filterKeyword.ID}
err = suite.state.DB.PutFilterKeyword(ctx, filterKeyword)
suite.NoError(err)
err = suite.state.DB.PutFilter(ctx, filter)
suite.NoError(err)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(
suite.T().Context(),
testStatus,
requestingAccount,
gtsmodel.FilterContextHome,
[]*gtsmodel.Filter{filter},
)
if err != nil {
suite.FailNow(err.Error())
@ -1559,7 +1568,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments
testStatus := suite.testStatuses["remote_account_2_status_1"]
requestingAccount := suite.testAccounts["admin_account"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -1886,7 +1895,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
*testStatus = *suite.testStatuses["admin_account_status_1"]
testStatus.Language = ""
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -2047,7 +2056,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendPartialInteraction
*testStatus = *suite.testStatuses["local_account_1_status_3"]
testStatus.Language = ""
requestingAccount := suite.testAccounts["admin_account"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -2161,7 +2170,6 @@ func (suite *InternalToFrontendTestSuite) TestStatusToAPIStatusPendingApproval()
testStatus,
requestingAccount,
gtsmodel.FilterContextNone,
nil,
)
if err != nil {
suite.FailNow(err.Error())

57
internal/typeutils/util.go
View file

@ -350,60 +350,3 @@ func ContentToContentLanguage(
return contentStr, langTagStr
}
// filterableFields returns text fields from
// a status that we might want to filter on:
//
// - content warning
// - content (converted to plaintext from HTML)
// - media descriptions
// - poll options
//
// Each field should be filtered separately.
// This avoids scenarios where false-positive
// multiple-word matches can be made by matching
// the last word of one field + the first word
// of the next field together.
func filterableFields(s *gtsmodel.Status) []string {
// Estimate length of fields.
fieldCount := 2 + len(s.Attachments)
if s.Poll != nil {
fieldCount += len(s.Poll.Options)
}
fields := make([]string, 0, fieldCount)
// Content warning / title.
if s.ContentWarning != "" {
fields = append(fields, s.ContentWarning)
}
// Status content. Though we have raw text
// available for statuses created on our
// instance, use the plaintext version to
// remove markdown-formatting characters
// and ensure more consistent filtering.
if s.Content != "" {
text := text.ParseHTMLToPlain(s.Content)
if text != "" {
fields = append(fields, text)
}
}
// Media descriptions.
for _, attachment := range s.Attachments {
if attachment.Description != "" {
fields = append(fields, attachment.Description)
}
}
// Poll options.
if s.Poll != nil {
for _, opt := range s.Poll.Options {
if opt != "" {
fields = append(fields, opt)
}
}
}
return fields
}

60
internal/typeutils/util_test.go
View file

@ -23,7 +23,6 @@ import (
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/language"
"github.com/stretchr/testify/assert"
)
func TestMisskeyReportContentURLs1(t *testing.T) {
@ -157,62 +156,3 @@ func TestContentToContentLanguage(t *testing.T) {
}
}
}
func TestFilterableText(t *testing.T) {
type testcase struct {
status *gtsmodel.Status
expectedFields []string
}
for _, testcase := range []testcase{
{
status: &gtsmodel.Status{
ContentWarning: "This is a test status",
Content: `<p>Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> instance.</p>`,
},
expectedFields: []string{
"This is a test status",
"Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> instance.",
},
},
{
status: &gtsmodel.Status{
Content: `<p><span class="h-card"><a href="https://example.org/@zlatko" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>zlatko</span></a></span> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)</p><p><a href="https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863" rel="nofollow noreferrer noopener" target="_blank">https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863</a></p>`,
},
expectedFields: []string{
"@zlatko <https://example.org/@zlatko> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)\n\nhttps://codeberg.org/superseriousbusiness/gotosocial/pulls/2863 <https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863>",
},
},
{
status: &gtsmodel.Status{
ContentWarning: "Nerd stuff",
Content: `<p>Latest graphs for <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> on <a href="https://github.com/ncruces/go-sqlite3" rel="nofollow noreferrer noopener" target="_blank">Wasm sqlite3</a> with <a href="https://codeberg.org/gruf/go-ffmpreg" rel="nofollow noreferrer noopener" target="_blank">embedded Wasm ffmpeg</a>, both running on <a href="https://wazero.io/" rel="nofollow noreferrer noopener" target="_blank">Wazero</a>, and configured with a <a href="https://codeberg.org/superseriousbusiness/gotosocial/src/commit/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266" rel="nofollow noreferrer noopener" target="_blank">50MiB db cache target</a>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.</p>`,
Attachments: []*gtsmodel.MediaAttachment{
{
Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`,
},
{
Description: `Another media attachment`,
},
},
Poll: &gtsmodel.Poll{
Options: []string{
"Poll option 1",
"Poll option 2",
},
},
},
expectedFields: []string{
"Nerd stuff",
"Latest graphs for #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> on Wasm sqlite3 <https://github.com/ncruces/go-sqlite3> with embedded Wasm ffmpeg <https://codeberg.org/gruf/go-ffmpreg>, both running on Wazero <https://wazero.io/>, and configured with a 50MiB db cache target <https://codeberg.org/superseriousbusiness/gotosocial/src/commit/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.",
"Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.",
"Another media attachment",
"Poll option 1",
"Poll option 2",
},
},
} {
fields := filterableFields(testcase.status)
assert.Equal(t, testcase.expectedFields, fields)
}
}

2
internal/webpush/realsender_test.go
View file

@ -190,7 +190,7 @@ func (suite *RealSenderStandardTestSuite) simulatePushNotification(
}, nil
}
apiNotif, err := suite.typeconverter.NotificationToAPINotification(ctx, notification, nil)
apiNotif, err := suite.typeconverter.NotificationToAPINotification(ctx, notification, false)
suite.NoError(err)
// Send the push notification.