mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 08:02:26 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[performance] add statusfilter cache to cache calculated status filtering results (#4303)

Browse source 
this adds another 'filter' type cache, similar to the visibility and mute caches, to cache the results of status filtering checks. for the moment this keeps all the check calls themselves within the frontend typeconversion code, but i may move this out of the typeconverter in a future PR (also removing the ErrHideStatus means of propagating a hidden status).

also tweaks some of the cache invalidation hooks to not make unnecessary calls.

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4303
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2025-07-01 16:00:04 +02:00 • committed by kim
commit 4f2aa792b3
50 changed files with 1017 additions and 544 deletions
Download patch file Download diff file Expand all files Collapse all files

3
internal/typeutils/converter.go
View file

@ -27,6 +27,7 @@ import (
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/filter/interaction"
"code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/filter/visibility"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/state"
@ -37,6 +38,7 @@ type Converter struct {
defaultAvatars []string
randAvatars sync.Map
visFilter *visibility.Filter
statusFilter *status.Filter
intFilter *interaction.Filter
randStats atomic.Pointer[apimodel.RandomStats]
}
@ -46,6 +48,7 @@ func NewConverter(state *state.State) *Converter {
state: state,
defaultAvatars: populateDefaultAvatars(),
visFilter: visibility.NewFilter(state),
statusFilter: status.NewFilter(state),
intFilter: interaction.NewFilter(state),
}
}

260
internal/typeutils/internaltofrontend.go
View file

@ -24,14 +24,12 @@ import (
"fmt"
"math"
"slices"
"strconv"
"strings"
"time"
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/db"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/id"
@ -53,6 +51,10 @@ const (
instanceMastodonVersion = "3.5.3"
)
// ErrHideStatus indicates that a status has
// been filtered and should not be returned at all.
var ErrHideStatus = errors.New("hide status")
var instanceStatusesSupportedMimeTypes = []string{
string(apimodel.StatusContentTypePlain),
string(apimodel.StatusContentTypeMarkdown),
@ -849,14 +851,12 @@ func (c *Converter) StatusToAPIStatus(
status *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) (*apimodel.Status, error) {
return c.statusToAPIStatus(
ctx,
status,
requestingAccount,
filterCtx,
filters,
true,
true,
)
@ -871,7 +871,6 @@ func (c *Converter) statusToAPIStatus(
status *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
placeholdAttachments bool,
addPendingNote bool,
) (*apimodel.Status, error) {
@ -880,7 +879,6 @@ func (c *Converter) statusToAPIStatus(
status,
requestingAccount, // Can be nil.
filterCtx, // Can be empty.
filters,
)
if err != nil {
return nil, err
@ -938,103 +936,6 @@ func (c *Converter) statusToAPIStatus(
return apiStatus, nil
}
// statusToAPIFilterResults applies filters and mutes to a status and returns an API filter result object.
// The result may be nil if no filters matched.
// If the status should not be returned at all, it returns the ErrHideStatus error.
func (c *Converter) statusToAPIFilterResults(
ctx context.Context,
s *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) ([]apimodel.FilterResult, error) {
// If there are no filters or mutes, we're done.
// We never hide statuses authored by the requesting account,
// since not being able to see your own posts is confusing.
if filterCtx == 0 || (len(filters) == 0) || s.AccountID == requestingAccount.ID {
return nil, nil
}
// Both mutes and
// filters can expire.
now := time.Now()
// Key this status based on ID + last updated time,
// to ensure we always filter on latest version.
statusKey := s.ID + strconv.FormatInt(s.UpdatedAt().Unix(), 10)
// Check if we have filterable fields cached for this status.
cache := c.state.Caches.StatusesFilterableFields
fields, stored := cache.Get(statusKey)
if !stored {
// We don't have filterable fields
// cached, calculate + cache now.
fields = filterableFields(s)
cache.Set(statusKey, fields)
}
// Record all matching warn filters and the reasons they matched.
filterResults := make([]apimodel.FilterResult, 0, len(filters))
for _, filter := range filters {
if !filter.Contexts.Applies(filterCtx) {
// Filter doesn't apply
// to this context.
continue
}
if filter.Expired(now) {
// Filter doesn't
// apply anymore.
continue
}
// Assemble matching keywords (if any) from this filter.
keywordMatches := make([]string, 0, len(filter.Keywords))
for _, keyword := range filter.Keywords {
// Check if at least one filterable field
// in the status matches on this filter.
if slices.ContainsFunc(
fields,
func(field string) bool {
return keyword.Regexp.MatchString(field)
},
) {
// At least one field matched on this filter.
keywordMatches = append(keywordMatches, keyword.Keyword)
}
}
// A status has only one ID. Not clear
// why this is a list in the Mastodon API.
statusMatches := make([]string, 0, 1)
for _, filterStatus := range filter.Statuses {
if s.ID == filterStatus.StatusID {
statusMatches = append(statusMatches, filterStatus.StatusID)
break
}
}
if len(keywordMatches) > 0 || len(statusMatches) > 0 {
switch filter.Action {
case gtsmodel.FilterActionWarn:
// Record what matched.
filterResults = append(filterResults, apimodel.FilterResult{
Filter: *FilterToAPIFilterV2(filter),
KeywordMatches: keywordMatches,
StatusMatches: statusMatches,
})
case gtsmodel.FilterActionHide:
// Don't show this status. Immediate return.
return nil, statusfilter.ErrHideStatus
}
}
}
return filterResults, nil
}
// StatusToWebStatus converts a gts model status into an
// api representation suitable for serving into a web template.
//
@ -1046,7 +947,6 @@ func (c *Converter) StatusToWebStatus(
apiStatus, err := c.statusToFrontend(ctx, s,
nil, // No authed requester.
gtsmodel.FilterContextNone, // No filters.
nil, // No filters.
)
if err != nil {
return nil, err
@ -1216,7 +1116,6 @@ func (c *Converter) statusToFrontend(
status *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) (
*apimodel.Status,
error,
@ -1225,7 +1124,6 @@ func (c *Converter) statusToFrontend(
status,
requestingAccount,
filterCtx,
filters,
)
if err != nil {
return nil, err
@ -1236,9 +1134,8 @@ func (c *Converter) statusToFrontend(
status.BoostOf,
requestingAccount,
filterCtx,
filters,
)
if errors.Is(err, statusfilter.ErrHideStatus) {
if errors.Is(err, ErrHideStatus) {
// If we'd hide the original status, hide the boost.
return nil, err
} else if err != nil {
@ -1266,10 +1163,9 @@ func (c *Converter) statusToFrontend(
// account to api/web model -- the caller must do that.
func (c *Converter) baseStatusToFrontend(
ctx context.Context,
s *gtsmodel.Status,
requestingAccount *gtsmodel.Account,
status *gtsmodel.Status,
requester *gtsmodel.Account,
filterCtx gtsmodel.FilterContext,
filters []*gtsmodel.Filter,
) (
*apimodel.Status,
error,
@ -1277,12 +1173,12 @@ func (c *Converter) baseStatusToFrontend(
// Try to populate status struct pointer fields.
// We can continue in many cases of partial failure,
// but there are some fields we actually need.
if err := c.state.DB.PopulateStatus(ctx, s); err != nil {
if err := c.state.DB.PopulateStatus(ctx, status); err != nil {
switch {
case s.Account == nil:
case status.Account == nil:
return nil, gtserror.Newf("error(s) populating status, required account not set: %w", err)
case s.BoostOfID != "" && s.BoostOf == nil:
case status.BoostOfID != "" && status.BoostOf == nil:
return nil, gtserror.Newf("error(s) populating status, required boost not set: %w", err)
default:
@ -1290,37 +1186,37 @@ func (c *Converter) baseStatusToFrontend(
}
}
repliesCount, err := c.state.DB.CountStatusReplies(ctx, s.ID)
repliesCount, err := c.state.DB.CountStatusReplies(ctx, status.ID)
if err != nil {
return nil, gtserror.Newf("error counting replies: %w", err)
}
reblogsCount, err := c.state.DB.CountStatusBoosts(ctx, s.ID)
reblogsCount, err := c.state.DB.CountStatusBoosts(ctx, status.ID)
if err != nil {
return nil, gtserror.Newf("error counting reblogs: %w", err)
}
favesCount, err := c.state.DB.CountStatusFaves(ctx, s.ID)
favesCount, err := c.state.DB.CountStatusFaves(ctx, status.ID)
if err != nil {
return nil, gtserror.Newf("error counting faves: %w", err)
}
apiAttachments, err := c.convertAttachmentsToAPIAttachments(ctx, s.Attachments, s.AttachmentIDs)
apiAttachments, err := c.convertAttachmentsToAPIAttachments(ctx, status.Attachments, status.AttachmentIDs)
if err != nil {
log.Errorf(ctx, "error converting status attachments: %v", err)
}
apiMentions, err := c.convertMentionsToAPIMentions(ctx, s.Mentions, s.MentionIDs)
apiMentions, err := c.convertMentionsToAPIMentions(ctx, status.Mentions, status.MentionIDs)
if err != nil {
log.Errorf(ctx, "error converting status mentions: %v", err)
}
apiTags, err := c.convertTagsToAPITags(ctx, s.Tags, s.TagIDs)
apiTags, err := c.convertTagsToAPITags(ctx, status.Tags, status.TagIDs)
if err != nil {
log.Errorf(ctx, "error converting status tags: %v", err)
}
apiEmojis, err := c.convertEmojisToAPIEmojis(ctx, s.Emojis, s.EmojiIDs)
apiEmojis, err := c.convertEmojisToAPIEmojis(ctx, status.Emojis, status.EmojiIDs)
if err != nil {
log.Errorf(ctx, "error converting status emojis: %v", err)
}
@ -1328,32 +1224,30 @@ func (c *Converter) baseStatusToFrontend(
// Take status's interaction policy, or
// fall back to default for its visibility.
var p *gtsmodel.InteractionPolicy
if s.InteractionPolicy != nil {
p = s.InteractionPolicy
} else {
p = gtsmodel.DefaultInteractionPolicyFor(s.Visibility)
if p = status.InteractionPolicy; p == nil {
p = gtsmodel.DefaultInteractionPolicyFor(status.Visibility)
}
apiInteractionPolicy, err := c.InteractionPolicyToAPIInteractionPolicy(ctx, p, s, requestingAccount)
apiInteractionPolicy, err := c.InteractionPolicyToAPIInteractionPolicy(ctx, p, status, requester)
if err != nil {
return nil, gtserror.Newf("error converting interaction policy: %w", err)
}
apiStatus := &apimodel.Status{
ID: s.ID,
CreatedAt: util.FormatISO8601(s.CreatedAt),
ID: status.ID,
CreatedAt: util.FormatISO8601(status.CreatedAt),
InReplyToID: nil, // Set below.
InReplyToAccountID: nil, // Set below.
Sensitive: *s.Sensitive,
Visibility: VisToAPIVis(s.Visibility),
LocalOnly: s.IsLocalOnly(),
Sensitive: *status.Sensitive,
Visibility: VisToAPIVis(status.Visibility),
LocalOnly: status.IsLocalOnly(),
Language: nil, // Set below.
URI: s.URI,
URL: s.URL,
URI: status.URI,
URL: status.URL,
RepliesCount: repliesCount,
ReblogsCount: reblogsCount,
FavouritesCount: favesCount,
Content: s.Content,
Content: status.Content,
Reblog: nil, // Set below.
Application: nil, // Set below.
Account: nil, // Caller must do this.
@ -1362,37 +1256,37 @@ func (c *Converter) baseStatusToFrontend(
Tags: apiTags,
Emojis: apiEmojis,
Card: nil, // TODO: implement cards
Text: s.Text,
ContentType: ContentTypeToAPIContentType(s.ContentType),
Text: status.Text,
ContentType: ContentTypeToAPIContentType(status.ContentType),
InteractionPolicy: *apiInteractionPolicy,
// Mastodon API says spoiler_text should be *text*, not HTML, so
// parse any HTML back to plaintext when serializing via the API,
// attempting to preserve semantic intent to keep it readable.
SpoilerText: text.ParseHTMLToPlain(s.ContentWarning),
SpoilerText: text.ParseHTMLToPlain(status.ContentWarning),
}
if at := s.EditedAt; !at.IsZero() {
if at := status.EditedAt; !at.IsZero() {
timestamp := util.FormatISO8601(at)
apiStatus.EditedAt = util.Ptr(timestamp)
}
apiStatus.InReplyToID = util.PtrIf(s.InReplyToID)
apiStatus.InReplyToAccountID = util.PtrIf(s.InReplyToAccountID)
apiStatus.Language = util.PtrIf(s.Language)
apiStatus.InReplyToID = util.PtrIf(status.InReplyToID)
apiStatus.InReplyToAccountID = util.PtrIf(status.InReplyToAccountID)
apiStatus.Language = util.PtrIf(status.Language)
switch {
case s.CreatedWithApplication != nil:
case status.CreatedWithApplication != nil:
// App exists for this status and is set.
apiStatus.Application, err = c.AppToAPIAppPublic(ctx, s.CreatedWithApplication)
apiStatus.Application, err = c.AppToAPIAppPublic(ctx, status.CreatedWithApplication)
if err != nil {
return nil, gtserror.Newf(
"error converting application %s: %w",
s.CreatedWithApplicationID, err,
status.CreatedWithApplicationID, err,
)
}
case s.CreatedWithApplicationID != "":
case status.CreatedWithApplicationID != "":
// App existed for this status but not
// anymore, it's probably been cleaned up.
// Set a dummy application.
@ -1405,13 +1299,13 @@ func (c *Converter) baseStatusToFrontend(
// status, so nothing to do (app is optional).
}
if s.Poll != nil {
if status.Poll != nil {
// Set originating
// status on the poll.
poll := s.Poll
poll.Status = s
poll := status.Poll
poll.Status = status
apiStatus.Poll, err = c.PollToAPIPoll(ctx, requestingAccount, poll)
apiStatus.Poll, err = c.PollToAPIPoll(ctx, requester, poll)
if err != nil {
return nil, fmt.Errorf("error converting poll: %w", err)
}
@ -1419,15 +1313,15 @@ func (c *Converter) baseStatusToFrontend(
// Status interactions.
//
if s.BoostOf != nil { //nolint
if status.BoostOf != nil { //nolint
// populated *outside* this
// function to prevent recursion.
} else {
interacts, err := c.interactionsWithStatusForAccount(ctx, s, requestingAccount)
interacts, err := c.interactionsWithStatusForAccount(ctx, status, requester)
if err != nil {
log.Errorf(ctx,
"error getting interactions for status %s for account %s: %v",
s.ID, requestingAccount.ID, err,
status.URI, requester.URI, err,
)
// Ensure non-nil object.
@ -1442,21 +1336,24 @@ func (c *Converter) baseStatusToFrontend(
// If web URL is empty for whatever
// reason, provide AP URI as fallback.
if s.URL == "" {
s.URL = s.URI
if apiStatus.URL == "" {
apiStatus.URL = apiStatus.URI
}
// Apply filters.
filterResults, err := c.statusToAPIFilterResults(ctx, s, requestingAccount, filterCtx, filters)
var hide bool
// Pass the status through any stored filters of requesting account's, in context.
apiStatus.Filtered, hide, err = c.statusFilter.StatusFilterResultsInContext(ctx,
requester,
status,
filterCtx,
)
if err != nil {
if errors.Is(err, statusfilter.ErrHideStatus) {
return nil, err
}
return nil, fmt.Errorf("error applying filters: %w", err)
return nil, gtserror.Newf("error filtering status %s: %w", status.URI, err)
} else if hide {
return nil, ErrHideStatus
}
apiStatus.Filtered = filterResults
return apiStatus, nil
}
@ -1968,30 +1865,35 @@ func (c *Converter) RelationshipToAPIRelationship(ctx context.Context, r *gtsmod
// NotificationToAPINotification converts a gts notification into a api notification
func (c *Converter) NotificationToAPINotification(
ctx context.Context,
n *gtsmodel.Notification,
filters []*gtsmodel.Filter,
notif *gtsmodel.Notification,
filter bool,
) (*apimodel.Notification, error) {
// Ensure notif populated.
if err := c.state.DB.PopulateNotification(ctx, n); err != nil {
if err := c.state.DB.PopulateNotification(ctx, notif); err != nil {
return nil, gtserror.Newf("error populating notification: %w", err)
}
// Get account that triggered this notif.
apiAccount, err := c.AccountToAPIAccountPublic(ctx, n.OriginAccount)
apiAccount, err := c.AccountToAPIAccountPublic(ctx, notif.OriginAccount)
if err != nil {
return nil, gtserror.Newf("error converting account to api: %w", err)
}
// Get status that triggered this notif, if set.
var apiStatus *apimodel.Status
if n.Status != nil {
apiStatus, err = c.StatusToAPIStatus(
ctx, n.Status,
n.TargetAccount,
gtsmodel.FilterContextNotifications,
filters,
if notif.Status != nil {
var filterCtx gtsmodel.FilterContext
if filter {
filterCtx = gtsmodel.FilterContextNotifications
}
apiStatus, err = c.StatusToAPIStatus(ctx,
notif.Status,
notif.TargetAccount,
filterCtx,
)
if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
if err != nil && !errors.Is(err, ErrHideStatus) {
return nil, gtserror.Newf("error converting status to api: %w", err)
}
@ -2009,9 +1911,9 @@ func (c *Converter) NotificationToAPINotification(
}
return &apimodel.Notification{
ID: n.ID,
Type: n.NotificationType.String(),
CreatedAt: util.FormatISO8601(n.CreatedAt),
ID: notif.ID,
Type: notif.NotificationType.String(),
CreatedAt: util.FormatISO8601(notif.CreatedAt),
Account: apiAccount,
Status: apiStatus,
}, nil
@ -2040,9 +1942,8 @@ func (c *Converter) ConversationToAPIConversation(
conversation.LastStatus,
requester,
gtsmodel.FilterContextNotifications,
filters,
)
if err != nil && !errors.Is(err, statusfilter.ErrHideStatus) {
if err != nil && !errors.Is(err, ErrHideStatus) {
return nil, gtserror.Newf(
"error converting status %s to API representation: %w",
conversation.LastStatus.ID,
@ -2309,7 +2210,6 @@ func (c *Converter) ReportToAdminAPIReport(ctx context.Context, r *gtsmodel.Repo
s,
requestingAccount,
gtsmodel.FilterContextNone,
nil, // No filters.
true, // Placehold unknown attachments.
// Don't add note about
@ -3014,7 +2914,6 @@ func (c *Converter) InteractionReqToAPIInteractionReq(
req.Status,
requestingAcct,
gtsmodel.FilterContextNone,
nil, // No filters.
)
if err != nil {
err := gtserror.Newf("error converting interacted status: %w", err)
@ -3028,7 +2927,6 @@ func (c *Converter) InteractionReqToAPIInteractionReq(
req.Reply,
requestingAcct,
gtsmodel.FilterContextNone,
nil, // No filters.
true, // Placehold unknown attachments.
// Don't add note about pending;

66
internal/typeutils/internaltofrontend_test.go
View file

@ -27,8 +27,9 @@ import (
apimodel "code.superseriousbusiness.org/gotosocial/internal/api/model"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/db"
statusfilter "code.superseriousbusiness.org/gotosocial/internal/filter/status"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/id"
"code.superseriousbusiness.org/gotosocial/internal/typeutils"
"code.superseriousbusiness.org/gotosocial/internal/util"
"code.superseriousbusiness.org/gotosocial/testrig"
"github.com/stretchr/testify/suite"
@ -465,7 +466,7 @@ func (suite *InternalToFrontendTestSuite) TestLocalInstanceAccountToFrontendBloc
func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
testStatus := suite.testStatuses["admin_account_status_1"]
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -628,7 +629,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendHTMLContentWarning
testStatus.ContentWarning = `<p>First paragraph of content warning</p><h4>Here's the title!</h4><p></p><p>Big boobs<br>Tee hee!<br><br>Some more text<br>And a bunch more<br><br>Hasta la victoria siempre!</p>`
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -794,7 +795,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted
}
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(ctx, testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(ctx, testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -952,6 +953,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted
// Modify a fixture status into a status that should be filtered,
// and then filter it, returning the API status or any error from converting it.
func (suite *InternalToFrontendTestSuite) filteredStatusToFrontend(action gtsmodel.FilterAction, boost bool) (*apimodel.Status, error) {
ctx := suite.T().Context()
testStatus := suite.testStatuses["admin_account_status_1"]
testStatus.Content += " fnord"
testStatus.Text += " fnord"
@ -969,19 +972,14 @@ func (suite *InternalToFrontendTestSuite) filteredStatusToFrontend(action gtsmod
expectedMatchingFilter := suite.testFilters["local_account_1_filter_1"]
expectedMatchingFilter.Action = action
expectedMatchingFilterKeyword := suite.testFilterKeywords["local_account_1_filter_1_keyword_1"]
suite.NoError(expectedMatchingFilterKeyword.Compile())
expectedMatchingFilter.Keywords = []*gtsmodel.FilterKeyword{expectedMatchingFilterKeyword}
requestingAccountFilters := []*gtsmodel.Filter{expectedMatchingFilter}
err := suite.state.DB.UpdateFilter(ctx, expectedMatchingFilter, "action")
suite.NoError(err)
return suite.typeconverter.StatusToAPIStatus(
suite.T().Context(),
testStatus,
requestingAccount,
gtsmodel.FilterContextHome,
requestingAccountFilters,
)
}
@ -1480,17 +1478,19 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() {
// Test that a status which is filtered with a hide filter by the requesting user results in the ErrHideStatus error.
func (suite *InternalToFrontendTestSuite) TestHideFilteredStatusToFrontend() {
_, err := suite.filteredStatusToFrontend(gtsmodel.FilterActionHide, false)
suite.ErrorIs(err, statusfilter.ErrHideStatus)
suite.ErrorIs(err, typeutils.ErrHideStatus)
}
// Test that a status which is filtered with a hide filter by the requesting user results in the ErrHideStatus error for a boost of that status.
func (suite *InternalToFrontendTestSuite) TestHideFilteredBoostToFrontend() {
_, err := suite.filteredStatusToFrontend(gtsmodel.FilterActionHide, true)
suite.ErrorIs(err, statusfilter.ErrHideStatus)
suite.ErrorIs(err, typeutils.ErrHideStatus)
}
// Test that a hashtag filter for a hashtag in Mastodon HTML content works the way most users would expect.
func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wholeWord bool, boost bool) {
ctx := suite.T().Context()
testStatus := new(gtsmodel.Status)
*testStatus = *suite.testStatuses["admin_account_status_1"]
testStatus.Content = `<p>doggo doggin' it</p><p><a href="https://example.test/tags/dogsofmastodon" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>dogsofmastodon</span></a></p>`
@ -1508,29 +1508,38 @@ func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wh
testStatus = boost
}
var err error
requestingAccount := suite.testAccounts["local_account_1"]
filterKeyword := &gtsmodel.FilterKeyword{
Keyword: "#dogsofmastodon",
WholeWord: &wholeWord,
Regexp: nil,
}
if err := filterKeyword.Compile(); err != nil {
suite.FailNow(err.Error())
filter := &gtsmodel.Filter{
ID: id.NewULID(),
Title: id.NewULID(),
AccountID: requestingAccount.ID,
Action: gtsmodel.FilterActionWarn,
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome),
}
filter := &gtsmodel.Filter{
Action: gtsmodel.FilterActionWarn,
Keywords: []*gtsmodel.FilterKeyword{filterKeyword},
Contexts: gtsmodel.FilterContexts(gtsmodel.FilterContextHome),
filterKeyword := &gtsmodel.FilterKeyword{
ID: id.NewULID(),
FilterID: filter.ID,
Keyword: "#dogsofmastodon",
WholeWord: &wholeWord,
}
filter.KeywordIDs = []string{filterKeyword.ID}
err = suite.state.DB.PutFilterKeyword(ctx, filterKeyword)
suite.NoError(err)
err = suite.state.DB.PutFilter(ctx, filter)
suite.NoError(err)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(
suite.T().Context(),
testStatus,
requestingAccount,
gtsmodel.FilterContextHome,
[]*gtsmodel.Filter{filter},
)
if err != nil {
suite.FailNow(err.Error())
@ -1559,7 +1568,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownAttachments
testStatus := suite.testStatuses["remote_account_2_status_1"]
requestingAccount := suite.testAccounts["admin_account"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -1886,7 +1895,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
*testStatus = *suite.testStatuses["admin_account_status_1"]
testStatus.Language = ""
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -2047,7 +2056,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendPartialInteraction
*testStatus = *suite.testStatuses["local_account_1_status_3"]
testStatus.Language = ""
requestingAccount := suite.testAccounts["admin_account"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone, nil)
apiStatus, err := suite.typeconverter.StatusToAPIStatus(suite.T().Context(), testStatus, requestingAccount, gtsmodel.FilterContextNone)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
@ -2161,7 +2170,6 @@ func (suite *InternalToFrontendTestSuite) TestStatusToAPIStatusPendingApproval()
testStatus,
requestingAccount,
gtsmodel.FilterContextNone,
nil,
)
if err != nil {
suite.FailNow(err.Error())

57
internal/typeutils/util.go
View file

@ -350,60 +350,3 @@ func ContentToContentLanguage(
return contentStr, langTagStr
}
// filterableFields returns text fields from
// a status that we might want to filter on:
//
// - content warning
// - content (converted to plaintext from HTML)
// - media descriptions
// - poll options
//
// Each field should be filtered separately.
// This avoids scenarios where false-positive
// multiple-word matches can be made by matching
// the last word of one field + the first word
// of the next field together.
func filterableFields(s *gtsmodel.Status) []string {
// Estimate length of fields.
fieldCount := 2 + len(s.Attachments)
if s.Poll != nil {
fieldCount += len(s.Poll.Options)
}
fields := make([]string, 0, fieldCount)
// Content warning / title.
if s.ContentWarning != "" {
fields = append(fields, s.ContentWarning)
}
// Status content. Though we have raw text
// available for statuses created on our
// instance, use the plaintext version to
// remove markdown-formatting characters
// and ensure more consistent filtering.
if s.Content != "" {
text := text.ParseHTMLToPlain(s.Content)
if text != "" {
fields = append(fields, text)
}
}
// Media descriptions.
for _, attachment := range s.Attachments {
if attachment.Description != "" {
fields = append(fields, attachment.Description)
}
}
// Poll options.
if s.Poll != nil {
for _, opt := range s.Poll.Options {
if opt != "" {
fields = append(fields, opt)
}
}
}
return fields
}

60
internal/typeutils/util_test.go
View file

@ -23,7 +23,6 @@ import (
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/language"
"github.com/stretchr/testify/assert"
)
func TestMisskeyReportContentURLs1(t *testing.T) {
@ -157,62 +156,3 @@ func TestContentToContentLanguage(t *testing.T) {
}
}
}
func TestFilterableText(t *testing.T) {
type testcase struct {
status *gtsmodel.Status
expectedFields []string
}
for _, testcase := range []testcase{
{
status: &gtsmodel.Status{
ContentWarning: "This is a test status",
Content: `<p>Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> instance.</p>`,
},
expectedFields: []string{
"This is a test status",
"Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> instance.",
},
},
{
status: &gtsmodel.Status{
Content: `<p><span class="h-card"><a href="https://example.org/@zlatko" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>zlatko</span></a></span> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)</p><p><a href="https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863" rel="nofollow noreferrer noopener" target="_blank">https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863</a></p>`,
},
expectedFields: []string{
"@zlatko <https://example.org/@zlatko> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)\n\nhttps://codeberg.org/superseriousbusiness/gotosocial/pulls/2863 <https://codeberg.org/superseriousbusiness/gotosocial/pulls/2863>",
},
},
{
status: &gtsmodel.Status{
ContentWarning: "Nerd stuff",
Content: `<p>Latest graphs for <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> on <a href="https://github.com/ncruces/go-sqlite3" rel="nofollow noreferrer noopener" target="_blank">Wasm sqlite3</a> with <a href="https://codeberg.org/gruf/go-ffmpreg" rel="nofollow noreferrer noopener" target="_blank">embedded Wasm ffmpeg</a>, both running on <a href="https://wazero.io/" rel="nofollow noreferrer noopener" target="_blank">Wazero</a>, and configured with a <a href="https://codeberg.org/superseriousbusiness/gotosocial/src/commit/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266" rel="nofollow noreferrer noopener" target="_blank">50MiB db cache target</a>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.</p>`,
Attachments: []*gtsmodel.MediaAttachment{
{
Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`,
},
{
Description: `Another media attachment`,
},
},
Poll: &gtsmodel.Poll{
Options: []string{
"Poll option 1",
"Poll option 2",
},
},
},
expectedFields: []string{
"Nerd stuff",
"Latest graphs for #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> on Wasm sqlite3 <https://github.com/ncruces/go-sqlite3> with embedded Wasm ffmpeg <https://codeberg.org/gruf/go-ffmpreg>, both running on Wazero <https://wazero.io/>, and configured with a 50MiB db cache target <https://codeberg.org/superseriousbusiness/gotosocial/src/commit/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.",
"Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.",
"Another media attachment",
"Poll option 1",
"Poll option 2",
},
},
} {
fields := filterableFields(testcase.status)
assert.Equal(t, testcase.expectedFields, fields)
}
}