[feature] Filters v1 (#2594)

* Implement client-side v1 filters

* Exclude linter false positives

* Update test/envparsing.sh

* Fix minor Swagger, style, and Bun usage issues

* Regenerate Swagger

* De-generify filter keywords

* Remove updating filter statuses

This is an operation that the Mastodon v2 filter API doesn't actually have, because filter statuses, unlike keywords, don't have options: the only info they contain is the status ID to be filtered.

* Add a test for filter statuses specifically

* De-generify filter statuses

* Inline FilterEntry

* Use vertical style for Bun operations consistently

* Add comment on Filter DB interface

* Remove GoLand linter control comments

Our existing linters should catch these, or they don't matter very much

* Reduce memory ratio for filters
This commit is contained in:
Vyr Cossont 2024-03-06 02:15:58 -08:00 committed by GitHub
commit 61a2b91f45
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
50 changed files with 4672 additions and 52 deletions

View file

@ -61,6 +61,9 @@ func (c *Caches) Init() {
c.initDomainBlock()
c.initEmoji()
c.initEmojiCategory()
c.initFilter()
c.initFilterKeyword()
c.initFilterStatus()
c.initFollow()
c.initFollowIDs()
c.initFollowRequest()
@ -119,6 +122,9 @@ func (c *Caches) Sweep(threshold float64) {
c.GTS.BlockIDs.Trim(threshold)
c.GTS.Emoji.Trim(threshold)
c.GTS.EmojiCategory.Trim(threshold)
c.GTS.Filter.Trim(threshold)
c.GTS.FilterKeyword.Trim(threshold)
c.GTS.FilterStatus.Trim(threshold)
c.GTS.Follow.Trim(threshold)
c.GTS.FollowIDs.Trim(threshold)
c.GTS.FollowRequest.Trim(threshold)

108
internal/cache/db.go vendored
View file

@ -67,6 +67,15 @@ type GTSCaches struct {
// EmojiCategory provides access to the gtsmodel EmojiCategory database cache.
EmojiCategory structr.Cache[*gtsmodel.EmojiCategory]
// Filter provides access to the gtsmodel Filter database cache.
Filter structr.Cache[*gtsmodel.Filter]
// FilterKeyword provides access to the gtsmodel FilterKeyword database cache.
FilterKeyword structr.Cache[*gtsmodel.FilterKeyword]
// FilterStatus provides access to the gtsmodel FilterStatus database cache.
FilterStatus structr.Cache[*gtsmodel.FilterStatus]
// Follow provides access to the gtsmodel Follow database cache.
Follow structr.Cache[*gtsmodel.Follow]
@ -409,6 +418,105 @@ func (c *Caches) initEmojiCategory() {
})
}
func (c *Caches) initFilter() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(
sizeofFilter(), // model in-mem size.
config.GetCacheFilterMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
copyF := func(filter1 *gtsmodel.Filter) *gtsmodel.Filter {
filter2 := new(gtsmodel.Filter)
*filter2 = *filter1
// Don't include ptr fields that
// will be populated separately.
// See internal/db/bundb/filter.go.
filter2.Keywords = nil
filter2.Statuses = nil
return filter2
}
c.GTS.Filter.Init(structr.Config[*gtsmodel.Filter]{
Indices: []structr.IndexConfig{
{Fields: "ID"},
{Fields: "AccountID", Multiple: true},
},
MaxSize: cap,
IgnoreErr: ignoreErrors,
CopyValue: copyF,
})
}
func (c *Caches) initFilterKeyword() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(
sizeofFilterKeyword(), // model in-mem size.
config.GetCacheFilterKeywordMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
copyF := func(filterKeyword1 *gtsmodel.FilterKeyword) *gtsmodel.FilterKeyword {
filterKeyword2 := new(gtsmodel.FilterKeyword)
*filterKeyword2 = *filterKeyword1
// Don't include ptr fields that
// will be populated separately.
// See internal/db/bundb/filter.go.
filterKeyword2.Filter = nil
return filterKeyword2
}
c.GTS.FilterKeyword.Init(structr.Config[*gtsmodel.FilterKeyword]{
Indices: []structr.IndexConfig{
{Fields: "ID"},
{Fields: "AccountID", Multiple: true},
{Fields: "FilterID", Multiple: true},
},
MaxSize: cap,
IgnoreErr: ignoreErrors,
CopyValue: copyF,
})
}
func (c *Caches) initFilterStatus() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(
sizeofFilterStatus(), // model in-mem size.
config.GetCacheFilterStatusMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
copyF := func(filterStatus1 *gtsmodel.FilterStatus) *gtsmodel.FilterStatus {
filterStatus2 := new(gtsmodel.FilterStatus)
*filterStatus2 = *filterStatus1
// Don't include ptr fields that
// will be populated separately.
// See internal/db/bundb/filter.go.
filterStatus2.Filter = nil
return filterStatus2
}
c.GTS.FilterStatus.Init(structr.Config[*gtsmodel.FilterStatus]{
Indices: []structr.IndexConfig{
{Fields: "ID"},
{Fields: "AccountID", Multiple: true},
{Fields: "FilterID", Multiple: true},
},
MaxSize: cap,
IgnoreErr: ignoreErrors,
CopyValue: copyF,
})
}
func (c *Caches) initFollow() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(

View file

@ -309,6 +309,38 @@ func sizeofEmojiCategory() uintptr {
}))
}
func sizeofFilter() uintptr {
return uintptr(size.Of(&gtsmodel.Filter{
ID: exampleID,
CreatedAt: exampleTime,
UpdatedAt: exampleTime,
ExpiresAt: exampleTime,
AccountID: exampleID,
Title: exampleTextSmall,
Action: gtsmodel.FilterActionHide,
}))
}
func sizeofFilterKeyword() uintptr {
return uintptr(size.Of(&gtsmodel.FilterKeyword{
ID: exampleID,
CreatedAt: exampleTime,
UpdatedAt: exampleTime,
FilterID: exampleID,
Keyword: exampleTextSmall,
}))
}
func sizeofFilterStatus() uintptr {
return uintptr(size.Of(&gtsmodel.FilterStatus{
ID: exampleID,
CreatedAt: exampleTime,
UpdatedAt: exampleTime,
FilterID: exampleID,
StatusID: exampleID,
}))
}
func sizeofFollow() uintptr {
return uintptr(size.Of(&gtsmodel.Follow{
ID: exampleID,