[performance] filter model and database table improvements (#4277)

- removes unnecessary fields / columns (created_at, updated_at)
- replaces filter.context_* columns with singular filter.contexts bit field which should save both struct memory and database space
- replaces filter.action string with integer enum type which should save both struct memory and database space
- adds links from filter to filter_* tables with Filter{}.KeywordIDs and Filter{}.StatusIDs fields (this also means we now have those ID slices cached, which reduces some lookups)
- removes account_id fields from filter_* tables, since there's a more direct connection between filter and filter_* tables, and filter.account_id already exists
- refactors a bunch of the filter processor logic to save on code repetition, factor in the above changes, fix a few bugs with missed error returns and bring it more in-line with some of our newer code

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4277
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>

internal/config/config.go

@@ -226,6 +226,7 @@ type CacheConfiguration struct {
 	EmojiMemRatio                         float64       `name:"emoji-mem-ratio"`
 	EmojiCategoryMemRatio                 float64       `name:"emoji-category-mem-ratio"`
 	FilterMemRatio                        float64       `name:"filter-mem-ratio"`
+	FilterIDsMemRatio                     float64       `name:"filter-ids-mem-ratio"`
 	FilterKeywordMemRatio                 float64       `name:"filter-keyword-mem-ratio"`
 	FilterStatusMemRatio                  float64       `name:"filter-status-mem-ratio"`
 	FollowMemRatio                        float64       `name:"follow-mem-ratio"`

internal/config/defaults.go

@@ -193,6 +193,7 @@ var Defaults = Configuration{
 		EmojiMemRatio:                         3,
 		EmojiCategoryMemRatio:                 0.1,
 		FilterMemRatio:                        0.5,
+		FilterIDsMemRatio:                     2,
 		FilterKeywordMemRatio:                 0.5,
 		FilterStatusMemRatio:                  0.5,
 		FollowMemRatio:                        2,

internal/config/helpers.gen.go

@@ -170,6 +170,7 @@ const (
 	CacheEmojiMemRatioFlag                         = "cache-emoji-mem-ratio"
 	CacheEmojiCategoryMemRatioFlag                 = "cache-emoji-category-mem-ratio"
 	CacheFilterMemRatioFlag                        = "cache-filter-mem-ratio"
+	CacheFilterIDsMemRatioFlag                     = "cache-filter-ids-mem-ratio"
 	CacheFilterKeywordMemRatioFlag                 = "cache-filter-keyword-mem-ratio"
 	CacheFilterStatusMemRatioFlag                  = "cache-filter-status-mem-ratio"
 	CacheFollowMemRatioFlag                        = "cache-follow-mem-ratio"
@@ -362,6 +363,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
 	flags.Float64("cache-emoji-mem-ratio", cfg.Cache.EmojiMemRatio, "")
 	flags.Float64("cache-emoji-category-mem-ratio", cfg.Cache.EmojiCategoryMemRatio, "")
 	flags.Float64("cache-filter-mem-ratio", cfg.Cache.FilterMemRatio, "")
+	flags.Float64("cache-filter-ids-mem-ratio", cfg.Cache.FilterIDsMemRatio, "")
 	flags.Float64("cache-filter-keyword-mem-ratio", cfg.Cache.FilterKeywordMemRatio, "")
 	flags.Float64("cache-filter-status-mem-ratio", cfg.Cache.FilterStatusMemRatio, "")
 	flags.Float64("cache-follow-mem-ratio", cfg.Cache.FollowMemRatio, "")
@@ -406,7 +408,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
 }
 
 func (cfg *Configuration) MarshalMap() map[string]any {
-	cfgmap := make(map[string]any, 189)
+	cfgmap := make(map[string]any, 190)
 	cfgmap["log-level"] = cfg.LogLevel
 	cfgmap["log-timestamp-format"] = cfg.LogTimestampFormat
 	cfgmap["log-db-queries"] = cfg.LogDbQueries
@@ -548,6 +550,7 @@ func (cfg *Configuration) MarshalMap() map[string]any {
 	cfgmap["cache-emoji-mem-ratio"] = cfg.Cache.EmojiMemRatio
 	cfgmap["cache-emoji-category-mem-ratio"] = cfg.Cache.EmojiCategoryMemRatio
 	cfgmap["cache-filter-mem-ratio"] = cfg.Cache.FilterMemRatio
+	cfgmap["cache-filter-ids-mem-ratio"] = cfg.Cache.FilterIDsMemRatio
 	cfgmap["cache-filter-keyword-mem-ratio"] = cfg.Cache.FilterKeywordMemRatio
 	cfgmap["cache-filter-status-mem-ratio"] = cfg.Cache.FilterStatusMemRatio
 	cfgmap["cache-follow-mem-ratio"] = cfg.Cache.FollowMemRatio
@@ -1767,6 +1770,14 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error {
 		}
 	}
 
+	if ival, ok := cfgmap["cache-filter-ids-mem-ratio"]; ok {
+		var err error
+		cfg.Cache.FilterIDsMemRatio, err = cast.ToFloat64E(ival)
+		if err != nil {
+			return fmt.Errorf("error casting %#v -> float64 for 'cache-filter-ids-mem-ratio': %w", ival, err)
+		}
+	}
+
 	if ival, ok := cfgmap["cache-filter-keyword-mem-ratio"]; ok {
 		var err error
 		cfg.Cache.FilterKeywordMemRatio, err = cast.ToFloat64E(ival)
@@ -5278,6 +5289,28 @@ func GetCacheFilterMemRatio() float64 { return global.GetCacheFilterMemRatio() }
 // SetCacheFilterMemRatio safely sets the value for global configuration 'Cache.FilterMemRatio' field
 func SetCacheFilterMemRatio(v float64) { global.SetCacheFilterMemRatio(v) }
 
+// GetCacheFilterIDsMemRatio safely fetches the Configuration value for state's 'Cache.FilterIDsMemRatio' field
+func (st *ConfigState) GetCacheFilterIDsMemRatio() (v float64) {
+	st.mutex.RLock()
+	v = st.config.Cache.FilterIDsMemRatio
+	st.mutex.RUnlock()
+	return
+}
+
+// SetCacheFilterIDsMemRatio safely sets the Configuration value for state's 'Cache.FilterIDsMemRatio' field
+func (st *ConfigState) SetCacheFilterIDsMemRatio(v float64) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.Cache.FilterIDsMemRatio = v
+	st.reloadToViper()
+}
+
+// GetCacheFilterIDsMemRatio safely fetches the value for global configuration 'Cache.FilterIDsMemRatio' field
+func GetCacheFilterIDsMemRatio() float64 { return global.GetCacheFilterIDsMemRatio() }
+
+// SetCacheFilterIDsMemRatio safely sets the value for global configuration 'Cache.FilterIDsMemRatio' field
+func SetCacheFilterIDsMemRatio(v float64) { global.SetCacheFilterIDsMemRatio(v) }
+
 // GetCacheFilterKeywordMemRatio safely fetches the Configuration value for state's 'Cache.FilterKeywordMemRatio' field
 func (st *ConfigState) GetCacheFilterKeywordMemRatio() (v float64) {
 	st.mutex.RLock()
@@ -6359,6 +6392,7 @@ func (st *ConfigState) GetTotalOfMemRatios() (total float64) {
 	total += st.config.Cache.EmojiMemRatio
 	total += st.config.Cache.EmojiCategoryMemRatio
 	total += st.config.Cache.FilterMemRatio
+	total += st.config.Cache.FilterIDsMemRatio
 	total += st.config.Cache.FilterKeywordMemRatio
 	total += st.config.Cache.FilterStatusMemRatio
 	total += st.config.Cache.FollowMemRatio
@@ -6910,6 +6944,17 @@ func flattenConfigMap(cfgmap map[string]any) {
 		}
 	}
 
+	for _, key := range [][]string{
+		{"cache", "filter-ids-mem-ratio"},
+	} {
+		ival, ok := mapGet(cfgmap, key...)
+		if ok {
+			cfgmap["cache-filter-ids-mem-ratio"] = ival
+			nestedKeys[key[0]] = struct{}{}
+			break
+		}
+	}
+
 	for _, key := range [][]string{
 		{"cache", "filter-keyword-mem-ratio"},
 	} {