mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-11-17 08:07:30 -06:00
[feature] simpler cache size configuration (#2051)
* add automatic cache max size generation based on ratios of a singular fixed memory target
Signed-off-by: kim <grufwub@gmail.com>
* remove now-unused cache max-size config variables
Signed-off-by: kim <grufwub@gmail.com>
* slight ratio tweak
Signed-off-by: kim <grufwub@gmail.com>
* remove unused visibility config var
Signed-off-by: kim <grufwub@gmail.com>
* add secret little ratio config trick
Signed-off-by: kim <grufwub@gmail.com>
* fixed a word
Signed-off-by: kim <grufwub@gmail.com>
* update cache library to remove use of TTL in result caches + slice cache
Signed-off-by: kim <grufwub@gmail.com>
* update other cache usages to use correct interface
Signed-off-by: kim <grufwub@gmail.com>
* update example config to explain the cache memory target
Signed-off-by: kim <grufwub@gmail.com>
* update env parsing test with new config values
Signed-off-by: kim <grufwub@gmail.com>
* do some ratio twiddling
Signed-off-by: kim <grufwub@gmail.com>
* add missing header
* update envparsing with latest defaults
Signed-off-by: kim <grufwub@gmail.com>
* update size calculations to take into account result cache, simple cache and extra map overheads
Signed-off-by: kim <grufwub@gmail.com>
* tweak the ratios some more
Signed-off-by: kim <grufwub@gmail.com>
* more nan rampaging
Signed-off-by: kim <grufwub@gmail.com>
* fix envparsing script
Signed-off-by: kim <grufwub@gmail.com>
* update cache library, add sweep function to keep caches trim
Signed-off-by: kim <grufwub@gmail.com>
* sweep caches once a minute
Signed-off-by: kim <grufwub@gmail.com>
* add a regular job to sweep caches and keep under 80% utilisation
Signed-off-by: kim <grufwub@gmail.com>
* remove dead code
Signed-off-by: kim <grufwub@gmail.com>
* add new size library used to libraries section of readme
Signed-off-by: kim <grufwub@gmail.com>
* add better explanations for the mem-ratio numbers
Signed-off-by: kim <grufwub@gmail.com>
* update go-cache
Signed-off-by: kim <grufwub@gmail.com>
* library version bump
Signed-off-by: kim <grufwub@gmail.com>
* update cache.result{} size model estimation
Signed-off-by: kim <grufwub@gmail.com>
---------
Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
GitHub
parent
e8a20f587c
commit
00adf18c24
32 changed files with 2115 additions and 2330 deletions
|
|
@ -175,113 +175,32 @@ type HTTPClientConfiguration struct {
|
|||
}
|
||||
|
||||
type CacheConfiguration struct {
|
||||
GTS GTSCacheConfiguration `name:"gts"`
|
||||
|
||||
VisibilityMaxSize int `name:"visibility-max-size"`
|
||||
VisibilityTTL time.Duration `name:"visibility-ttl"`
|
||||
VisibilitySweepFreq time.Duration `name:"visibility-sweep-freq"`
|
||||
}
|
||||
|
||||
type GTSCacheConfiguration struct {
|
||||
AccountMaxSize int `name:"account-max-size"`
|
||||
AccountTTL time.Duration `name:"account-ttl"`
|
||||
AccountSweepFreq time.Duration `name:"account-sweep-freq"`
|
||||
|
||||
AccountNoteMaxSize int `name:"account-note-max-size"`
|
||||
AccountNoteTTL time.Duration `name:"account-note-ttl"`
|
||||
AccountNoteSweepFreq time.Duration `name:"account-note-sweep-freq"`
|
||||
|
||||
BlockMaxSize int `name:"block-max-size"`
|
||||
BlockTTL time.Duration `name:"block-ttl"`
|
||||
BlockSweepFreq time.Duration `name:"block-sweep-freq"`
|
||||
|
||||
BlockIDsMaxSize int `name:"block-ids-max-size"`
|
||||
BlockIDsTTL time.Duration `name:"block-ids-ttl"`
|
||||
BlockIDsSweepFreq time.Duration `name:"block-ids-sweep-freq"`
|
||||
|
||||
DomainBlockMaxSize int `name:"domain-block-max-size"`
|
||||
DomainBlockTTL time.Duration `name:"domain-block-ttl"`
|
||||
DomainBlockSweepFreq time.Duration `name:"domain-block-sweep-freq"`
|
||||
|
||||
EmojiMaxSize int `name:"emoji-max-size"`
|
||||
EmojiTTL time.Duration `name:"emoji-ttl"`
|
||||
EmojiSweepFreq time.Duration `name:"emoji-sweep-freq"`
|
||||
|
||||
EmojiCategoryMaxSize int `name:"emoji-category-max-size"`
|
||||
EmojiCategoryTTL time.Duration `name:"emoji-category-ttl"`
|
||||
EmojiCategorySweepFreq time.Duration `name:"emoji-category-sweep-freq"`
|
||||
|
||||
FollowMaxSize int `name:"follow-max-size"`
|
||||
FollowTTL time.Duration `name:"follow-ttl"`
|
||||
FollowSweepFreq time.Duration `name:"follow-sweep-freq"`
|
||||
|
||||
FollowIDsMaxSize int `name:"follow-ids-max-size"`
|
||||
FollowIDsTTL time.Duration `name:"follow-ids-ttl"`
|
||||
FollowIDsSweepFreq time.Duration `name:"follow-ids-sweep-freq"`
|
||||
|
||||
FollowRequestMaxSize int `name:"follow-request-max-size"`
|
||||
FollowRequestTTL time.Duration `name:"follow-request-ttl"`
|
||||
FollowRequestSweepFreq time.Duration `name:"follow-request-sweep-freq"`
|
||||
|
||||
FollowRequestIDsMaxSize int `name:"follow-request-ids-max-size"`
|
||||
FollowRequestIDsTTL time.Duration `name:"follow-request-ids-ttl"`
|
||||
FollowRequestIDsSweepFreq time.Duration `name:"follow-request-ids-sweep-freq"`
|
||||
|
||||
InstanceMaxSize int `name:"instance-max-size"`
|
||||
InstanceTTL time.Duration `name:"instance-ttl"`
|
||||
InstanceSweepFreq time.Duration `name:"instance-sweep-freq"`
|
||||
|
||||
ListMaxSize int `name:"list-max-size"`
|
||||
ListTTL time.Duration `name:"list-ttl"`
|
||||
ListSweepFreq time.Duration `name:"list-sweep-freq"`
|
||||
|
||||
ListEntryMaxSize int `name:"list-entry-max-size"`
|
||||
ListEntryTTL time.Duration `name:"list-entry-ttl"`
|
||||
ListEntrySweepFreq time.Duration `name:"list-entry-sweep-freq"`
|
||||
|
||||
MarkerMaxSize int `name:"marker-max-size"`
|
||||
MarkerTTL time.Duration `name:"marker-ttl"`
|
||||
MarkerSweepFreq time.Duration `name:"marker-sweep-freq"`
|
||||
|
||||
MediaMaxSize int `name:"media-max-size"`
|
||||
MediaTTL time.Duration `name:"media-ttl"`
|
||||
MediaSweepFreq time.Duration `name:"media-sweep-freq"`
|
||||
|
||||
MentionMaxSize int `name:"mention-max-size"`
|
||||
MentionTTL time.Duration `name:"mention-ttl"`
|
||||
MentionSweepFreq time.Duration `name:"mention-sweep-freq"`
|
||||
|
||||
NotificationMaxSize int `name:"notification-max-size"`
|
||||
NotificationTTL time.Duration `name:"notification-ttl"`
|
||||
NotificationSweepFreq time.Duration `name:"notification-sweep-freq"`
|
||||
|
||||
ReportMaxSize int `name:"report-max-size"`
|
||||
ReportTTL time.Duration `name:"report-ttl"`
|
||||
ReportSweepFreq time.Duration `name:"report-sweep-freq"`
|
||||
|
||||
StatusMaxSize int `name:"status-max-size"`
|
||||
StatusTTL time.Duration `name:"status-ttl"`
|
||||
StatusSweepFreq time.Duration `name:"status-sweep-freq"`
|
||||
|
||||
StatusFaveMaxSize int `name:"status-fave-max-size"`
|
||||
StatusFaveTTL time.Duration `name:"status-fave-ttl"`
|
||||
StatusFaveSweepFreq time.Duration `name:"status-fave-sweep-freq"`
|
||||
|
||||
TagMaxSize int `name:"tag-max-size"`
|
||||
TagTTL time.Duration `name:"tag-ttl"`
|
||||
TagSweepFreq time.Duration `name:"tag-sweep-freq"`
|
||||
|
||||
TombstoneMaxSize int `name:"tombstone-max-size"`
|
||||
TombstoneTTL time.Duration `name:"tombstone-ttl"`
|
||||
TombstoneSweepFreq time.Duration `name:"tombstone-sweep-freq"`
|
||||
|
||||
UserMaxSize int `name:"user-max-size"`
|
||||
UserTTL time.Duration `name:"user-ttl"`
|
||||
UserSweepFreq time.Duration `name:"user-sweep-freq"`
|
||||
|
||||
WebfingerMaxSize int `name:"webfinger-max-size"`
|
||||
WebfingerTTL time.Duration `name:"webfinger-ttl"`
|
||||
WebfingerSweepFreq time.Duration `name:"webfinger-sweep-freq"`
|
||||
MemoryTarget bytesize.Size `name:"memory-target"`
|
||||
AccountMemRatio float64 `name:"account-mem-ratio"`
|
||||
AccountNoteMemRatio float64 `name:"account-note-mem-ratio"`
|
||||
BlockMemRatio float64 `name:"block-mem-ratio"`
|
||||
BlockIDsMemRatio float64 `name:"block-mem-ratio"`
|
||||
EmojiMemRatio float64 `name:"emoji-mem-ratio"`
|
||||
EmojiCategoryMemRatio float64 `name:"emoji-category-mem-ratio"`
|
||||
FollowMemRatio float64 `name:"follow-mem-ratio"`
|
||||
FollowIDsMemRatio float64 `name:"follow-ids-mem-ratio"`
|
||||
FollowRequestMemRatio float64 `name:"follow-request-mem-ratio"`
|
||||
FollowRequestIDsMemRatio float64 `name:"follow-request-ids-mem-ratio"`
|
||||
InstanceMemRatio float64 `name:"instance-mem-ratio"`
|
||||
ListMemRatio float64 `name:"list-mem-ratio"`
|
||||
ListEntryMemRatio float64 `name:"list-entry-mem-ratio"`
|
||||
MarkerMemRatio float64 `name:"marker-mem-ratio"`
|
||||
MediaMemRatio float64 `name:"media-mem-ratio"`
|
||||
MentionMemRatio float64 `name:"mention-mem-ratio"`
|
||||
NotificationMemRatio float64 `name:"notification-mem-ratio"`
|
||||
ReportMemRatio float64 `name:"report-mem-ratio"`
|
||||
StatusMemRatio float64 `name:"status-mem-ratio"`
|
||||
StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"`
|
||||
TagMemRatio float64 `name:"tag-mem-ratio"`
|
||||
TombstoneMemRatio float64 `name:"tombstone-mem-ratio"`
|
||||
UserMemRatio float64 `name:"user-mem-ratio"`
|
||||
WebfingerMemRatio float64 `name:"webfinger-mem-ratio"`
|
||||
VisibilityMemRatio float64 `name:"visibility-mem-ratio"`
|
||||
}
|
||||
|
||||
// MarshalMap will marshal current Configuration into a map structure (useful for JSON/TOML/YAML).
|
||||
|
|
|
|||
|
|
@ -126,111 +126,50 @@ var Defaults = Configuration{
|
|||
AdvancedSenderMultiplier: 2, // 2 senders per CPU
|
||||
|
||||
Cache: CacheConfiguration{
|
||||
GTS: GTSCacheConfiguration{
|
||||
AccountMaxSize: 2000,
|
||||
AccountTTL: time.Minute * 30,
|
||||
AccountSweepFreq: time.Minute,
|
||||
// Rough memory target that the total
|
||||
// size of all State.Caches will attempt
|
||||
// to remain with. Emphasis on *rough*.
|
||||
MemoryTarget: 200 * bytesize.MiB,
|
||||
|
||||
AccountNoteMaxSize: 1000,
|
||||
AccountNoteTTL: time.Minute * 30,
|
||||
AccountNoteSweepFreq: time.Minute,
|
||||
|
||||
BlockMaxSize: 1000,
|
||||
BlockTTL: time.Minute * 30,
|
||||
BlockSweepFreq: time.Minute,
|
||||
|
||||
BlockIDsMaxSize: 500,
|
||||
BlockIDsTTL: time.Minute * 30,
|
||||
BlockIDsSweepFreq: time.Minute,
|
||||
|
||||
DomainBlockMaxSize: 2000,
|
||||
DomainBlockTTL: time.Hour * 24,
|
||||
DomainBlockSweepFreq: time.Minute,
|
||||
|
||||
EmojiMaxSize: 2000,
|
||||
EmojiTTL: time.Minute * 30,
|
||||
EmojiSweepFreq: time.Minute,
|
||||
|
||||
EmojiCategoryMaxSize: 100,
|
||||
EmojiCategoryTTL: time.Minute * 30,
|
||||
EmojiCategorySweepFreq: time.Minute,
|
||||
|
||||
FollowMaxSize: 2000,
|
||||
FollowTTL: time.Minute * 30,
|
||||
FollowSweepFreq: time.Minute,
|
||||
|
||||
FollowIDsMaxSize: 500,
|
||||
FollowIDsTTL: time.Minute * 30,
|
||||
FollowIDsSweepFreq: time.Minute,
|
||||
|
||||
FollowRequestMaxSize: 2000,
|
||||
FollowRequestTTL: time.Minute * 30,
|
||||
FollowRequestSweepFreq: time.Minute,
|
||||
|
||||
FollowRequestIDsMaxSize: 500,
|
||||
FollowRequestIDsTTL: time.Minute * 30,
|
||||
FollowRequestIDsSweepFreq: time.Minute,
|
||||
|
||||
InstanceMaxSize: 2000,
|
||||
InstanceTTL: time.Minute * 30,
|
||||
InstanceSweepFreq: time.Minute,
|
||||
|
||||
ListMaxSize: 2000,
|
||||
ListTTL: time.Minute * 30,
|
||||
ListSweepFreq: time.Minute,
|
||||
|
||||
ListEntryMaxSize: 2000,
|
||||
ListEntryTTL: time.Minute * 30,
|
||||
ListEntrySweepFreq: time.Minute,
|
||||
|
||||
MarkerMaxSize: 2000,
|
||||
MarkerTTL: time.Hour * 6,
|
||||
MarkerSweepFreq: time.Minute,
|
||||
|
||||
MediaMaxSize: 1000,
|
||||
MediaTTL: time.Minute * 30,
|
||||
MediaSweepFreq: time.Minute,
|
||||
|
||||
MentionMaxSize: 2000,
|
||||
MentionTTL: time.Minute * 30,
|
||||
MentionSweepFreq: time.Minute,
|
||||
|
||||
NotificationMaxSize: 1000,
|
||||
NotificationTTL: time.Minute * 30,
|
||||
NotificationSweepFreq: time.Minute,
|
||||
|
||||
ReportMaxSize: 100,
|
||||
ReportTTL: time.Minute * 30,
|
||||
ReportSweepFreq: time.Minute,
|
||||
|
||||
StatusMaxSize: 2000,
|
||||
StatusTTL: time.Minute * 30,
|
||||
StatusSweepFreq: time.Minute,
|
||||
|
||||
StatusFaveMaxSize: 2000,
|
||||
StatusFaveTTL: time.Minute * 30,
|
||||
StatusFaveSweepFreq: time.Minute,
|
||||
|
||||
TagMaxSize: 2000,
|
||||
TagTTL: time.Minute * 30,
|
||||
TagSweepFreq: time.Minute,
|
||||
|
||||
TombstoneMaxSize: 500,
|
||||
TombstoneTTL: time.Minute * 30,
|
||||
TombstoneSweepFreq: time.Minute,
|
||||
|
||||
UserMaxSize: 500,
|
||||
UserTTL: time.Minute * 30,
|
||||
UserSweepFreq: time.Minute,
|
||||
|
||||
WebfingerMaxSize: 250,
|
||||
WebfingerTTL: time.Hour * 24,
|
||||
WebfingerSweepFreq: time.Minute * 15,
|
||||
},
|
||||
|
||||
VisibilityMaxSize: 2000,
|
||||
VisibilityTTL: time.Minute * 30,
|
||||
VisibilitySweepFreq: time.Minute,
|
||||
// These ratios signal what percentage
|
||||
// of the available cache target memory
|
||||
// is allocated to each object type's
|
||||
// cache.
|
||||
//
|
||||
// These are weighted by a totally
|
||||
// assorted mixture of priority, and
|
||||
// manual twiddling to get the generated
|
||||
// cache capacity ratios within normal
|
||||
// amounts dependent size of the models.
|
||||
//
|
||||
// when TODO items in the size.go source
|
||||
// file have been addressed, these should
|
||||
// be able to make some more sense :D
|
||||
AccountMemRatio: 18,
|
||||
AccountNoteMemRatio: 0.1,
|
||||
BlockMemRatio: 3,
|
||||
BlockIDsMemRatio: 3,
|
||||
EmojiMemRatio: 3,
|
||||
EmojiCategoryMemRatio: 0.1,
|
||||
FollowMemRatio: 4,
|
||||
FollowIDsMemRatio: 4,
|
||||
FollowRequestMemRatio: 2,
|
||||
FollowRequestIDsMemRatio: 2,
|
||||
InstanceMemRatio: 1,
|
||||
ListMemRatio: 3,
|
||||
ListEntryMemRatio: 3,
|
||||
MarkerMemRatio: 0.5,
|
||||
MediaMemRatio: 4,
|
||||
MentionMemRatio: 5,
|
||||
NotificationMemRatio: 5,
|
||||
ReportMemRatio: 1,
|
||||
StatusMemRatio: 18,
|
||||
StatusFaveMemRatio: 5,
|
||||
TagMemRatio: 3,
|
||||
TombstoneMemRatio: 2,
|
||||
UserMemRatio: 0.1,
|
||||
WebfingerMemRatio: 0.1,
|
||||
VisibilityMemRatio: 2,
|
||||
},
|
||||
|
||||
HTTPClient: HTTPClientConfiguration{
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue