mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-10-28 20:02:24 -05:00
[performance] rewrite timelines to rely on new timeline cache type (#3941)
* start work rewriting timeline cache type
* further work rewriting timeline caching
* more work integration new timeline code
* remove old code
* add local timeline, fix up merge conflicts
* remove old use of go-bytes
* implement new timeline code into more areas of codebase, pull in latest go-mangler, go-mutexes, go-structr
* remove old timeline package, add local timeline cache
* remove references to old timeline types that needed starting up in tests
* start adding page validation
* fix test-identified timeline cache package issues
* fix up more tests, fix missing required changes, etc
* add exclusion for test.out in gitignore
* clarify some things better in code comments
* tweak cache size limits
* fix list timeline cache fetching
* further list timeline fixes
* linter, ssssssssshhhhhhhhhhhh please
* fix linter hints
* reslice the output if it's beyond length of 'lim'
* remove old timeline initialization code, bump go-structr to v0.9.4
* continued from previous commit
* improved code comments
* don't allow multiple entries for BoostOfID values to prevent repeated boosts of same boosts
* finish writing more code comments
* some variable renaming, for ease of following
* change the way we update lo,hi paging values during timeline load
* improved code comments for updated / returned lo , hi paging values
* finish writing code comments for the StatusTimeline{} type itself
* fill in more code comments
* update go-structr version to latest with changed timeline unique indexing logic
* have a local and public timeline *per user*
* rewrite calls to public / local timeline calls
* remove the zero length check, as lo, hi values might still be set
* simplify timeline cache loading, fix lo/hi returns, fix timeline invalidation side-effects missing for some federated actions
* swap the lo, hi values 🤦
* add (now) missing slice reverse of tag timeline statuses when paging ASC
* remove local / public caches (is out of scope for this work), share more timeline code
* remove unnecessary change
* again, remove more unused code
* remove unused function to appease the linter
* move boost checking to prepare function
* fix use of timeline.lastOrder, fix incorrect range functions used
* remove comments for repeat code
* remove the boost logic from prepare function
* do a maximum of 5 loads, not 10
* add repeat boost filtering logic, update go-structr, general improvements
* more code comments
* add important note
* fix timeline tests now that timelines are returned in page order
* remove unused field
* add StatusTimeline{} tests
* add more status timeline tests
* start adding preloading support
* ensure repeat boosts are marked in preloaded entries
* share a bunch of the database load code in timeline cache, don't clear timelines on relationship change
* add logic to allow dynamic clear / preloading of timelines
* comment-out unused functions, but leave in place as we might end-up using them
* fix timeline preload state check
* much improved status timeline code comments
* more code comments, don't bother inserting statuses if timeline not preloaded
* shift around some logic to make sure things aren't accidentally left set
* finish writing code comments
* remove trim-after-insert behaviour
* fix-up some comments referring to old logic
* remove unsetting of lo, hi
* fix preload repeatBoost checking logic
* don't return on status filter errors, these are usually transient
* better concurrency safety in Clear() and Done()
* fix test broken due to addition of preloader
* fix repeatBoost logic that doesn't account for already-hidden repeatBoosts
* ensure edit submodels are dropped on cache insertion
* update code-comment to expand CAS accronym
* use a plus1hULID() instead of 24h
* remove unused functions
* add note that public / local timeline requester can be nil
* fix incorrect visibility filtering of tag timeline statuses
* ensure we filter home timeline statuses on local only
* some small re-orderings to confirm query params in correct places
* fix the local only home timeline filter func
This commit is contained in:
kim
GitHub
parent
ffde1b150f
commit
6a6a499333
108 changed files with 2935 additions and 5213 deletions
|
|
@ -57,12 +57,10 @@ import (
|
|||
"github.com/superseriousbusiness/gotosocial/internal/observability"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/oidc"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/processing"
|
||||
tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/router"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/state"
|
||||
gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/timeline"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/transport"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/web"
|
||||
|
|
@ -139,20 +137,6 @@ var Start action.GTSAction = func(ctx context.Context) error {
|
|||
// Noop on unstarted workers.
|
||||
state.Workers.Stop()
|
||||
|
||||
if state.Timelines.Home != nil {
|
||||
// Home timeline mgr was setup, ensure it gets stopped.
|
||||
if err := state.Timelines.Home.Stop(); err != nil {
|
||||
log.Errorf(ctx, "error stopping home timeline: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if state.Timelines.List != nil {
|
||||
// List timeline mgr was setup, ensure it gets stopped.
|
||||
if err := state.Timelines.List.Stop(); err != nil {
|
||||
log.Errorf(ctx, "error stopping list timeline: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if process != nil {
|
||||
const timeout = time.Minute
|
||||
|
||||
|
|
@ -323,26 +307,6 @@ var Start action.GTSAction = func(ctx context.Context) error {
|
|||
// Create a Web Push notification sender.
|
||||
webPushSender := webpush.NewSender(client, state, typeConverter)
|
||||
|
||||
// Initialize both home / list timelines.
|
||||
state.Timelines.Home = timeline.NewManager(
|
||||
tlprocessor.HomeTimelineGrab(state),
|
||||
tlprocessor.HomeTimelineFilter(state, visFilter),
|
||||
tlprocessor.HomeTimelineStatusPrepare(state, typeConverter),
|
||||
tlprocessor.SkipInsert(),
|
||||
)
|
||||
if err := state.Timelines.Home.Start(); err != nil {
|
||||
return fmt.Errorf("error starting home timeline: %s", err)
|
||||
}
|
||||
state.Timelines.List = timeline.NewManager(
|
||||
tlprocessor.ListTimelineGrab(state),
|
||||
tlprocessor.ListTimelineFilter(state, visFilter),
|
||||
tlprocessor.ListTimelineStatusPrepare(state, typeConverter),
|
||||
tlprocessor.SkipInsert(),
|
||||
)
|
||||
if err := state.Timelines.List.Start(); err != nil {
|
||||
return fmt.Errorf("error starting list timeline: %s", err)
|
||||
}
|
||||
|
||||
// Start the job scheduler
|
||||
// (this is required for cleaner).
|
||||
state.Workers.StartScheduler()
|
||||
|
|
|
|||
|
|
@ -42,12 +42,10 @@ import (
|
|||
"github.com/superseriousbusiness/gotosocial/internal/middleware"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/observability"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/oidc"
|
||||
tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/router"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/state"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/storage"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/timeline"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/web"
|
||||
"github.com/superseriousbusiness/gotosocial/testrig"
|
||||
|
|
@ -89,20 +87,6 @@ var Start action.GTSAction = func(ctx context.Context) error {
|
|||
// tasks from being executed.
|
||||
testrig.StopWorkers(state)
|
||||
|
||||
if state.Timelines.Home != nil {
|
||||
// Home timeline mgr was setup, ensure it gets stopped.
|
||||
if err := state.Timelines.Home.Stop(); err != nil {
|
||||
log.Errorf(ctx, "error stopping home timeline: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if state.Timelines.List != nil {
|
||||
// List timeline mgr was setup, ensure it gets stopped.
|
||||
if err := state.Timelines.List.Stop(); err != nil {
|
||||
log.Errorf(ctx, "error stopping list timeline: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if state.Storage != nil {
|
||||
// If storage was created, ensure torn down.
|
||||
testrig.StandardStorageTeardown(state.Storage)
|
||||
|
|
@ -172,26 +156,6 @@ var Start action.GTSAction = func(ctx context.Context) error {
|
|||
typeConverter := typeutils.NewConverter(state)
|
||||
filter := visibility.NewFilter(state)
|
||||
|
||||
// Initialize both home / list timelines.
|
||||
state.Timelines.Home = timeline.NewManager(
|
||||
tlprocessor.HomeTimelineGrab(state),
|
||||
tlprocessor.HomeTimelineFilter(state, filter),
|
||||
tlprocessor.HomeTimelineStatusPrepare(state, typeConverter),
|
||||
tlprocessor.SkipInsert(),
|
||||
)
|
||||
if err := state.Timelines.Home.Start(); err != nil {
|
||||
return fmt.Errorf("error starting home timeline: %s", err)
|
||||
}
|
||||
state.Timelines.List = timeline.NewManager(
|
||||
tlprocessor.ListTimelineGrab(state),
|
||||
tlprocessor.ListTimelineFilter(state, filter),
|
||||
tlprocessor.ListTimelineStatusPrepare(state, typeConverter),
|
||||
tlprocessor.SkipInsert(),
|
||||
)
|
||||
if err := state.Timelines.List.Start(); err != nil {
|
||||
return fmt.Errorf("error starting list timeline: %s", err)
|
||||
}
|
||||
|
||||
processor := testrig.NewTestProcessor(state, federator, emailSender, webPushSender, mediaManager)
|
||||
|
||||
// Initialize workers.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue