mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-02 04:32:24 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Prune timelines once per hour to plug memory leak (#1117)

Browse source 
* export highest/lowest ULIDs as proper const

* add stop + start to timeline manager, other small fixes

* unexport unused interface funcs + tidy up

* add LastGot func

* add timeline Prune function

* test prune

* update lastGot
This commit is contained in:
tobi 2022-11-22 19:38:10 +01:00 • committed by GitHub
commit 50dc179d33
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 594 additions and 602 deletions
Download patch file Download diff file Expand all files Collapse all files

80
internal/timeline/timeline.go
View file

@ -21,6 +21,7 @@ package timeline
import (
"context"
"sync"
"time"
)
// GrabFunction is used by a Timeline to grab more items to index.
@ -73,26 +74,9 @@ type Timeline interface {
// If prepareNext is true, then the next predicted query will be prepared already in a goroutine,
// to make the next call to Get faster.
Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error)
// GetXFromTop returns x amount of items from the top of the timeline, from newest to oldest.
GetXFromTop(ctx context.Context, amount int) ([]Preparable, error)
// GetXBehindID returns x amount of items from the given id onwards, from newest to oldest.
// This will NOT include the item with the given ID.
//
// This corresponds to an api call to /timelines/home?max_id=WHATEVER
GetXBehindID(ctx context.Context, amount int, fromID string, attempts *int) ([]Preparable, error)
// GetXBeforeID returns x amount of items up to the given id, from newest to oldest.
// This will NOT include the item with the given ID.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER
GetXBeforeID(ctx context.Context, amount int, sinceID string, startFromTop bool) ([]Preparable, error)
// GetXBetweenID returns x amount of items from the given maxID, up to the given id, from newest to oldest.
// This will NOT include the item with the given IDs.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE
GetXBetweenID(ctx context.Context, amount int, maxID string, sinceID string) ([]Preparable, error)
/*
INDEXING FUNCTIONS
INDEXING + PREPARATION FUNCTIONS
*/
// IndexOne puts a item into the timeline at the appropriate place according to its 'createdAt' property.
@ -100,35 +84,14 @@ type Timeline interface {
// The returned bool indicates whether or not the item was actually inserted into the timeline. This will be false
// if the item is a boost and the original item or another boost of it already exists < boostReinsertionDepth back in the timeline.
IndexOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// OldestIndexedItemID returns the id of the rearmost (ie., the oldest) indexed item, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
OldestIndexedItemID(ctx context.Context) (string, error)
// NewestIndexedItemID returns the id of the frontmost (ie., the newest) indexed item, or an error if something goes wrong.
// If nothing goes wrong but there's no newest item, an empty string will be returned so make sure to check for this.
NewestIndexedItemID(ctx context.Context) (string, error)
IndexBefore(ctx context.Context, itemID string, amount int) error
IndexBehind(ctx context.Context, itemID string, amount int) error
/*
PREPARATION FUNCTIONS
*/
// PrepareXFromTop instructs the timeline to prepare x amount of items from the top of the timeline.
PrepareFromTop(ctx context.Context, amount int) error
// PrepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
// If include is true, then the given item ID will also be prepared, otherwise only entries behind it will be prepared.
PrepareBehind(ctx context.Context, itemID string, amount int) error
// IndexOne puts a item into the timeline at the appropriate place according to its 'createdAt' property,
// IndexAndPrepareOne puts a item into the timeline at the appropriate place according to its 'createdAt' property,
// and then immediately prepares it.
//
// The returned bool indicates whether or not the item was actually inserted into the timeline. This will be false
// if the item is a boost and the original item or another boost of it already exists < boostReinsertionDepth back in the timeline.
IndexAndPrepareOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// OldestPreparedItemID returns the id of the rearmost (ie., the oldest) prepared item, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
OldestPreparedItemID(ctx context.Context) (string, error)
// PrepareXFromTop instructs the timeline to prepare x amount of items from the top of the timeline, useful during init.
PrepareFromTop(ctx context.Context, amount int) error
/*
INFO FUNCTIONS
@ -136,13 +99,24 @@ type Timeline interface {
// ActualPostIndexLength returns the actual length of the item index at this point in time.
ItemIndexLength(ctx context.Context) int
// OldestIndexedItemID returns the id of the rearmost (ie., the oldest) indexed item, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
OldestIndexedItemID(ctx context.Context) (string, error)
// NewestIndexedItemID returns the id of the frontmost (ie., the newest) indexed item, or an error if something goes wrong.
// If nothing goes wrong but there's no newest item, an empty string will be returned so make sure to check for this.
NewestIndexedItemID(ctx context.Context) (string, error)
/*
UTILITY FUNCTIONS
*/
// Reset instructs the timeline to reset to its base state -- cache only the minimum amount of items.
Reset() error
// LastGot returns the time that Get was last called.
LastGot() time.Time
// Prune prunes preparedItems and indexedItems in this timeline to the desired lengths.
// This will be a no-op if the lengths are already < the desired values.
// Prune acquires a lock on the timeline before pruning.
// The return value is the combined total of items pruned from preparedItems and indexedItems.
Prune(desiredPreparedItemsLength int, desiredIndexedItemsLength int) int
// Remove removes a item from both the index and prepared items.
//
// If a item has multiple entries in a timeline, they will all be removed.
@ -157,12 +131,13 @@ type Timeline interface {
// timeline fulfils the Timeline interface
type timeline struct {
itemIndex *itemIndex
indexedItems *indexedItems
preparedItems *preparedItems
grabFunction GrabFunction
filterFunction FilterFunction
prepareFunction PrepareFunction
accountID string
lastGot time.Time
sync.Mutex
}
@ -175,7 +150,7 @@ func NewTimeline(
prepareFunction PrepareFunction,
skipInsertFunction SkipInsertFunction) (Timeline, error) {
return &timeline{
itemIndex: &itemIndex{
indexedItems: &indexedItems{
skipInsert: skipInsertFunction,
},
preparedItems: &preparedItems{
@ -185,17 +160,6 @@ func NewTimeline(
filterFunction: filterFunction,
prepareFunction: prepareFunction,
accountID: timelineAccountID,
lastGot: time.Time{},
}, nil
}
func (t *timeline) Reset() error {
return nil
}
func (t *timeline) ItemIndexLength(ctx context.Context) int {
if t.itemIndex == nil || t.itemIndex.data == nil {
return 0
}
return t.itemIndex.data.Len()
}