mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-12-15 11:23:01 -06:00
[bugfix/chore] Refactor timeline code (#1656)
* start poking timelines * OK yes we're refactoring, but it's nothing like the last time so don't worry * more fiddling * update tests, simplify Get * thanks linter, you're the best, mwah mwah kisses * do a bit more tidying up * start buggering about with the prepare function * fix little oopsie * start merging lists into 1 * ik heb een heel zwaar leven nee nee echt waar * hey it works we did it reddit * regenerate swagger docs * tidy up a wee bit * adjust paging * fix little error, remove unused functions
This commit is contained in:
tobi
GitHub
parent
c54510bc74
commit
3510454768
22 changed files with 1319 additions and 1365 deletions
|
|
@ -24,103 +24,205 @@ import (
|
|||
"fmt"
|
||||
|
||||
"codeberg.org/gruf/go-kv"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
)
|
||||
|
||||
func (t *timeline) ItemIndexLength(ctx context.Context) int {
|
||||
if t.indexedItems == nil || t.indexedItems.data == nil {
|
||||
return 0
|
||||
}
|
||||
return t.indexedItems.data.Len()
|
||||
}
|
||||
func (t *timeline) indexXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) error {
|
||||
l := log.
|
||||
WithContext(ctx).
|
||||
WithFields(kv.Fields{
|
||||
{"amount", amount},
|
||||
{"behindID", behindID},
|
||||
{"beforeID", beforeID},
|
||||
{"frontToBack", frontToBack},
|
||||
}...)
|
||||
l.Trace("entering indexXBetweenIDs")
|
||||
|
||||
func (t *timeline) indexBehind(ctx context.Context, itemID string, amount int) error {
|
||||
l := log.WithContext(ctx).
|
||||
WithFields(kv.Fields{{"amount", amount}}...)
|
||||
|
||||
// lazily initialize index if it hasn't been done already
|
||||
if t.indexedItems.data == nil {
|
||||
t.indexedItems.data = &list.List{}
|
||||
t.indexedItems.data.Init()
|
||||
}
|
||||
|
||||
// If we're already indexedBehind given itemID by the required amount, we can return nil.
|
||||
// First find position of itemID (or as near as possible).
|
||||
var position int
|
||||
positionLoop:
|
||||
for e := t.indexedItems.data.Front(); e != nil; e = e.Next() {
|
||||
entry, ok := e.Value.(*indexedItemsEntry)
|
||||
if !ok {
|
||||
return errors.New("indexBehind: could not parse e as an itemIndexEntry")
|
||||
}
|
||||
|
||||
if entry.itemID <= itemID {
|
||||
// we've found it
|
||||
break positionLoop
|
||||
}
|
||||
position++
|
||||
}
|
||||
|
||||
// now check if the length of indexed items exceeds the amount of items required (position of itemID, plus amount of posts requested after that)
|
||||
if t.indexedItems.data.Len() > position+amount {
|
||||
// we have enough indexed behind already to satisfy amount, so don't need to make db calls
|
||||
l.Trace("returning nil since we already have enough items indexed")
|
||||
if beforeID >= behindID {
|
||||
// This is an impossible situation, we
|
||||
// can't index anything between these.
|
||||
return nil
|
||||
}
|
||||
|
||||
toIndex := []Timelineable{}
|
||||
offsetID := itemID
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
|
||||
l.Trace("entering grabloop")
|
||||
grabloop:
|
||||
for i := 0; len(toIndex) < amount && i < 5; i++ { // try the grabloop 5 times only
|
||||
// first grab items using the caller-provided grab function
|
||||
l.Trace("grabbing...")
|
||||
items, stop, err := t.grabFunction(ctx, t.accountID, offsetID, "", "", amount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stop {
|
||||
break grabloop
|
||||
// Lazily init indexed items.
|
||||
if t.items.data == nil {
|
||||
t.items.data = &list.List{}
|
||||
t.items.data.Init()
|
||||
}
|
||||
|
||||
// Start by mapping out the list so we know what
|
||||
// we have to do. Depending on the current state
|
||||
// of the list we might not have to do *anything*.
|
||||
var (
|
||||
position int
|
||||
listLen = t.items.data.Len()
|
||||
behindIDPosition int
|
||||
beforeIDPosition int
|
||||
)
|
||||
|
||||
for e := t.items.data.Front(); e != nil; e = e.Next() {
|
||||
entry := e.Value.(*indexedItemsEntry) //nolint:forcetypeassert
|
||||
|
||||
position++
|
||||
|
||||
if entry.itemID > behindID {
|
||||
l.Trace("item is too new, continuing")
|
||||
continue
|
||||
}
|
||||
|
||||
l.Trace("filtering...")
|
||||
// now filter each item using the caller-provided filter function
|
||||
for _, item := range items {
|
||||
shouldIndex, err := t.filterFunction(ctx, t.accountID, item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if shouldIndex {
|
||||
toIndex = append(toIndex, item)
|
||||
}
|
||||
offsetID = item.GetID()
|
||||
if behindIDPosition == 0 {
|
||||
// Gone far enough through the list
|
||||
// and found our behindID mark.
|
||||
// We only need to set this once.
|
||||
l.Tracef("found behindID mark %s at position %d", entry.itemID, position)
|
||||
behindIDPosition = position
|
||||
}
|
||||
|
||||
if entry.itemID >= beforeID {
|
||||
// Push the beforeID mark back
|
||||
// one place every iteration.
|
||||
l.Tracef("setting beforeID mark %s at position %d", entry.itemID, position)
|
||||
beforeIDPosition = position
|
||||
}
|
||||
|
||||
if entry.itemID <= beforeID {
|
||||
// We've gone beyond the bounds of
|
||||
// items we're interested in; stop.
|
||||
l.Trace("reached older items, breaking")
|
||||
break
|
||||
}
|
||||
}
|
||||
l.Trace("left grabloop")
|
||||
|
||||
// index the items we got
|
||||
for _, s := range toIndex {
|
||||
if _, err := t.IndexOne(ctx, s.GetID(), s.GetBoostOfID(), s.GetAccountID(), s.GetBoostOfAccountID()); err != nil {
|
||||
return fmt.Errorf("indexBehind: error indexing item with id %s: %s", s.GetID(), err)
|
||||
// We can now figure out if we need to make db calls.
|
||||
var grabMore bool
|
||||
switch {
|
||||
case listLen < amount:
|
||||
// The whole list is shorter than the
|
||||
// amount we're being asked to return,
|
||||
// make up the difference.
|
||||
grabMore = true
|
||||
amount -= listLen
|
||||
case beforeIDPosition-behindIDPosition < amount:
|
||||
// Not enough items between behindID and
|
||||
// beforeID to return amount required,
|
||||
// try to get more.
|
||||
grabMore = true
|
||||
}
|
||||
|
||||
if !grabMore {
|
||||
// We're good!
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fetch additional items.
|
||||
items, err := t.grab(ctx, amount, behindID, beforeID, frontToBack)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Index all the items we got. We already have
|
||||
// a lock on the timeline, so don't call IndexOne
|
||||
// here, since that will also try to get a lock!
|
||||
for _, item := range items {
|
||||
entry := &indexedItemsEntry{
|
||||
itemID: item.GetID(),
|
||||
boostOfID: item.GetBoostOfID(),
|
||||
accountID: item.GetAccountID(),
|
||||
boostOfAccountID: item.GetBoostOfAccountID(),
|
||||
}
|
||||
|
||||
if _, err := t.items.insertIndexed(ctx, entry); err != nil {
|
||||
return fmt.Errorf("error inserting entry with itemID %s into index: %w", entry.itemID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *timeline) IndexOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
// grab wraps the timeline's grabFunction in paging + filtering logic.
|
||||
func (t *timeline) grab(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) ([]Timelineable, error) {
|
||||
var (
|
||||
sinceID string
|
||||
minID string
|
||||
grabbed int
|
||||
maxID = behindID
|
||||
filtered = make([]Timelineable, 0, amount)
|
||||
)
|
||||
|
||||
postIndexEntry := &indexedItemsEntry{
|
||||
itemID: itemID,
|
||||
boostOfID: boostOfID,
|
||||
accountID: accountID,
|
||||
boostOfAccountID: boostOfAccountID,
|
||||
if frontToBack {
|
||||
sinceID = beforeID
|
||||
} else {
|
||||
minID = beforeID
|
||||
}
|
||||
|
||||
return t.indexedItems.insertIndexed(ctx, postIndexEntry)
|
||||
for attempts := 0; attempts < 5; attempts++ {
|
||||
if grabbed >= amount {
|
||||
// We got everything we needed.
|
||||
break
|
||||
}
|
||||
|
||||
items, stop, err := t.grabFunction(
|
||||
ctx,
|
||||
t.accountID,
|
||||
maxID,
|
||||
sinceID,
|
||||
minID,
|
||||
// Don't grab more than we need to.
|
||||
amount-grabbed,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
// Grab function already checks for
|
||||
// db.ErrNoEntries, so if an error
|
||||
// is returned then it's a real one.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if stop || len(items) == 0 {
|
||||
// No items left.
|
||||
break
|
||||
}
|
||||
|
||||
// Set next query parameters.
|
||||
if frontToBack {
|
||||
// Page down.
|
||||
maxID = items[len(items)-1].GetID()
|
||||
if maxID <= beforeID {
|
||||
// Can't go any further.
|
||||
break
|
||||
}
|
||||
} else {
|
||||
// Page up.
|
||||
minID = items[0].GetID()
|
||||
if minID >= behindID {
|
||||
// Can't go any further.
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
ok, err := t.filterFunction(ctx, t.accountID, item)
|
||||
if err != nil {
|
||||
if !errors.Is(err, db.ErrNoEntries) {
|
||||
// Real error here.
|
||||
return nil, err
|
||||
}
|
||||
log.Warnf(ctx, "errNoEntries while filtering item %s: %s", item.GetID(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
if ok {
|
||||
filtered = append(filtered, item)
|
||||
grabbed++ // count this as grabbed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
|
||||
|
|
@ -134,46 +236,49 @@ func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusID string, boos
|
|||
boostOfAccountID: boostOfAccountID,
|
||||
}
|
||||
|
||||
inserted, err := t.indexedItems.insertIndexed(ctx, postIndexEntry)
|
||||
if inserted, err := t.items.insertIndexed(ctx, postIndexEntry); err != nil {
|
||||
return false, fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %w", err)
|
||||
} else if !inserted {
|
||||
// Entry wasn't inserted, so
|
||||
// don't bother preparing it.
|
||||
return false, nil
|
||||
}
|
||||
|
||||
preparable, err := t.prepareFunction(ctx, t.accountID, statusID)
|
||||
if err != nil {
|
||||
return inserted, fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err)
|
||||
return true, fmt.Errorf("IndexAndPrepareOne: error preparing: %w", err)
|
||||
}
|
||||
postIndexEntry.prepared = preparable
|
||||
|
||||
if inserted {
|
||||
if err := t.prepare(ctx, statusID); err != nil {
|
||||
return inserted, fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
return inserted, nil
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (t *timeline) OldestIndexedItemID(ctx context.Context) (string, error) {
|
||||
var id string
|
||||
if t.indexedItems == nil || t.indexedItems.data == nil || t.indexedItems.data.Back() == nil {
|
||||
// return an empty string if postindex hasn't been initialized yet
|
||||
return id, nil
|
||||
func (t *timeline) Len() int {
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
|
||||
if t.items == nil || t.items.data == nil {
|
||||
// indexedItems hasnt been initialized yet.
|
||||
return 0
|
||||
}
|
||||
|
||||
e := t.indexedItems.data.Back()
|
||||
entry, ok := e.Value.(*indexedItemsEntry)
|
||||
if !ok {
|
||||
return id, errors.New("OldestIndexedItemID: could not parse e as itemIndexEntry")
|
||||
}
|
||||
return entry.itemID, nil
|
||||
return t.items.data.Len()
|
||||
}
|
||||
|
||||
func (t *timeline) NewestIndexedItemID(ctx context.Context) (string, error) {
|
||||
var id string
|
||||
if t.indexedItems == nil || t.indexedItems.data == nil || t.indexedItems.data.Front() == nil {
|
||||
// return an empty string if postindex hasn't been initialized yet
|
||||
return id, nil
|
||||
func (t *timeline) OldestIndexedItemID() string {
|
||||
t.Lock()
|
||||
defer t.Unlock()
|
||||
|
||||
if t.items == nil || t.items.data == nil {
|
||||
// indexedItems hasnt been initialized yet.
|
||||
return ""
|
||||
}
|
||||
|
||||
e := t.indexedItems.data.Front()
|
||||
entry, ok := e.Value.(*indexedItemsEntry)
|
||||
if !ok {
|
||||
return id, errors.New("NewestIndexedItemID: could not parse e as itemIndexEntry")
|
||||
e := t.items.data.Back()
|
||||
if e == nil {
|
||||
// List was empty.
|
||||
return ""
|
||||
}
|
||||
return entry.itemID, nil
|
||||
|
||||
return e.Value.(*indexedItemsEntry).itemID //nolint:forcetypeassert
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue