From f8f446456ce4fc587c243fe8317b265787f49675 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 15 Aug 2021 18:05:38 +0200 Subject: [PATCH] make the timeline code a bit lazier --- internal/timeline/get.go | 3 --- internal/timeline/index.go | 40 ++++++++++++++++++++++++++++++++++-- internal/timeline/prepare.go | 21 ++++++++++++++++--- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/internal/timeline/get.go b/internal/timeline/get.go index e8154797d..ac5714086 100644 --- a/internal/timeline/get.go +++ b/internal/timeline/get.go @@ -174,9 +174,6 @@ findMarkLoop: // we didn't find it, so we need to make sure it's indexed and prepared and then try again // this can happen when a user asks for really old posts if behindIDMark == nil { - if err := t.IndexBehind(behindID, true, amount); err != nil { - return nil, fmt.Errorf("GetXBehindID: error indexing behind and including ID %s", behindID) - } if err := t.PrepareBehind(behindID, amount); err != nil { return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID) } diff --git a/internal/timeline/index.go b/internal/timeline/index.go index b4c34051f..1e1a9d7bb 100644 --- a/internal/timeline/index.go +++ b/internal/timeline/index.go @@ -19,6 +19,7 @@ package timeline import ( + "container/list" "errors" "fmt" "time" @@ -29,6 +30,12 @@ import ( ) func (t *timeline) IndexBefore(statusID string, include bool, amount int) error { + // lazily initialize index if it hasn't been done already + if t.postIndex.data == nil { + t.postIndex.data = &list.List{} + t.postIndex.data.Init() + } + filtered := []*gtsmodel.Status{} offsetStatus := statusID @@ -79,6 +86,35 @@ func (t *timeline) IndexBehind(statusID string, include bool, amount int) error "amount": amount, }) + // lazily initialize index if it hasn't been done already + if t.postIndex.data == nil { + t.postIndex.data = &list.List{} + t.postIndex.data.Init() + } + + // If we're already indexedBehind given statusID by the required amount, we can return nil. + // First find position of statusID (or as near as possible). + var position int +positionLoop: + for e := t.postIndex.data.Front(); e != nil; e = e.Next() { + entry, ok := e.Value.(*postIndexEntry) + if !ok { + return errors.New("IndexBehind: could not parse e as a postIndexEntry") + } + + if entry.statusID <= statusID { + // we've found it + break positionLoop + } + position++ + } + // now check if the length of indexed posts exceeds the amount of posts required (position of statusID, plus amount of posts requested after that) + if t.postIndex.data.Len() > position+amount { + // we have enough indexed behind already to satisfy amount, so don't need to make db calls + l.Trace("returning nil since we already have enough posts indexed") + return nil + } + filtered := []*gtsmodel.Status{} offsetStatus := statusID @@ -168,7 +204,7 @@ func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string func (t *timeline) OldestIndexedPostID() (string, error) { var id string - if t.postIndex == nil || t.postIndex.data == nil { + if t.postIndex == nil || t.postIndex.data == nil || t.postIndex.data.Back() == nil { // return an empty string if postindex hasn't been initialized yet return id, nil } @@ -183,7 +219,7 @@ func (t *timeline) OldestIndexedPostID() (string, error) { func (t *timeline) NewestIndexedPostID() (string, error) { var id string - if t.postIndex == nil || t.postIndex.data == nil { + if t.postIndex == nil || t.postIndex.data == nil || t.postIndex.data.Front() == nil { // return an empty string if postindex hasn't been initialized yet return id, nil } diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go index 06c1f98ec..a9228ea3b 100644 --- a/internal/timeline/prepare.go +++ b/internal/timeline/prepare.go @@ -29,18 +29,30 @@ import ( ) func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, minID string) error { + l := t.log.WithFields(logrus.Fields{ + "func": "prepareNextQuery", + "amount": amount, + "maxID": maxID, + "sinceID": sinceID, + "minID": minID, + }) + + var err error // maxID is defined but sinceID isn't so take from behind if maxID != "" && sinceID == "" { + l.Debug("preparing behind maxID") err = t.PrepareBehind(maxID, amount) } // maxID isn't defined, but sinceID || minID are, so take x before if maxID == "" && sinceID != "" { + l.Debug("preparing before sinceID") err = t.PrepareBefore(sinceID, false, amount) } if maxID == "" && minID != "" { + l.Debug("preparing before minID") err = t.PrepareBefore(minID, false, amount) } @@ -48,15 +60,16 @@ func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, mi } func (t *timeline) PrepareBehind(statusID string, amount int) error { - t.Lock() - defer t.Unlock() - // lazily initialize prepared posts if it hasn't been done already if t.preparedPosts.data == nil { t.preparedPosts.data = &list.List{} t.preparedPosts.data.Init() } + if err := t.IndexBehind(statusID, true, amount); err != nil { + return fmt.Errorf("PrepareBehind: error indexing behind id %s: %s", statusID, err) + } + // if the postindex is nil, nothing has been indexed yet so there's nothing to prepare if t.postIndex.data == nil { return nil @@ -64,6 +77,8 @@ func (t *timeline) PrepareBehind(statusID string, amount int) error { var prepared int var preparing bool + t.Lock() + defer t.Unlock() prepareloop: for e := t.postIndex.data.Front(); e != nil; e = e.Next() { entry, ok := e.Value.(*postIndexEntry)