From 566e2b1d38195fa5680b27ace8f30850dd78d68e Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 20 Mar 2025 13:34:34 +0000 Subject: [PATCH] remove old timeline package, add local timeline cache --- internal/cache/cache.go | 2 + internal/cache/timeline.go | 11 + internal/db/bundb/timeline.go | 105 +--- internal/db/timeline.go | 3 + internal/processing/timeline/public.go | 79 ++- internal/timeline/get.go | 428 --------------- internal/timeline/get_test.go | 704 ------------------------- internal/timeline/index.go | 283 ---------- internal/timeline/index_test.go | 92 ---- internal/timeline/indexeditems.go | 120 ----- internal/timeline/manager.go | 259 --------- internal/timeline/prepare.go | 146 ----- internal/timeline/prune.go | 83 --- internal/timeline/prune_test.go | 103 ---- internal/timeline/remove.go | 97 ---- internal/timeline/timeline.go | 172 ------ internal/timeline/timeline_test.go | 98 ---- internal/timeline/timelines.go | 37 -- internal/timeline/types.go | 34 -- internal/timeline/unprepare.go | 50 -- internal/timeline/unprepare_test.go | 142 ----- 21 files changed, 105 insertions(+), 2943 deletions(-) delete mode 100644 internal/timeline/get.go delete mode 100644 internal/timeline/get_test.go delete mode 100644 internal/timeline/index.go delete mode 100644 internal/timeline/index_test.go delete mode 100644 internal/timeline/indexeditems.go delete mode 100644 internal/timeline/manager.go delete mode 100644 internal/timeline/prepare.go delete mode 100644 internal/timeline/prune.go delete mode 100644 internal/timeline/prune_test.go delete mode 100644 internal/timeline/remove.go delete mode 100644 internal/timeline/timeline.go delete mode 100644 internal/timeline/timeline_test.go delete mode 100644 internal/timeline/timelines.go delete mode 100644 internal/timeline/types.go delete mode 100644 internal/timeline/unprepare.go delete mode 100644 internal/timeline/unprepare_test.go diff --git a/internal/cache/cache.go b/internal/cache/cache.go index a63c35ae9..5a9f015d7 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -98,6 +98,7 @@ func (c *Caches) Init() { c.initListIDs() c.initListedIDs() c.initListTimelines() + c.initLocalTimeline() c.initMarker() c.initMedia() c.initMention() @@ -216,6 +217,7 @@ func (c *Caches) Sweep(threshold float64) { c.Timelines.Home.Trim(threshold) c.Timelines.List.Trim(threshold) c.Timelines.Public.Trim(threshold) + c.Timelines.Local.Trim(threshold) c.Visibility.Trim(threshold) } diff --git a/internal/cache/timeline.go b/internal/cache/timeline.go index 14c91bcb9..3ff56b597 100644 --- a/internal/cache/timeline.go +++ b/internal/cache/timeline.go @@ -32,6 +32,9 @@ type TimelineCaches struct { // Public ... Public timeline.StatusTimeline + + // Local ... + Local timeline.StatusTimeline } func (c *Caches) initHomeTimelines() { @@ -57,3 +60,11 @@ func (c *Caches) initPublicTimeline() { c.Timelines.Public.Init(cap) } + +func (c *Caches) initLocalTimeline() { + cap := 1000 + + log.Infof(nil, "cache size = %d", cap) + + c.Timelines.Local.Init(cap) +} diff --git a/internal/db/bundb/timeline.go b/internal/db/bundb/timeline.go index 9504e9d0d..8bdcb3b61 100644 --- a/internal/db/bundb/timeline.go +++ b/internal/db/bundb/timeline.go @@ -21,13 +21,11 @@ import ( "context" "errors" "slices" - "time" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/paging" "github.com/superseriousbusiness/gotosocial/internal/state" "github.com/uptrace/bun" @@ -150,88 +148,29 @@ func (t *timelineDB) GetPublicTimeline(ctx context.Context, page *paging.Page) ( ) } -func (t *timelineDB) getLocalTimeline( - ctx context.Context, - maxID string, - sinceID string, - minID string, - limit int, -) ([]*gtsmodel.Status, error) { - // Make educated guess for slice size - var ( - statusIDs = make([]string, 0, limit) - frontToBack = true +func (t *timelineDB) GetLocalTimeline(ctx context.Context, page *paging.Page) ([]*gtsmodel.Status, error) { + return loadStatusTimelinePage(ctx, t.db, t.state, + + // Paging + // params. + page, + + func(q *bun.SelectQuery) (*bun.SelectQuery, error) { + // Local only. + q = q.Where("? = ?", bun.Ident("status.local"), true) + + // Public only. + q = q.Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityPublic) + + // Ignore boosts. + q = q.Where("? IS NULL", bun.Ident("boost_of_id")) + + // Only include statuses that aren't pending approval. + q = q.Where("? = ?", bun.Ident("pending_approval"), false) + + return q, nil + }, ) - - q := t.db. - NewSelect(). - TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")). - // Local only. - Where("? = ?", bun.Ident("status.local"), true). - // Public only. - Where("? = ?", bun.Ident("status.visibility"), gtsmodel.VisibilityPublic). - // Only include statuses that aren't pending approval. - Where("? = ?", bun.Ident("status.pending_approval"), false). - // Ignore boosts. - Where("? IS NULL", bun.Ident("status.boost_of_id")). - // Select only IDs from table - Column("status.id") - - if maxID == "" || maxID >= id.Highest { - const future = 24 * time.Hour - - // don't return statuses more than 24hr in the future - maxID = id.NewULIDFromTime(time.Now().Add(future)) - } - - // return only statuses LOWER (ie., older) than maxID - q = q.Where("? < ?", bun.Ident("status.id"), maxID) - - if sinceID != "" { - // return only statuses HIGHER (ie., newer) than sinceID - q = q.Where("? > ?", bun.Ident("status.id"), sinceID) - } - - if minID != "" { - // return only statuses HIGHER (ie., newer) than minID - q = q.Where("? > ?", bun.Ident("status.id"), minID) - - // page up - frontToBack = false - } - - if limit > 0 { - // limit amount of statuses returned - q = q.Limit(limit) - } - - if frontToBack { - // Page down. - q = q.Order("status.id DESC") - } else { - // Page up. - q = q.Order("status.id ASC") - } - - if err := q.Scan(ctx, &statusIDs); err != nil { - return nil, err - } - - if len(statusIDs) == 0 { - return nil, nil - } - - // If we're paging up, we still want statuses - // to be sorted by ID desc, so reverse ids slice. - // https://zchee.github.io/golang-wiki/SliceTricks/#reversing - if !frontToBack { - for l, r := 0, len(statusIDs)-1; l < r; l, r = l+1, r-1 { - statusIDs[l], statusIDs[r] = statusIDs[r], statusIDs[l] - } - } - - // Return status IDs loaded from cache + db. - return t.state.DB.GetStatusesByIDs(ctx, statusIDs) } // TODO optimize this query and the logic here, because it's slow as balls -- it takes like a literal second to return with a limit of 20! diff --git a/internal/db/timeline.go b/internal/db/timeline.go index 68e494261..3d639eb2c 100644 --- a/internal/db/timeline.go +++ b/internal/db/timeline.go @@ -37,6 +37,9 @@ type Timeline interface { // Statuses should be returned in descending order of when they were created (newest first). GetPublicTimeline(ctx context.Context, page *paging.Page) ([]*gtsmodel.Status, error) + // GetLocalTimeline ... + GetLocalTimeline(ctx context.Context, page *paging.Page) ([]*gtsmodel.Status, error) + // GetFavedTimeline fetches the account's FAVED timeline -- ie., posts and replies that the requesting account has faved. // It will use the given filters and try to return as many statuses as possible up to the limit. // diff --git a/internal/processing/timeline/public.go b/internal/processing/timeline/public.go index f6910a4d4..3c02b0d62 100644 --- a/internal/processing/timeline/public.go +++ b/internal/processing/timeline/public.go @@ -19,7 +19,6 @@ package timeline import ( "context" - "net/url" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" @@ -37,6 +36,20 @@ func (p *Processor) PublicTimelineGet( ) ( *apimodel.PageableResponse, gtserror.WithCode, +) { + if local { + return p.localTimelineGet(ctx, requester, page) + } + return p.publicTimelineGet(ctx, requester, page) +} + +func (p *Processor) publicTimelineGet( + ctx context.Context, + requester *gtsmodel.Account, + page *paging.Page, +) ( + *apimodel.PageableResponse, + gtserror.WithCode, ) { return p.getStatusTimeline(ctx, @@ -58,12 +71,7 @@ func (p *Processor) PublicTimelineGet( // page query flag, (this map // later gets copied before // any further usage). - func() url.Values { - if local { - return localOnlyTrue - } - return localOnlyFalse - }(), + localOnlyFalse, // Status filter context. statusfilter.FilterContextPublic, @@ -81,11 +89,58 @@ func (p *Processor) PublicTimelineGet( // i.e. filter after caching. func(s *gtsmodel.Status) (bool, error) { - // Remove any non-local statuses - // if requester wants local-only. - if local && !*s.Local { - return true, nil - } + // Check the visibility of passed status to requesting user. + ok, err := p.visFilter.StatusPublicTimelineable(ctx, requester, s) + return !ok, err + }, + ) +} + +func (p *Processor) localTimelineGet( + ctx context.Context, + requester *gtsmodel.Account, + page *paging.Page, +) ( + *apimodel.PageableResponse, + gtserror.WithCode, +) { + return p.getStatusTimeline(ctx, + + // Auth'd + // account. + requester, + + // Global local timeline cache. + &p.state.Caches.Timelines.Local, + + // Current + // page. + page, + + // Public timeline endpoint. + "/api/v1/timelines/public", + + // Set local-only timeline + // page query flag, (this map + // later gets copied before + // any further usage). + localOnlyTrue, + + // Status filter context. + statusfilter.FilterContextPublic, + + // Database load function. + func(pg *paging.Page) (statuses []*gtsmodel.Status, err error) { + return p.state.DB.GetLocalTimeline(ctx, pg) + }, + + // Pre-filtering function, + // i.e. filter before caching. + nil, + + // Post-filtering function, + // i.e. filter after caching. + func(s *gtsmodel.Status) (bool, error) { // Check the visibility of passed status to requesting user. ok, err := p.visFilter.StatusPublicTimelineable(ctx, requester, s) diff --git a/internal/timeline/get.go b/internal/timeline/get.go deleted file mode 100644 index 06ee8c174..000000000 --- a/internal/timeline/get.go +++ /dev/null @@ -1,428 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "container/list" - "context" - "errors" - "time" - - "codeberg.org/gruf/go-kv" - "github.com/superseriousbusiness/gotosocial/internal/db" - statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -func (t *timeline) LastGot() time.Time { - t.Lock() - defer t.Unlock() - return t.lastGot -} - -func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error) { - l := log.WithContext(ctx). - WithFields(kv.Fields{ - {"accountID", t.timelineID}, - {"amount", amount}, - {"maxID", maxID}, - {"sinceID", sinceID}, - {"minID", minID}, - }...) - l.Trace("entering get and updating t.lastGot") - - // Regardless of what happens below, update the - // last time Get was called for this timeline. - t.Lock() - t.lastGot = time.Now() - t.Unlock() - - var ( - items []Preparable - err error - ) - - switch { - case maxID == "" && sinceID == "" && minID == "": - // No params are defined so just fetch from the top. - // This is equivalent to a user starting to view - // their timeline from newest -> older posts. - items, err = t.getXBetweenIDs(ctx, amount, id.Highest, id.Lowest, true) - - // Cache expected next query to speed up scrolling. - // Assume the user will be scrolling downwards from - // the final ID in items. - if prepareNext && err == nil && len(items) != 0 { - nextMaxID := items[len(items)-1].GetID() - t.prepareNextQuery(amount, nextMaxID, "", "") - } - - case maxID != "" && sinceID == "" && minID == "": - // Only maxID is defined, so fetch from maxID onwards. - // This is equivalent to a user paging further down - // their timeline from newer -> older posts. - items, err = t.getXBetweenIDs(ctx, amount, maxID, id.Lowest, true) - - // Cache expected next query to speed up scrolling. - // Assume the user will be scrolling downwards from - // the final ID in items. - if prepareNext && err == nil && len(items) != 0 { - nextMaxID := items[len(items)-1].GetID() - t.prepareNextQuery(amount, nextMaxID, "", "") - } - - // In the next cases, maxID is defined, and so are - // either sinceID or minID. This is equivalent to - // a user opening an in-progress timeline and asking - // for a slice of posts somewhere in the middle, or - // trying to "fill in the blanks" between two points, - // paging either up or down. - case maxID != "" && sinceID != "": - items, err = t.getXBetweenIDs(ctx, amount, maxID, sinceID, true) - - // Cache expected next query to speed up scrolling. - // We can assume the caller is scrolling downwards. - // Guess id.Lowest as sinceID, since we don't actually - // know what the next sinceID would be. - if prepareNext && err == nil && len(items) != 0 { - nextMaxID := items[len(items)-1].GetID() - t.prepareNextQuery(amount, nextMaxID, id.Lowest, "") - } - - case maxID != "" && minID != "": - items, err = t.getXBetweenIDs(ctx, amount, maxID, minID, false) - - // Cache expected next query to speed up scrolling. - // We can assume the caller is scrolling upwards. - // Guess id.Highest as maxID, since we don't actually - // know what the next maxID would be. - if prepareNext && err == nil && len(items) != 0 { - prevMinID := items[0].GetID() - t.prepareNextQuery(amount, id.Highest, "", prevMinID) - } - - // In the final cases, maxID is not defined, but - // either sinceID or minID are. This is equivalent to - // a user either "pulling up" at the top of their timeline - // to refresh it and check if newer posts have come in, or - // trying to scroll upwards from an old post to see what - // they missed since then. - // - // In these calls, we use the highest possible ulid as - // behindID because we don't have a cap for newest that - // we're interested in. - case maxID == "" && sinceID != "": - items, err = t.getXBetweenIDs(ctx, amount, id.Highest, sinceID, true) - - // We can't cache an expected next query for this one, - // since presumably the caller is at the top of their - // timeline already. - - case maxID == "" && minID != "": - items, err = t.getXBetweenIDs(ctx, amount, id.Highest, minID, false) - - // Cache expected next query to speed up scrolling. - // We can assume the caller is scrolling upwards. - // Guess id.Highest as maxID, since we don't actually - // know what the next maxID would be. - if prepareNext && err == nil && len(items) != 0 { - prevMinID := items[0].GetID() - t.prepareNextQuery(amount, id.Highest, "", prevMinID) - } - - default: - err = gtserror.New("switch statement exhausted with no results") - } - - return items, err -} - -// getXBetweenIDs returns x amount of items somewhere between (not including) the given IDs. -// -// If frontToBack is true, items will be served paging down from behindID. -// This corresponds to an api call to /timelines/home?max_id=WHATEVER&since_id=WHATEVER -// -// If frontToBack is false, items will be served paging up from beforeID. -// This corresponds to an api call to /timelines/home?max_id=WHATEVER&min_id=WHATEVER -func (t *timeline) getXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) ([]Preparable, error) { - l := log. - WithContext(ctx). - WithFields(kv.Fields{ - {"amount", amount}, - {"behindID", behindID}, - {"beforeID", beforeID}, - {"frontToBack", frontToBack}, - }...) - l.Trace("entering getXBetweenID") - - // Assume length we need to return. - items := make([]Preparable, 0, amount) - - if beforeID >= behindID { - // This is an impossible situation, we - // can't serve anything between these. - return items, nil - } - - // Try to ensure we have enough items prepared. - if err := t.prepareXBetweenIDs(ctx, amount, behindID, beforeID, frontToBack); err != nil { - // An error here doesn't necessarily mean we - // can't serve anything, so log + keep going. - l.Debugf("error calling prepareXBetweenIDs: %s", err) - } - - var ( - beforeIDMark *list.Element - served int - // Our behavior while ranging through the - // list changes depending on if we're - // going front-to-back or back-to-front. - // - // To avoid checking which one we're doing - // in each loop iteration, define our range - // function here outside the loop. - // - // The bool indicates to the caller whether - // iteration should continue (true) or stop - // (false). - rangeF func(e *list.Element) (bool, error) - // If we get certain errors on entries as we're - // looking through, we might want to cheekily - // remove their elements from the timeline. - // Everything added to this slice will be removed. - removeElements = []*list.Element{} - ) - - defer func() { - for _, e := range removeElements { - t.items.data.Remove(e) - } - }() - - if frontToBack { - // We're going front-to-back, which means we - // don't need to look for a mark per se, we - // just keep serving items until we've reached - // a point where the items are out of the range - // we're interested in. - rangeF = func(e *list.Element) (bool, error) { - entry := e.Value.(*indexedItemsEntry) - - if entry.itemID >= behindID { - // ID of this item is too high, - // just keep iterating. - l.Trace("item is too new, continuing") - return true, nil - } - - if entry.itemID <= beforeID { - // We've gone as far as we can through - // the list and reached entries that are - // now too old for us, stop here. - l.Trace("reached older items, breaking") - return false, nil - } - - l.Trace("entry is just right") - - if entry.prepared == nil { - // Whoops, this entry isn't prepared yet; some - // race condition? That's OK, we can do it now. - prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID) - if err != nil { - if errors.Is(err, statusfilter.ErrHideStatus) { - // This item has been filtered out by the requesting user's filters. - // Remove it and skip past it. - removeElements = append(removeElements, e) - return true, nil - } - if errors.Is(err, db.ErrNoEntries) { - // ErrNoEntries means something has been deleted, - // so we'll likely not be able to ever prepare this. - // This means we can remove it and skip past it. - l.Debugf("db.ErrNoEntries while trying to prepare %s; will remove from timeline", entry.itemID) - removeElements = append(removeElements, e) - return true, nil - } - // We've got a proper db error. - err = gtserror.Newf("db error while trying to prepare %s: %w", entry.itemID, err) - return false, err - } - entry.prepared = prepared - } - - items = append(items, entry.prepared) - - served++ - return served < amount, nil - } - } else { - // Iterate through the list from the top, until - // we reach an item with id smaller than beforeID; - // ie., an item OLDER than beforeID. At that point, - // we can stop looking because we're not interested - // in older entries. - rangeF = func(e *list.Element) (bool, error) { - // Move the mark back one place each loop. - beforeIDMark = e - - if entry := e.Value.(*indexedItemsEntry); entry.itemID <= beforeID { - // We've gone as far as we can through - // the list and reached entries that are - // now too old for us, stop here. - l.Trace("reached older items, breaking") - return false, nil - } - - return true, nil - } - } - - // Iterate through the list until the function - // we defined above instructs us to stop. - for e := t.items.data.Front(); e != nil; e = e.Next() { - keepGoing, err := rangeF(e) - if err != nil { - return nil, err - } - - if !keepGoing { - break - } - } - - if frontToBack || beforeIDMark == nil { - // If we're serving front to back, then - // items should be populated by now. If - // we're serving back to front but didn't - // find any items newer than beforeID, - // we can just return empty items. - return items, nil - } - - // We're serving back to front, so iterate upwards - // towards the front of the list from the mark we found, - // until we either get to the front, serve enough - // items, or reach behindID. - // - // To preserve ordering, we need to reverse the slice - // when we're finished. - for e := beforeIDMark; e != nil; e = e.Prev() { - entry := e.Value.(*indexedItemsEntry) - - if entry.itemID == beforeID { - // Don't include the beforeID - // entry itself, just continue. - l.Trace("entry item ID is equal to beforeID, skipping") - continue - } - - if entry.itemID >= behindID { - // We've reached items that are - // newer than what we're looking - // for, just stop here. - l.Trace("reached newer items, breaking") - break - } - - if entry.prepared == nil { - // Whoops, this entry isn't prepared yet; some - // race condition? That's OK, we can do it now. - prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID) - if err != nil { - if errors.Is(err, statusfilter.ErrHideStatus) { - // This item has been filtered out by the requesting user's filters. - // Remove it and skip past it. - removeElements = append(removeElements, e) - continue - } - if errors.Is(err, db.ErrNoEntries) { - // ErrNoEntries means something has been deleted, - // so we'll likely not be able to ever prepare this. - // This means we can remove it and skip past it. - l.Debugf("db.ErrNoEntries while trying to prepare %s; will remove from timeline", entry.itemID) - removeElements = append(removeElements, e) - continue - } - // We've got a proper db error. - err = gtserror.Newf("db error while trying to prepare %s: %w", entry.itemID, err) - return nil, err - } - entry.prepared = prepared - } - - items = append(items, entry.prepared) - - served++ - if served >= amount { - break - } - } - - // Reverse order of items. - // https://zchee.github.io/golang-wiki/SliceTricks/#reversing - for l, r := 0, len(items)-1; l < r; l, r = l+1, r-1 { - items[l], items[r] = items[r], items[l] - } - - return items, nil -} - -func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, minID string) { - var ( - // We explicitly use context.Background() rather than - // accepting a context param because we don't want this - // to stop/break when the calling context finishes. - ctx = context.Background() - err error - ) - - // Always perform this async so caller doesn't have to wait. - go func() { - switch { - case maxID == "" && sinceID == "" && minID == "": - err = t.prepareXBetweenIDs(ctx, amount, id.Highest, id.Lowest, true) - case maxID != "" && sinceID == "" && minID == "": - err = t.prepareXBetweenIDs(ctx, amount, maxID, id.Lowest, true) - case maxID != "" && sinceID != "": - err = t.prepareXBetweenIDs(ctx, amount, maxID, sinceID, true) - case maxID != "" && minID != "": - err = t.prepareXBetweenIDs(ctx, amount, maxID, minID, false) - case maxID == "" && sinceID != "": - err = t.prepareXBetweenIDs(ctx, amount, id.Highest, sinceID, true) - case maxID == "" && minID != "": - err = t.prepareXBetweenIDs(ctx, amount, id.Highest, minID, false) - default: - err = gtserror.New("switch statement exhausted with no results") - } - - if err != nil { - log. - WithContext(ctx). - WithFields(kv.Fields{ - {"amount", amount}, - {"maxID", maxID}, - {"sinceID", sinceID}, - {"minID", minID}, - }...). - Warnf("error preparing next query: %s", err) - } - }() -} diff --git a/internal/timeline/get_test.go b/internal/timeline/get_test.go deleted file mode 100644 index 91a456560..000000000 --- a/internal/timeline/get_test.go +++ /dev/null @@ -1,704 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline_test - -import ( - "context" - "sync" - "testing" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/gtscontext" - "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/timeline" -) - -type GetTestSuite struct { - TimelineStandardTestSuite -} - -func (suite *GetTestSuite) checkStatuses(statuses []timeline.Preparable, maxID string, minID string, expectedLength int) { - if l := len(statuses); l != expectedLength { - suite.FailNow("", "expected %d statuses in slice, got %d", expectedLength, l) - } else if l == 0 { - // Can't test empty slice. - return - } - - // Check ordering + bounds of statuses. - highest := statuses[0].GetID() - for _, status := range statuses { - id := status.GetID() - - if id >= maxID { - suite.FailNow("", "%s greater than maxID %s", id, maxID) - } - - if id <= minID { - suite.FailNow("", "%s smaller than minID %s", id, minID) - } - - if id > highest { - suite.FailNow("", "statuses in slice were not ordered highest -> lowest ID") - } - - highest = id - } -} - -func (suite *GetTestSuite) emptyAccountFollows(ctx context.Context, accountID string) { - // Get all of account's follows. - follows, err := suite.state.DB.GetAccountFollows( - gtscontext.SetBarebones(ctx), - accountID, - nil, // select all - ) - if err != nil { - suite.FailNow(err.Error()) - } - - // Remove each follow. - for _, follow := range follows { - if err := suite.state.DB.DeleteFollowByID(ctx, follow.ID); err != nil { - suite.FailNow(err.Error()) - } - } - - // Ensure no follows left. - follows, err = suite.state.DB.GetAccountFollows( - gtscontext.SetBarebones(ctx), - accountID, - nil, // select all - ) - if err != nil { - suite.FailNow(err.Error()) - } - if len(follows) != 0 { - suite.FailNow("follows should be empty") - } -} - -func (suite *GetTestSuite) emptyAccountStatuses(ctx context.Context, accountID string) { - // Get all of account's statuses. - statuses, err := suite.state.DB.GetAccountStatuses( - ctx, - accountID, - 9999, - false, - false, - id.Highest, - id.Lowest, - false, - false, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - // Remove each status. - for _, status := range statuses { - if err := suite.state.DB.DeleteStatusByID(ctx, status.ID); err != nil { - suite.FailNow(err.Error()) - } - } -} - -func (suite *GetTestSuite) TestGetNewTimelinePageDown() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "" - limit = 5 - local = false - ) - - // Get 5 from the top. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 5) - - // Get 5 from next maxID. - maxID = statuses[len(statuses)-1].GetID() - statuses, err = suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - suite.checkStatuses(statuses, maxID, id.Lowest, 5) -} - -func (suite *GetTestSuite) TestGetNewTimelinePageUp() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = id.Lowest - limit = 5 - local = false - ) - - // Get 5 from the back. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - suite.checkStatuses(statuses, id.Highest, minID, 5) - - // Page up from next minID. - minID = statuses[0].GetID() - statuses, err = suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - suite.checkStatuses(statuses, id.Highest, minID, 5) -} - -func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossible() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "" - limit = 100 - local = false - ) - - // Get 100 from the top. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 22) -} - -func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossiblePageUp() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = id.Lowest - limit = 100 - local = false - ) - - // Get 100 from the back. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 22) -} - -func (suite *GetTestSuite) TestGetNewTimelineNoFollowing() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "" - limit = 10 - local = false - ) - - suite.emptyAccountFollows(ctx, testAccount.ID) - - // Try to get 10 from the top of the timeline. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 9) - - for _, s := range statuses { - if s.GetAccountID() != testAccount.ID { - suite.FailNow("timeline with no follows should only contain posts by timeline owner account") - } - } -} - -func (suite *GetTestSuite) TestGetNewTimelineNoFollowingNoStatuses() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "" - limit = 5 - local = false - ) - - suite.emptyAccountFollows(ctx, testAccount.ID) - suite.emptyAccountStatuses(ctx, testAccount.ID) - - // Try to get 5 from the top of the timeline. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - suite.checkStatuses(statuses, id.Highest, id.Lowest, 0) -} - -func (suite *GetTestSuite) TestGetNoParams() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "" - limit = 10 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Get 10 statuses from the top (no params). - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - suite.checkStatuses(statuses, id.Highest, id.Lowest, 10) - - // First status should have the highest ID in the testrig. - suite.Equal(suite.highestStatusID, statuses[0].GetID()) -} - -func (suite *GetTestSuite) TestGetMaxID() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "01F8MHBQCBTDKN6X5VHGMMN4MA" - sinceID = "" - minID = "" - limit = 10 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 10 with a max ID somewhere in the middle of the stack. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - // We'll only get 6 statuses back. - suite.checkStatuses(statuses, maxID, id.Lowest, 6) -} - -func (suite *GetTestSuite) TestGetSinceID() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "01F8MHBQCBTDKN6X5VHGMMN4MA" - minID = "" - limit = 10 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 10 with a since ID somewhere in the middle of the stack. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - suite.checkStatuses(statuses, id.Highest, sinceID, 10) - - // The first status in the stack should have the highest ID of all - // in the testrig, because we're paging down. - suite.Equal(suite.highestStatusID, statuses[0].GetID()) -} - -func (suite *GetTestSuite) TestGetSinceIDOneOnly() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "01F8MHBQCBTDKN6X5VHGMMN4MA" - minID = "" - limit = 1 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 1 with a since ID somewhere in the middle of the stack. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - suite.checkStatuses(statuses, id.Highest, sinceID, 1) - - // The one status we got back should have the highest ID of all in - // the testrig, because using sinceID means we're paging down. - suite.Equal(suite.highestStatusID, statuses[0].GetID()) -} - -func (suite *GetTestSuite) TestGetMinID() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "01F8MHBQCBTDKN6X5VHGMMN4MA" - limit = 5 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 5 with a min ID somewhere in the middle of the stack. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - suite.checkStatuses(statuses, id.Highest, minID, 5) - - // We're paging up so even the highest status ID in the pile - // shouldn't be the highest ID we have. - suite.NotEqual(suite.highestStatusID, statuses[0]) -} - -func (suite *GetTestSuite) TestGetMinIDOneOnly() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "01F8MHBQCBTDKN6X5VHGMMN4MA" - limit = 1 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 1 with a min ID somewhere in the middle of the stack. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - suite.checkStatuses(statuses, id.Highest, minID, 1) - - // The one status we got back should have the an ID equal to the - // one ID immediately newer than it. - suite.Equal("01F8MHC0H0A7XHTVH5F596ZKBM", statuses[0].GetID()) -} - -func (suite *GetTestSuite) TestGetMinIDFromLowestInTestrig() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = suite.lowestStatusID - limit = 1 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 1 with minID equal to the lowest status in the testrig. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - suite.checkStatuses(statuses, id.Highest, minID, 1) - - // The one status we got back should have an id higher than - // the lowest status in the testrig, since minID is not inclusive. - suite.Greater(statuses[0].GetID(), suite.lowestStatusID) -} - -func (suite *GetTestSuite) TestGetMinIDFromLowestPossible() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = id.Lowest - limit = 1 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 1 with the lowest possible min ID. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - suite.checkStatuses(statuses, id.Highest, minID, 1) - - // The one status we got back should have the an ID equal to the - // lowest ID status in the test rig. - suite.Equal(suite.lowestStatusID, statuses[0].GetID()) -} - -func (suite *GetTestSuite) TestGetBetweenID() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "01F8MHCP5P2NWYQ416SBA0XSEV" - sinceID = "" - minID = "01F8MHBQCBTDKN6X5VHGMMN4MA" - limit = 10 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 10 between these two IDs - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - // There's only two statuses between these two IDs. - suite.checkStatuses(statuses, maxID, minID, 2) -} - -func (suite *GetTestSuite) TestGetBetweenIDImpossible() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = id.Lowest - sinceID = "" - minID = id.Highest - limit = 10 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Ask for 10 between these two IDs which present - // an impossible query. - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - // We should have nothing back. - suite.checkStatuses(statuses, maxID, minID, 0) -} - -func (suite *GetTestSuite) TestGetTimelinesAsync() { - var ( - ctx = context.Background() - accountToNuke = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "" - limit = 5 - local = false - multiplier = 5 - ) - - // Nuke one account's statuses and follows, - // as though the account had just been created. - suite.emptyAccountFollows(ctx, accountToNuke.ID) - suite.emptyAccountStatuses(ctx, accountToNuke.ID) - - // Get 5 statuses from each timeline in - // our testrig at the same time, five times. - wg := new(sync.WaitGroup) - wg.Add(len(suite.testAccounts) * multiplier) - - for i := 0; i < multiplier; i++ { - go func() { - for _, testAccount := range suite.testAccounts { - if _, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ); err != nil { - suite.Fail(err.Error()) - } - - wg.Done() - } - }() - } - - wg.Wait() // Wait until all get calls have returned. -} - -func TestGetTestSuite(t *testing.T) { - suite.Run(t, new(GetTestSuite)) -} diff --git a/internal/timeline/index.go b/internal/timeline/index.go deleted file mode 100644 index 6abb6d28d..000000000 --- a/internal/timeline/index.go +++ /dev/null @@ -1,283 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "container/list" - "context" - "errors" - - "codeberg.org/gruf/go-kv" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -func (t *timeline) indexXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) error { - l := log. - WithContext(ctx). - WithFields(kv.Fields{ - {"amount", amount}, - {"behindID", behindID}, - {"beforeID", beforeID}, - {"frontToBack", frontToBack}, - }...) - l.Trace("entering indexXBetweenIDs") - - if beforeID >= behindID { - // This is an impossible situation, we - // can't index anything between these. - return nil - } - - t.Lock() - defer t.Unlock() - - // Lazily init indexed items. - if t.items.data == nil { - t.items.data = &list.List{} - t.items.data.Init() - } - - // Start by mapping out the list so we know what - // we have to do. Depending on the current state - // of the list we might not have to do *anything*. - var ( - position int - listLen = t.items.data.Len() - behindIDPosition int - beforeIDPosition int - ) - - for e := t.items.data.Front(); e != nil; e = e.Next() { - entry := e.Value.(*indexedItemsEntry) - - position++ - - if entry.itemID > behindID { - l.Trace("item is too new, continuing") - continue - } - - if behindIDPosition == 0 { - // Gone far enough through the list - // and found our behindID mark. - // We only need to set this once. - l.Tracef("found behindID mark %s at position %d", entry.itemID, position) - behindIDPosition = position - } - - if entry.itemID >= beforeID { - // Push the beforeID mark back - // one place every iteration. - l.Tracef("setting beforeID mark %s at position %d", entry.itemID, position) - beforeIDPosition = position - } - - if entry.itemID <= beforeID { - // We've gone beyond the bounds of - // items we're interested in; stop. - l.Trace("reached older items, breaking") - break - } - } - - // We can now figure out if we need to make db calls. - var grabMore bool - switch { - case listLen < amount: - // The whole list is shorter than the - // amount we're being asked to return, - // make up the difference. - grabMore = true - amount -= listLen - case beforeIDPosition-behindIDPosition < amount: - // Not enough items between behindID and - // beforeID to return amount required, - // try to get more. - grabMore = true - } - - if !grabMore { - // We're good! - return nil - } - - // Fetch additional items. - items, err := t.grab(ctx, amount, behindID, beforeID, frontToBack) - if err != nil { - return err - } - - // Index all the items we got. We already have - // a lock on the timeline, so don't call IndexOne - // here, since that will also try to get a lock! - for _, item := range items { - entry := &indexedItemsEntry{ - itemID: item.GetID(), - boostOfID: item.GetBoostOfID(), - accountID: item.GetAccountID(), - boostOfAccountID: item.GetBoostOfAccountID(), - } - - if _, err := t.items.insertIndexed(ctx, entry); err != nil { - return gtserror.Newf("error inserting entry with itemID %s into index: %w", entry.itemID, err) - } - } - - return nil -} - -// grab wraps the timeline's grabFunction in paging + filtering logic. -func (t *timeline) grab(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) ([]Timelineable, error) { - var ( - sinceID string - minID string - grabbed int - maxID = behindID - filtered = make([]Timelineable, 0, amount) - ) - - if frontToBack { - sinceID = beforeID - } else { - minID = beforeID - } - - for attempts := 0; attempts < 5; attempts++ { - if grabbed >= amount { - // We got everything we needed. - break - } - - items, stop, err := t.grabFunction( - ctx, - t.timelineID, - maxID, - sinceID, - minID, - // Don't grab more than we need to. - amount-grabbed, - ) - if err != nil { - // Grab function already checks for - // db.ErrNoEntries, so if an error - // is returned then it's a real one. - return nil, err - } - - if stop || len(items) == 0 { - // No items left. - break - } - - // Set next query parameters. - if frontToBack { - // Page down. - maxID = items[len(items)-1].GetID() - if maxID <= beforeID { - // Can't go any further. - break - } - } else { - // Page up. - minID = items[0].GetID() - if minID >= behindID { - // Can't go any further. - break - } - } - - for _, item := range items { - ok, err := t.filterFunction(ctx, t.timelineID, item) - if err != nil { - if !errors.Is(err, db.ErrNoEntries) { - // Real error here. - return nil, err - } - log.Warnf(ctx, "errNoEntries while filtering item %s: %s", item.GetID(), err) - continue - } - - if ok { - filtered = append(filtered, item) - grabbed++ // count this as grabbed - } - } - } - - return filtered, nil -} - -func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) { - t.Lock() - defer t.Unlock() - - postIndexEntry := &indexedItemsEntry{ - itemID: statusID, - boostOfID: boostOfID, - accountID: accountID, - boostOfAccountID: boostOfAccountID, - } - - if inserted, err := t.items.insertIndexed(ctx, postIndexEntry); err != nil { - return false, gtserror.Newf("error inserting indexed: %w", err) - } else if !inserted { - // Entry wasn't inserted, so - // don't bother preparing it. - return false, nil - } - - preparable, err := t.prepareFunction(ctx, t.timelineID, statusID) - if err != nil { - return true, gtserror.Newf("error preparing: %w", err) - } - postIndexEntry.prepared = preparable - - return true, nil -} - -func (t *timeline) Len() int { - t.Lock() - defer t.Unlock() - - if t.items == nil || t.items.data == nil { - // indexedItems hasnt been initialized yet. - return 0 - } - - return t.items.data.Len() -} - -func (t *timeline) OldestIndexedItemID() string { - t.Lock() - defer t.Unlock() - - if t.items == nil || t.items.data == nil { - // indexedItems hasnt been initialized yet. - return "" - } - - e := t.items.data.Back() - if e == nil { - // List was empty. - return "" - } - - return e.Value.(*indexedItemsEntry).itemID -} diff --git a/internal/timeline/index_test.go b/internal/timeline/index_test.go deleted file mode 100644 index a7eeebb6e..000000000 --- a/internal/timeline/index_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline_test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" -) - -type IndexTestSuite struct { - TimelineStandardTestSuite -} - -func (suite *IndexTestSuite) TestOldestIndexedItemIDEmpty() { - var ( - ctx = context.Background() - testAccountID = suite.testAccounts["local_account_1"].ID - ) - - // the oldest indexed post should be an empty string since there's nothing indexed yet - postID := suite.state.Timelines.Home.GetOldestIndexedID(ctx, testAccountID) - suite.Empty(postID) - - // indexLength should be 0 - suite.Zero(0, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) -} - -func (suite *IndexTestSuite) TestIndexAlreadyIndexed() { - var ( - ctx = context.Background() - testAccountID = suite.testAccounts["local_account_1"].ID - testStatus = suite.testStatuses["local_account_1_status_1"] - ) - - // index one post -- it should be indexed - indexed, err := suite.state.Timelines.Home.IngestOne(ctx, testAccountID, testStatus) - suite.NoError(err) - suite.True(indexed) - - // try to index the same post again -- it should not be indexed - indexed, err = suite.state.Timelines.Home.IngestOne(ctx, testAccountID, testStatus) - suite.NoError(err) - suite.False(indexed) -} - -func (suite *IndexTestSuite) TestIndexBoostOfAlreadyIndexed() { - var ( - ctx = context.Background() - testAccountID = suite.testAccounts["local_account_1"].ID - testStatus = suite.testStatuses["local_account_1_status_1"] - boostOfTestStatus = >smodel.Status{ - CreatedAt: time.Now(), - ID: "01FD4TA6G2Z6M7W8NJQ3K5WXYD", - BoostOfID: testStatus.ID, - AccountID: "01FD4TAY1C0NGEJVE9CCCX7QKS", - BoostOfAccountID: testStatus.AccountID, - } - ) - - // index one post -- it should be indexed - indexed, err := suite.state.Timelines.Home.IngestOne(ctx, testAccountID, testStatus) - suite.NoError(err) - suite.True(indexed) - - // try to index the a boost of that post -- it should not be indexed - indexed, err = suite.state.Timelines.Home.IngestOne(ctx, testAccountID, boostOfTestStatus) - suite.NoError(err) - suite.False(indexed) -} - -func TestIndexTestSuite(t *testing.T) { - suite.Run(t, new(IndexTestSuite)) -} diff --git a/internal/timeline/indexeditems.go b/internal/timeline/indexeditems.go deleted file mode 100644 index 9b75e7256..000000000 --- a/internal/timeline/indexeditems.go +++ /dev/null @@ -1,120 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "container/list" - "context" - - "github.com/superseriousbusiness/gotosocial/internal/gtserror" -) - -type indexedItems struct { - data *list.List - skipInsert SkipInsertFunction -} - -type indexedItemsEntry struct { - itemID string - boostOfID string - accountID string - boostOfAccountID string - prepared Preparable -} - -// WARNING: ONLY CALL THIS FUNCTION IF YOU ALREADY HAVE -// A LOCK ON THE TIMELINE CONTAINING THIS INDEXEDITEMS! -func (i *indexedItems) insertIndexed(ctx context.Context, newEntry *indexedItemsEntry) (bool, error) { - // Lazily init indexed items. - if i.data == nil { - i.data = &list.List{} - i.data.Init() - } - - if i.data.Len() == 0 { - // We have no entries yet, meaning this is both the - // newest + oldest entry, so just put it in the front. - i.data.PushFront(newEntry) - return true, nil - } - - var ( - insertMark *list.Element - currentPosition int - ) - - // We need to iterate through the index to make sure we put - // this item in the appropriate place according to its id. - // We also need to make sure we're not inserting a duplicate - // item -- this can happen sometimes and it's sucky UX. - for e := i.data.Front(); e != nil; e = e.Next() { - currentPosition++ - - currentEntry := e.Value.(*indexedItemsEntry) - - // Check if we need to skip inserting this item based on - // the current item. - // - // For example, if the new item is a boost, and the current - // item is the original, we may not want to insert the boost - // if it would appear very shortly after the original. - if skip, err := i.skipInsert( - ctx, - newEntry.itemID, - newEntry.accountID, - newEntry.boostOfID, - newEntry.boostOfAccountID, - currentEntry.itemID, - currentEntry.accountID, - currentEntry.boostOfID, - currentEntry.boostOfAccountID, - currentPosition, - ); err != nil { - return false, gtserror.Newf("error calling skipInsert: %w", err) - } else if skip { - // We don't need to insert this at all, - // so we can safely bail. - return false, nil - } - - if insertMark != nil { - // We already found our mark. - continue - } - - if currentEntry.itemID > newEntry.itemID { - // We're still in items newer than - // the one we're trying to insert. - continue - } - - // We found our spot! - insertMark = e - } - - if insertMark == nil { - // We looked through the whole timeline and didn't find - // a mark, so the new item is the oldest item we've seen; - // insert it at the back. - i.data.PushBack(newEntry) - return true, nil - } - - i.data.InsertBefore(newEntry, insertMark) - return true, nil -} diff --git a/internal/timeline/manager.go b/internal/timeline/manager.go deleted file mode 100644 index b4f075138..000000000 --- a/internal/timeline/manager.go +++ /dev/null @@ -1,259 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "context" - "sync" - "time" - - "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -const ( - pruneLengthIndexed = 400 - pruneLengthPrepared = 50 -) - -// Manager abstracts functions for creating multiple timelines, and adding, removing, and fetching entries from those timelines. -// -// By the time a timelineable hits the manager interface, it should already have been filtered and it should be established that the item indeed -// belongs in the given timeline. -// -// The manager makes a distinction between *indexed* items and *prepared* items. -// -// Indexed items consist of just that item's ID (in the database) and the time it was created. An indexed item takes up very little memory, so -// it's not a huge priority to keep trimming the indexed items list. -// -// Prepared items consist of the item's database ID, the time it was created, AND the apimodel representation of that item, for quick serialization. -// Prepared items of course take up more memory than indexed items, so they should be regularly pruned if they're not being actively served. -type Manager interface { - // IngestOne takes one timelineable and indexes it into the given timeline, and then immediately prepares it for serving. - // This is useful in cases where we know the item will need to be shown at the top of a user's timeline immediately (eg., a new status is created). - // - // It should already be established before calling this function that the item actually belongs in the timeline! - // - // The returned bool indicates whether the item was actually put in the timeline. This could be false in cases where - // a status is a boost, but a boost of the original status or the status itself already exists recently in the timeline. - IngestOne(ctx context.Context, timelineID string, item Timelineable) (bool, error) - - // GetTimeline returns limit n amount of prepared entries from the given timeline, in descending chronological order. - GetTimeline(ctx context.Context, timelineID string, maxID string, sinceID string, minID string, limit int, local bool) ([]Preparable, error) - - // GetIndexedLength returns the amount of items that have been indexed for the given account ID. - GetIndexedLength(ctx context.Context, timelineID string) int - - // GetOldestIndexedID returns the id ID for the oldest item that we have indexed for the given timeline. - // Will be an empty string if nothing is (yet) indexed. - GetOldestIndexedID(ctx context.Context, timelineID string) string - - // Remove removes one item from the given timeline. - Remove(ctx context.Context, timelineID string, itemID string) (int, error) - - // RemoveTimeline completely removes one timeline. - RemoveTimeline(ctx context.Context, timelineID string) error - - // WipeItemFromAllTimelines removes one item from the index and prepared items of all timelines - WipeItemFromAllTimelines(ctx context.Context, itemID string) error - - // WipeStatusesFromAccountID removes all items by the given accountID from the given timeline. - WipeItemsFromAccountID(ctx context.Context, timelineID string, accountID string) error - - // UnprepareItem unprepares/uncaches the prepared version fo the given itemID from the given timelineID. - // Use this for cache invalidation when the prepared representation of an item has changed. - UnprepareItem(ctx context.Context, timelineID string, itemID string) error - - // UnprepareItemFromAllTimelines unprepares/uncaches the prepared version of the given itemID from all timelines. - // Use this for cache invalidation when the prepared representation of an item has changed. - UnprepareItemFromAllTimelines(ctx context.Context, itemID string) error - - // Prune manually triggers a prune operation for the given timelineID. - Prune(ctx context.Context, timelineID string, desiredPreparedItemsLength int, desiredIndexedItemsLength int) (int, error) - - // Start starts hourly cleanup jobs for this timeline manager. - Start() error - - // Stop stops the timeline manager (currently a stub, doesn't do anything). - Stop() error -} - -// NewManager returns a new timeline manager. -func NewManager(grabFunction GrabFunction, filterFunction FilterFunction, prepareFunction PrepareFunction, skipInsertFunction SkipInsertFunction) Manager { - return &manager{ - timelines: sync.Map{}, - grabFunction: grabFunction, - filterFunction: filterFunction, - prepareFunction: prepareFunction, - skipInsertFunction: skipInsertFunction, - } -} - -type manager struct { - timelines sync.Map - grabFunction GrabFunction - filterFunction FilterFunction - prepareFunction PrepareFunction - skipInsertFunction SkipInsertFunction -} - -func (m *manager) Start() error { - // Start a background goroutine which iterates - // through all stored timelines once per hour, - // and cleans up old entries if that timeline - // hasn't been accessed in the last hour. - go func() { - for now := range time.NewTicker(1 * time.Hour).C { - now := now // rescope - // Define the range function inside here, - // so that we can use the 'now' returned - // by the ticker, instead of having to call - // time.Now() multiple times. - // - // Unless it panics, this function always - // returns 'true', to continue the Range - // call through the sync.Map. - f := func(_ any, v any) bool { - timeline, ok := v.(Timeline) - if !ok { - log.Panic(nil, "couldn't parse timeline manager sync map value as Timeline, this should never happen so panic") - } - - if now.Sub(timeline.LastGot()) < 1*time.Hour { - // Timeline has been fetched in the - // last hour, move on to the next one. - return true - } - - if amountPruned := timeline.Prune(pruneLengthPrepared, pruneLengthIndexed); amountPruned > 0 { - log.WithField("accountID", timeline.TimelineID()).Infof("pruned %d indexed and prepared items from timeline", amountPruned) - } - - return true - } - - // Execute the function for each timeline. - m.timelines.Range(f) - } - }() - - return nil -} - -func (m *manager) Stop() error { - return nil -} - -func (m *manager) IngestOne(ctx context.Context, timelineID string, item Timelineable) (bool, error) { - return m.getOrCreateTimeline(ctx, timelineID).IndexAndPrepareOne( - ctx, - item.GetID(), - item.GetBoostOfID(), - item.GetAccountID(), - item.GetBoostOfAccountID(), - ) -} - -func (m *manager) Remove(ctx context.Context, timelineID string, itemID string) (int, error) { - return m.getOrCreateTimeline(ctx, timelineID).Remove(ctx, itemID) -} - -func (m *manager) RemoveTimeline(ctx context.Context, timelineID string) error { - m.timelines.Delete(timelineID) - return nil -} - -func (m *manager) GetTimeline(ctx context.Context, timelineID string, maxID string, sinceID string, minID string, limit int, local bool) ([]Preparable, error) { - return m.getOrCreateTimeline(ctx, timelineID).Get(ctx, limit, maxID, sinceID, minID, true) -} - -func (m *manager) GetIndexedLength(ctx context.Context, timelineID string) int { - return m.getOrCreateTimeline(ctx, timelineID).Len() -} - -func (m *manager) GetOldestIndexedID(ctx context.Context, timelineID string) string { - return m.getOrCreateTimeline(ctx, timelineID).OldestIndexedItemID() -} - -func (m *manager) WipeItemFromAllTimelines(ctx context.Context, itemID string) error { - errs := new(gtserror.MultiError) - - m.timelines.Range(func(_ any, v any) bool { - if _, err := v.(Timeline).Remove(ctx, itemID); err != nil { - errs.Append(err) - } - - return true // always continue range - }) - - if err := errs.Combine(); err != nil { - return gtserror.Newf("error(s) wiping status %s: %w", itemID, errs.Combine()) - } - - return nil -} - -func (m *manager) WipeItemsFromAccountID(ctx context.Context, timelineID string, accountID string) error { - _, err := m.getOrCreateTimeline(ctx, timelineID).RemoveAllByOrBoosting(ctx, accountID) - return err -} - -func (m *manager) UnprepareItemFromAllTimelines(ctx context.Context, itemID string) error { - errs := new(gtserror.MultiError) - - // Work through all timelines held by this - // manager, and call Unprepare for each. - m.timelines.Range(func(_ any, v any) bool { - if err := v.(Timeline).Unprepare(ctx, itemID); err != nil { - errs.Append(err) - } - - return true // always continue range - }) - - if err := errs.Combine(); err != nil { - return gtserror.Newf("error(s) unpreparing status %s: %w", itemID, errs.Combine()) - } - - return nil -} - -func (m *manager) UnprepareItem(ctx context.Context, timelineID string, itemID string) error { - return m.getOrCreateTimeline(ctx, timelineID).Unprepare(ctx, itemID) -} - -func (m *manager) Prune(ctx context.Context, timelineID string, desiredPreparedItemsLength int, desiredIndexedItemsLength int) (int, error) { - return m.getOrCreateTimeline(ctx, timelineID).Prune(desiredPreparedItemsLength, desiredIndexedItemsLength), nil -} - -// getOrCreateTimeline returns a timeline with the given id, -// creating a new timeline with that id if necessary. -func (m *manager) getOrCreateTimeline(ctx context.Context, timelineID string) Timeline { - i, ok := m.timelines.Load(timelineID) - if ok { - // Timeline already existed in sync.Map. - return i.(Timeline) - } - - // Timeline did not yet exist in sync.Map. - // Create + store it. - timeline := NewTimeline(ctx, timelineID, m.grabFunction, m.filterFunction, m.prepareFunction, m.skipInsertFunction) - m.timelines.Store(timelineID, timeline) - - return timeline -} diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go deleted file mode 100644 index ec595ce42..000000000 --- a/internal/timeline/prepare.go +++ /dev/null @@ -1,146 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "container/list" - "context" - "errors" - - "codeberg.org/gruf/go-kv" - "github.com/superseriousbusiness/gotosocial/internal/db" - statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -func (t *timeline) prepareXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) error { - l := log. - WithContext(ctx). - WithFields(kv.Fields{ - {"amount", amount}, - {"behindID", behindID}, - {"beforeID", beforeID}, - {"frontToBack", frontToBack}, - }...) - l.Trace("entering prepareXBetweenIDs") - - if beforeID >= behindID { - // This is an impossible situation, we - // can't prepare anything between these. - return nil - } - - if err := t.indexXBetweenIDs(ctx, amount, behindID, beforeID, frontToBack); err != nil { - // An error here doesn't necessarily mean we - // can't prepare anything, so log + keep going. - l.Debugf("error calling prepareXBetweenIDs: %s", err) - } - - t.Lock() - defer t.Unlock() - - // Try to prepare everything between (and including) the two points. - var ( - toPrepare = make(map[*list.Element]*indexedItemsEntry) - foundToPrepare int - ) - - if frontToBack { - // Paging forwards / down. - for e := t.items.data.Front(); e != nil; e = e.Next() { - entry := e.Value.(*indexedItemsEntry) - - if entry.itemID > behindID { - l.Trace("item is too new, continuing") - continue - } - - if entry.itemID < beforeID { - // We've gone beyond the bounds of - // items we're interested in; stop. - l.Trace("reached older items, breaking") - break - } - - // Only prepare entry if it's not - // already prepared, save db calls. - if entry.prepared == nil { - toPrepare[e] = entry - } - - foundToPrepare++ - if foundToPrepare >= amount { - break - } - } - } else { - // Paging backwards / up. - for e := t.items.data.Back(); e != nil; e = e.Prev() { - entry := e.Value.(*indexedItemsEntry) - - if entry.itemID < beforeID { - l.Trace("item is too old, continuing") - continue - } - - if entry.itemID > behindID { - // We've gone beyond the bounds of - // items we're interested in; stop. - l.Trace("reached newer items, breaking") - break - } - - if entry.prepared == nil { - toPrepare[e] = entry - } - - // Only prepare entry if it's not - // already prepared, save db calls. - foundToPrepare++ - if foundToPrepare >= amount { - break - } - } - } - - for e, entry := range toPrepare { - prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID) - if err != nil { - if errors.Is(err, statusfilter.ErrHideStatus) { - // This item has been filtered out by the requesting user's filters. - // Remove it and skip past it. - t.items.data.Remove(e) - continue - } - if errors.Is(err, db.ErrNoEntries) { - // ErrNoEntries means something has been deleted, - // so we'll likely not be able to ever prepare this. - // This means we can remove it and skip past it. - l.Debugf("db.ErrNoEntries while trying to prepare %s; will remove from timeline", entry.itemID) - t.items.data.Remove(e) - continue - } - // We've got a proper db error. - return gtserror.Newf("db error while trying to prepare %s: %w", entry.itemID, err) - } - entry.prepared = prepared - } - - return nil -} diff --git a/internal/timeline/prune.go b/internal/timeline/prune.go deleted file mode 100644 index 5c7476956..000000000 --- a/internal/timeline/prune.go +++ /dev/null @@ -1,83 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "container/list" -) - -func (t *timeline) Prune(desiredPreparedItemsLength int, desiredIndexedItemsLength int) int { - t.Lock() - defer t.Unlock() - - l := t.items.data - if l == nil { - // Nothing to prune. - return 0 - } - - var ( - position int - totalPruned int - toRemove *[]*list.Element - ) - - // Only initialize toRemove if we know we're - // going to need it, otherwise skiperino. - if toRemoveLen := t.items.data.Len() - desiredIndexedItemsLength; toRemoveLen > 0 { - toRemove = func() *[]*list.Element { tr := make([]*list.Element, 0, toRemoveLen); return &tr }() - } - - // Work from the front of the list until we get - // to the point where we need to start pruning. - for e := l.Front(); e != nil; e = e.Next() { - position++ - - if position <= desiredPreparedItemsLength { - // We're still within our allotted - // prepped length, nothing to do yet. - continue - } - - // We need to *at least* unprepare this entry. - // If we're beyond our indexed length already, - // we can just remove the item completely. - if position > desiredIndexedItemsLength { - *toRemove = append(*toRemove, e) - totalPruned++ - continue - } - - entry := e.Value.(*indexedItemsEntry) - if entry.prepared == nil { - // It's already unprepared (mood). - continue - } - - entry.prepared = nil // <- eat this up please garbage collector nom nom nom - totalPruned++ - } - - if toRemove != nil { - for _, e := range *toRemove { - l.Remove(e) - } - } - - return totalPruned -} diff --git a/internal/timeline/prune_test.go b/internal/timeline/prune_test.go deleted file mode 100644 index 6ff67d505..000000000 --- a/internal/timeline/prune_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/suite" -) - -type PruneTestSuite struct { - TimelineStandardTestSuite -} - -func (suite *PruneTestSuite) TestPrune() { - var ( - ctx = context.Background() - testAccountID = suite.testAccounts["local_account_1"].ID - desiredPreparedItemsLength = 5 - desiredIndexedItemsLength = 5 - ) - - suite.fillTimeline(testAccountID) - - pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) - suite.NoError(err) - suite.Equal(25, pruned) - suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) -} - -func (suite *PruneTestSuite) TestPruneTwice() { - var ( - ctx = context.Background() - testAccountID = suite.testAccounts["local_account_1"].ID - desiredPreparedItemsLength = 5 - desiredIndexedItemsLength = 5 - ) - - suite.fillTimeline(testAccountID) - - pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) - suite.NoError(err) - suite.Equal(25, pruned) - suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) - - // Prune same again, nothing should be pruned this time. - pruned, err = suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) - suite.NoError(err) - suite.Equal(0, pruned) - suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) -} - -func (suite *PruneTestSuite) TestPruneTo0() { - var ( - ctx = context.Background() - testAccountID = suite.testAccounts["local_account_1"].ID - desiredPreparedItemsLength = 0 - desiredIndexedItemsLength = 0 - ) - - suite.fillTimeline(testAccountID) - - pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) - suite.NoError(err) - suite.Equal(30, pruned) - suite.Equal(0, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) -} - -func (suite *PruneTestSuite) TestPruneToInfinityAndBeyond() { - var ( - ctx = context.Background() - testAccountID = suite.testAccounts["local_account_1"].ID - desiredPreparedItemsLength = 9999999 - desiredIndexedItemsLength = 9999999 - ) - - suite.fillTimeline(testAccountID) - - pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength) - suite.NoError(err) - suite.Equal(0, pruned) - suite.Equal(30, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID)) -} - -func TestPruneTestSuite(t *testing.T) { - suite.Run(t, new(PruneTestSuite)) -} diff --git a/internal/timeline/remove.go b/internal/timeline/remove.go deleted file mode 100644 index 86352b9fa..000000000 --- a/internal/timeline/remove.go +++ /dev/null @@ -1,97 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "container/list" - "context" - - "codeberg.org/gruf/go-kv" - "github.com/superseriousbusiness/gotosocial/internal/log" -) - -func (t *timeline) Remove(ctx context.Context, statusID string) (int, error) { - l := log.WithContext(ctx). - WithFields(kv.Fields{ - {"accountTimeline", t.timelineID}, - {"statusID", statusID}, - }...) - - t.Lock() - defer t.Unlock() - - if t.items == nil || t.items.data == nil { - // Nothing to do. - return 0, nil - } - - var toRemove []*list.Element - for e := t.items.data.Front(); e != nil; e = e.Next() { - entry := e.Value.(*indexedItemsEntry) - - if entry.itemID != statusID { - // Not relevant. - continue - } - - l.Debug("removing item") - toRemove = append(toRemove, e) - } - - for _, e := range toRemove { - t.items.data.Remove(e) - } - - return len(toRemove), nil -} - -func (t *timeline) RemoveAllByOrBoosting(ctx context.Context, accountID string) (int, error) { - l := log. - WithContext(ctx). - WithFields(kv.Fields{ - {"accountTimeline", t.timelineID}, - {"accountID", accountID}, - }...) - - t.Lock() - defer t.Unlock() - - if t.items == nil || t.items.data == nil { - // Nothing to do. - return 0, nil - } - - var toRemove []*list.Element - for e := t.items.data.Front(); e != nil; e = e.Next() { - entry := e.Value.(*indexedItemsEntry) - - if entry.accountID != accountID && entry.boostOfAccountID != accountID { - // Not relevant. - continue - } - - l.Debug("removing item") - toRemove = append(toRemove, e) - } - - for _, e := range toRemove { - t.items.data.Remove(e) - } - - return len(toRemove), nil -} diff --git a/internal/timeline/timeline.go b/internal/timeline/timeline.go deleted file mode 100644 index e7c609638..000000000 --- a/internal/timeline/timeline.go +++ /dev/null @@ -1,172 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "context" - "sync" - "time" -) - -// GrabFunction is used by a Timeline to grab more items to index. -// -// It should be provided to NewTimeline when the caller is creating a timeline -// (of statuses, notifications, etc). -// -// - timelineID: ID of the timeline. -// - maxID: the maximum item ID desired. -// - sinceID: the minimum item ID desired. -// - minID: see sinceID -// - limit: the maximum amount of items to be returned -// -// If an error is returned, the timeline will stop processing whatever request called GrabFunction, -// and return the error. If no error is returned, but stop = true, this indicates to the caller of GrabFunction -// that there are no more items to return, and processing should continue with the items already grabbed. -type GrabFunction func(ctx context.Context, timelineID string, maxID string, sinceID string, minID string, limit int) (items []Timelineable, stop bool, err error) - -// FilterFunction is used by a Timeline to filter whether or not a grabbed item should be indexed. -type FilterFunction func(ctx context.Context, timelineID string, item Timelineable) (shouldIndex bool, err error) - -// PrepareFunction converts a Timelineable into a Preparable. -// -// For example, this might result in the converstion of a *gtsmodel.Status with the given itemID into a serializable *apimodel.Status. -type PrepareFunction func(ctx context.Context, timelineID string, itemID string) (Preparable, error) - -// SkipInsertFunction indicates whether a new item about to be inserted in the prepared list should be skipped, -// based on the item itself, the next item in the timeline, and the depth at which nextItem has been found in the list. -// -// This will be called for every item found while iterating through a timeline, so callers should be very careful -// not to do anything expensive here. -type SkipInsertFunction func(ctx context.Context, - newItemID string, - newItemAccountID string, - newItemBoostOfID string, - newItemBoostOfAccountID string, - nextItemID string, - nextItemAccountID string, - nextItemBoostOfID string, - nextItemBoostOfAccountID string, - depth int) (bool, error) - -// Timeline represents a timeline for one account, and contains indexed and prepared items. -type Timeline interface { - /* - RETRIEVAL FUNCTIONS - */ - - // Get returns an amount of prepared items with the given parameters. - // If prepareNext is true, then the next predicted query will be prepared already in a goroutine, - // to make the next call to Get faster. - Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error) - - /* - INDEXING + PREPARATION FUNCTIONS - */ - - // IndexAndPrepareOne puts a item into the timeline at the appropriate place - // according to its id, and then immediately prepares it. - // - // The returned bool indicates whether or not the item was actually inserted - // into the timeline. This will be false if the item is a boost and the original - // item, or a boost of it, already exists recently in the timeline. - IndexAndPrepareOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) - - // Unprepare clears the prepared version of the given item (and any boosts - // thereof) from the timeline, but leaves the indexed version in place. - // - // This is useful for cache invalidation when the prepared version of the - // item has changed for some reason (edits, updates, etc), but the item does - // not need to be removed: it will be prepared again next time Get is called. - Unprepare(ctx context.Context, itemID string) error - - /* - INFO FUNCTIONS - */ - - // TimelineID returns the id of this timeline. - TimelineID() string - - // Len returns the length of the item index at this point in time. - Len() int - - // OldestIndexedItemID returns the id of the rearmost (ie., the oldest) indexed item. - // If there's no oldest item, an empty string will be returned so make sure to check for this. - OldestIndexedItemID() string - - /* - UTILITY FUNCTIONS - */ - - // LastGot returns the time that Get was last called. - LastGot() time.Time - - // Prune prunes prepared and indexed items in this timeline to the desired lengths. - // This will be a no-op if the lengths are already < the desired values. - // - // The returned int indicates the amount of entries that were removed or unprepared. - Prune(desiredPreparedItemsLength int, desiredIndexedItemsLength int) int - - // Remove removes an item with the given ID. - // - // If a item has multiple entries in a timeline, they will all be removed. - // - // The returned int indicates the amount of entries that were removed. - Remove(ctx context.Context, itemID string) (int, error) - - // RemoveAllByOrBoosting removes all items created by or boosting the given accountID. - // - // The returned int indicates the amount of entries that were removed. - RemoveAllByOrBoosting(ctx context.Context, accountID string) (int, error) -} - -// timeline fulfils the Timeline interface -type timeline struct { - items *indexedItems - grabFunction GrabFunction - filterFunction FilterFunction - prepareFunction PrepareFunction - timelineID string - lastGot time.Time - sync.Mutex -} - -func (t *timeline) TimelineID() string { - return t.timelineID -} - -// NewTimeline returns a new Timeline with -// the given ID, using the given functions. -func NewTimeline( - ctx context.Context, - timelineID string, - grabFunction GrabFunction, - filterFunction FilterFunction, - prepareFunction PrepareFunction, - skipInsertFunction SkipInsertFunction, -) Timeline { - return &timeline{ - items: &indexedItems{ - skipInsert: skipInsertFunction, - }, - grabFunction: grabFunction, - filterFunction: filterFunction, - prepareFunction: prepareFunction, - timelineID: timelineID, - lastGot: time.Time{}, - } -} diff --git a/internal/timeline/timeline_test.go b/internal/timeline/timeline_test.go deleted file mode 100644 index ffc6d6e53..000000000 --- a/internal/timeline/timeline_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline_test - -import ( - "context" - "sort" - - "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/state" - "github.com/superseriousbusiness/gotosocial/internal/typeutils" - "github.com/superseriousbusiness/gotosocial/testrig" -) - -type TimelineStandardTestSuite struct { - suite.Suite - state *state.State - - testAccounts map[string]*gtsmodel.Account - testStatuses map[string]*gtsmodel.Status - highestStatusID string - lowestStatusID string -} - -func (suite *TimelineStandardTestSuite) SetupSuite() { - suite.testAccounts = testrig.NewTestAccounts() - suite.testStatuses = testrig.NewTestStatuses() -} - -func (suite *TimelineStandardTestSuite) SetupTest() { - suite.state = new(state.State) - - suite.state.Caches.Init() - testrig.StartNoopWorkers(suite.state) - - testrig.InitTestConfig() - testrig.InitTestLog() - - suite.state.DB = testrig.NewTestDB(suite.state) - - testrig.StartTimelines( - suite.state, - visibility.NewFilter(suite.state), - typeutils.NewConverter(suite.state), - ) - - testrig.StandardDBSetup(suite.state.DB, nil) -} - -func (suite *TimelineStandardTestSuite) TearDownTest() { - testrig.StandardDBTeardown(suite.state.DB) - testrig.StopWorkers(suite.state) -} - -func (suite *TimelineStandardTestSuite) fillTimeline(timelineID string) { - // Put testrig statuses in a determinate order - // since we can't trust a map to keep order. - statuses := []*gtsmodel.Status{} - for _, s := range suite.testStatuses { - statuses = append(statuses, s) - } - - sort.Slice(statuses, func(i, j int) bool { - return statuses[i].ID > statuses[j].ID - }) - - // Statuses are now highest -> lowest. - suite.highestStatusID = statuses[0].ID - suite.lowestStatusID = statuses[len(statuses)-1].ID - if suite.highestStatusID < suite.lowestStatusID { - suite.FailNow("", "statuses weren't ordered properly by sort") - } - - // Put all test statuses into the timeline; we don't - // need to be fussy about who sees what for these tests. - for _, status := range statuses { - if _, err := suite.state.Timelines.Home.IngestOne(context.Background(), timelineID, status); err != nil { - suite.FailNow(err.Error()) - } - } -} diff --git a/internal/timeline/timelines.go b/internal/timeline/timelines.go deleted file mode 100644 index 8291fef5e..000000000 --- a/internal/timeline/timelines.go +++ /dev/null @@ -1,37 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -type Timelines struct { - // Home provides access to account home timelines. - Home Manager - - // List provides access to list timelines. - List Manager - - // prevent pass-by-value. - _ nocopy -} - -// nocopy when embedded will signal linter to -// error on pass-by-value of parent struct. -type nocopy struct{} - -func (*nocopy) Lock() {} - -func (*nocopy) Unlock() {} diff --git a/internal/timeline/types.go b/internal/timeline/types.go deleted file mode 100644 index 6243799f5..000000000 --- a/internal/timeline/types.go +++ /dev/null @@ -1,34 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -// Timelineable represents any item that can be indexed in a timeline. -type Timelineable interface { - GetID() string - GetAccountID() string - GetBoostOfID() string - GetBoostOfAccountID() string -} - -// Preparable represents any item that can be prepared in a timeline. -type Preparable interface { - GetID() string - GetAccountID() string - GetBoostOfID() string - GetBoostOfAccountID() string -} diff --git a/internal/timeline/unprepare.go b/internal/timeline/unprepare.go deleted file mode 100644 index 67a990287..000000000 --- a/internal/timeline/unprepare.go +++ /dev/null @@ -1,50 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline - -import ( - "context" -) - -func (t *timeline) Unprepare(ctx context.Context, itemID string) error { - t.Lock() - defer t.Unlock() - - if t.items == nil || t.items.data == nil { - // Nothing to do. - return nil - } - - for e := t.items.data.Front(); e != nil; e = e.Next() { - entry := e.Value.(*indexedItemsEntry) - - if entry.itemID != itemID && entry.boostOfID != itemID { - // Not relevant. - continue - } - - if entry.prepared == nil { - // It's already unprepared (mood). - continue - } - - entry.prepared = nil // <- eat this up please garbage collector nom nom nom - } - - return nil -} diff --git a/internal/timeline/unprepare_test.go b/internal/timeline/unprepare_test.go deleted file mode 100644 index 20bef7537..000000000 --- a/internal/timeline/unprepare_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package timeline_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/suite" - - apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/id" -) - -type UnprepareTestSuite struct { - TimelineStandardTestSuite -} - -func (suite *UnprepareTestSuite) TestUnprepareFromFave() { - var ( - ctx = context.Background() - testAccount = suite.testAccounts["local_account_1"] - maxID = "" - sinceID = "" - minID = "" - limit = 1 - local = false - ) - - suite.fillTimeline(testAccount.ID) - - // Get first status from the top (no params). - statuses, err := suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - if len(statuses) != 1 { - suite.FailNow("couldn't get top status") - } - - targetStatus := statuses[0].(*apimodel.Status) - - // Check fave stats of the top status. - suite.Equal(0, targetStatus.FavouritesCount) - suite.False(targetStatus.Favourited) - - // Fave the top status from testAccount. - if err := suite.state.DB.PutStatusFave(ctx, >smodel.StatusFave{ - ID: id.NewULID(), - AccountID: testAccount.ID, - TargetAccountID: targetStatus.Account.ID, - StatusID: targetStatus.ID, - URI: "https://example.org/some/activity/path", - }); err != nil { - suite.FailNow(err.Error()) - } - - // Repeat call to get first status from the top. - // Get first status from the top (no params). - statuses, err = suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - if len(statuses) != 1 { - suite.FailNow("couldn't get top status") - } - - targetStatus = statuses[0].(*apimodel.Status) - - // We haven't yet uncached/unprepared the status, - // we've only inserted the fave, so counts should - // stay the same... - suite.Equal(0, targetStatus.FavouritesCount) - suite.False(targetStatus.Favourited) - - // Now call unprepare. - suite.state.Timelines.Home.UnprepareItemFromAllTimelines(ctx, targetStatus.ID) - - // Now a Get should trigger a fresh prepare of the - // target status, and the counts should be updated. - // Repeat call to get first status from the top. - // Get first status from the top (no params). - statuses, err = suite.state.Timelines.Home.GetTimeline( - ctx, - testAccount.ID, - maxID, - sinceID, - minID, - limit, - local, - ) - if err != nil { - suite.FailNow(err.Error()) - } - - if len(statuses) != 1 { - suite.FailNow("couldn't get top status") - } - - targetStatus = statuses[0].(*apimodel.Status) - - suite.Equal(1, targetStatus.FavouritesCount) - suite.True(targetStatus.Favourited) -} - -func TestUnprepareTestSuite(t *testing.T) { - suite.Run(t, new(UnprepareTestSuite)) -}