mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-30 02:52:26 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove old timeline package, add local timeline cache

Browse source
This commit is contained in:
kim 2025-03-20 13:34:34 +00:00
commit 566e2b1d38
21 changed files with 105 additions and 2943 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/cache/cache.go vendored
View file

@ -98,6 +98,7 @@ func (c *Caches) Init() {
c.initListIDs() c.initListIDs()
c.initListedIDs() c.initListedIDs()
c.initListTimelines() c.initListTimelines()
c.initLocalTimeline()
c.initMarker() c.initMarker()
c.initMedia() c.initMedia()
c.initMention() c.initMention()
@ -216,6 +217,7 @@ func (c *Caches) Sweep(threshold float64) {
c.Timelines.Home.Trim(threshold) c.Timelines.Home.Trim(threshold)
c.Timelines.List.Trim(threshold) c.Timelines.List.Trim(threshold)
c.Timelines.Public.Trim(threshold) c.Timelines.Public.Trim(threshold)
c.Timelines.Local.Trim(threshold)
c.Visibility.Trim(threshold) c.Visibility.Trim(threshold)
} }

11
internal/cache/timeline.go vendored
View file

@ -32,6 +32,9 @@ type TimelineCaches struct {
// Public ... // Public ...
Public timeline.StatusTimeline Public timeline.StatusTimeline
// Local ...
Local timeline.StatusTimeline
} }
func (c *Caches) initHomeTimelines() { func (c *Caches) initHomeTimelines() {
@ -57,3 +60,11 @@ func (c *Caches) initPublicTimeline() {
c.Timelines.Public.Init(cap) c.Timelines.Public.Init(cap)
} }
func (c *Caches) initLocalTimeline() {
cap := 1000
log.Infof(nil, "cache size = %d", cap)
c.Timelines.Local.Init(cap)
}

105
internal/db/bundb/timeline.go
View file

@ -21,13 +21,11 @@ import (
"context" "context"
"errors" "errors"
"slices" "slices"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/paging" "github.com/superseriousbusiness/gotosocial/internal/paging"
"github.com/superseriousbusiness/gotosocial/internal/state" "github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/uptrace/bun" "github.com/uptrace/bun"
@ -150,88 +148,29 @@ func (t *timelineDB) GetPublicTimeline(ctx context.Context, page *paging.Page) (
) )
} }
func (t *timelineDB) getLocalTimeline( func (t *timelineDB) GetLocalTimeline(ctx context.Context, page *paging.Page) ([]*gtsmodel.Status, error) {
ctx context.Context, return loadStatusTimelinePage(ctx, t.db, t.state,
maxID string,
sinceID string, // Paging
minID string, // params.
limit int, page,
) ([]*gtsmodel.Status, error) {
// Make educated guess for slice size func(q *bun.SelectQuery) (*bun.SelectQuery, error) {
var ( // Local only.
statusIDs = make([]string, 0, limit) q = q.Where("? = ?", bun.Ident("status.local"), true)
frontToBack = true
// Public only.
q = q.Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityPublic)
// Ignore boosts.
q = q.Where("? IS NULL", bun.Ident("boost_of_id"))
// Only include statuses that aren't pending approval.
q = q.Where("? = ?", bun.Ident("pending_approval"), false)
return q, nil
},
) )
q := t.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
// Local only.
Where("? = ?", bun.Ident("status.local"), true).
// Public only.
Where("? = ?", bun.Ident("status.visibility"), gtsmodel.VisibilityPublic).
// Only include statuses that aren't pending approval.
Where("? = ?", bun.Ident("status.pending_approval"), false).
// Ignore boosts.
Where("? IS NULL", bun.Ident("status.boost_of_id")).
// Select only IDs from table
Column("status.id")
if maxID == "" || maxID >= id.Highest {
const future = 24 * time.Hour
// don't return statuses more than 24hr in the future
maxID = id.NewULIDFromTime(time.Now().Add(future))
}
// return only statuses LOWER (ie., older) than maxID
q = q.Where("? < ?", bun.Ident("status.id"), maxID)
if sinceID != "" {
// return only statuses HIGHER (ie., newer) than sinceID
q = q.Where("? > ?", bun.Ident("status.id"), sinceID)
}
if minID != "" {
// return only statuses HIGHER (ie., newer) than minID
q = q.Where("? > ?", bun.Ident("status.id"), minID)
// page up
frontToBack = false
}
if limit > 0 {
// limit amount of statuses returned
q = q.Limit(limit)
}
if frontToBack {
// Page down.
q = q.Order("status.id DESC")
} else {
// Page up.
q = q.Order("status.id ASC")
}
if err := q.Scan(ctx, &statusIDs); err != nil {
return nil, err
}
if len(statusIDs) == 0 {
return nil, nil
}
// If we're paging up, we still want statuses
// to be sorted by ID desc, so reverse ids slice.
// https://zchee.github.io/golang-wiki/SliceTricks/#reversing
if !frontToBack {
for l, r := 0, len(statusIDs)-1; l < r; l, r = l+1, r-1 {
statusIDs[l], statusIDs[r] = statusIDs[r], statusIDs[l]
}
}
// Return status IDs loaded from cache + db.
return t.state.DB.GetStatusesByIDs(ctx, statusIDs)
} }
// TODO optimize this query and the logic here, because it's slow as balls -- it takes like a literal second to return with a limit of 20! // TODO optimize this query and the logic here, because it's slow as balls -- it takes like a literal second to return with a limit of 20!

3
internal/db/timeline.go
View file

@ -37,6 +37,9 @@ type Timeline interface {
// Statuses should be returned in descending order of when they were created (newest first). // Statuses should be returned in descending order of when they were created (newest first).
GetPublicTimeline(ctx context.Context, page *paging.Page) ([]*gtsmodel.Status, error) GetPublicTimeline(ctx context.Context, page *paging.Page) ([]*gtsmodel.Status, error)
// GetLocalTimeline ...
GetLocalTimeline(ctx context.Context, page *paging.Page) ([]*gtsmodel.Status, error)
// GetFavedTimeline fetches the account's FAVED timeline -- ie., posts and replies that the requesting account has faved. // GetFavedTimeline fetches the account's FAVED timeline -- ie., posts and replies that the requesting account has faved.
// It will use the given filters and try to return as many statuses as possible up to the limit. // It will use the given filters and try to return as many statuses as possible up to the limit.
// //

79
internal/processing/timeline/public.go
View file

@ -19,7 +19,6 @@ package timeline
import ( import (
"context" "context"
"net/url"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status" statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
@ -37,6 +36,20 @@ func (p *Processor) PublicTimelineGet(
) ( ) (
*apimodel.PageableResponse, *apimodel.PageableResponse,
gtserror.WithCode, gtserror.WithCode,
) {
if local {
return p.localTimelineGet(ctx, requester, page)
}
return p.publicTimelineGet(ctx, requester, page)
}
func (p *Processor) publicTimelineGet(
ctx context.Context,
requester *gtsmodel.Account,
page *paging.Page,
) (
*apimodel.PageableResponse,
gtserror.WithCode,
) { ) {
return p.getStatusTimeline(ctx, return p.getStatusTimeline(ctx,
@ -58,12 +71,7 @@ func (p *Processor) PublicTimelineGet(
// page query flag, (this map // page query flag, (this map
// later gets copied before // later gets copied before
// any further usage). // any further usage).
func() url.Values { localOnlyFalse,
if local {
return localOnlyTrue
}
return localOnlyFalse
}(),
// Status filter context. // Status filter context.
statusfilter.FilterContextPublic, statusfilter.FilterContextPublic,
@ -81,11 +89,58 @@ func (p *Processor) PublicTimelineGet(
// i.e. filter after caching. // i.e. filter after caching.
func(s *gtsmodel.Status) (bool, error) { func(s *gtsmodel.Status) (bool, error) {
// Remove any non-local statuses // Check the visibility of passed status to requesting user.
// if requester wants local-only. ok, err := p.visFilter.StatusPublicTimelineable(ctx, requester, s)
if local && !*s.Local { return !ok, err
return true, nil },
} )
}
func (p *Processor) localTimelineGet(
ctx context.Context,
requester *gtsmodel.Account,
page *paging.Page,
) (
*apimodel.PageableResponse,
gtserror.WithCode,
) {
return p.getStatusTimeline(ctx,
// Auth'd
// account.
requester,
// Global local timeline cache.
&p.state.Caches.Timelines.Local,
// Current
// page.
page,
// Public timeline endpoint.
"/api/v1/timelines/public",
// Set local-only timeline
// page query flag, (this map
// later gets copied before
// any further usage).
localOnlyTrue,
// Status filter context.
statusfilter.FilterContextPublic,
// Database load function.
func(pg *paging.Page) (statuses []*gtsmodel.Status, err error) {
return p.state.DB.GetLocalTimeline(ctx, pg)
},
// Pre-filtering function,
// i.e. filter before caching.
nil,
// Post-filtering function,
// i.e. filter after caching.
func(s *gtsmodel.Status) (bool, error) {
// Check the visibility of passed status to requesting user. // Check the visibility of passed status to requesting user.
ok, err := p.visFilter.StatusPublicTimelineable(ctx, requester, s) ok, err := p.visFilter.StatusPublicTimelineable(ctx, requester, s)

428
internal/timeline/get.go
View file

@ -1,428 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"container/list"
"context"
"errors"
"time"
"codeberg.org/gruf/go-kv"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
func (t *timeline) LastGot() time.Time {
t.Lock()
defer t.Unlock()
return t.lastGot
}
func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error) {
l := log.WithContext(ctx).
WithFields(kv.Fields{
{"accountID", t.timelineID},
{"amount", amount},
{"maxID", maxID},
{"sinceID", sinceID},
{"minID", minID},
}...)
l.Trace("entering get and updating t.lastGot")
// Regardless of what happens below, update the
// last time Get was called for this timeline.
t.Lock()
t.lastGot = time.Now()
t.Unlock()
var (
items []Preparable
err error
)
switch {
case maxID == "" && sinceID == "" && minID == "":
// No params are defined so just fetch from the top.
// This is equivalent to a user starting to view
// their timeline from newest -> older posts.
items, err = t.getXBetweenIDs(ctx, amount, id.Highest, id.Lowest, true)
// Cache expected next query to speed up scrolling.
// Assume the user will be scrolling downwards from
// the final ID in items.
if prepareNext && err == nil && len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
t.prepareNextQuery(amount, nextMaxID, "", "")
}
case maxID != "" && sinceID == "" && minID == "":
// Only maxID is defined, so fetch from maxID onwards.
// This is equivalent to a user paging further down
// their timeline from newer -> older posts.
items, err = t.getXBetweenIDs(ctx, amount, maxID, id.Lowest, true)
// Cache expected next query to speed up scrolling.
// Assume the user will be scrolling downwards from
// the final ID in items.
if prepareNext && err == nil && len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
t.prepareNextQuery(amount, nextMaxID, "", "")
}
// In the next cases, maxID is defined, and so are
// either sinceID or minID. This is equivalent to
// a user opening an in-progress timeline and asking
// for a slice of posts somewhere in the middle, or
// trying to "fill in the blanks" between two points,
// paging either up or down.
case maxID != "" && sinceID != "":
items, err = t.getXBetweenIDs(ctx, amount, maxID, sinceID, true)
// Cache expected next query to speed up scrolling.
// We can assume the caller is scrolling downwards.
// Guess id.Lowest as sinceID, since we don't actually
// know what the next sinceID would be.
if prepareNext && err == nil && len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
t.prepareNextQuery(amount, nextMaxID, id.Lowest, "")
}
case maxID != "" && minID != "":
items, err = t.getXBetweenIDs(ctx, amount, maxID, minID, false)
// Cache expected next query to speed up scrolling.
// We can assume the caller is scrolling upwards.
// Guess id.Highest as maxID, since we don't actually
// know what the next maxID would be.
if prepareNext && err == nil && len(items) != 0 {
prevMinID := items[0].GetID()
t.prepareNextQuery(amount, id.Highest, "", prevMinID)
}
// In the final cases, maxID is not defined, but
// either sinceID or minID are. This is equivalent to
// a user either "pulling up" at the top of their timeline
// to refresh it and check if newer posts have come in, or
// trying to scroll upwards from an old post to see what
// they missed since then.
//
// In these calls, we use the highest possible ulid as
// behindID because we don't have a cap for newest that
// we're interested in.
case maxID == "" && sinceID != "":
items, err = t.getXBetweenIDs(ctx, amount, id.Highest, sinceID, true)
// We can't cache an expected next query for this one,
// since presumably the caller is at the top of their
// timeline already.
case maxID == "" && minID != "":
items, err = t.getXBetweenIDs(ctx, amount, id.Highest, minID, false)
// Cache expected next query to speed up scrolling.
// We can assume the caller is scrolling upwards.
// Guess id.Highest as maxID, since we don't actually
// know what the next maxID would be.
if prepareNext && err == nil && len(items) != 0 {
prevMinID := items[0].GetID()
t.prepareNextQuery(amount, id.Highest, "", prevMinID)
}
default:
err = gtserror.New("switch statement exhausted with no results")
}
return items, err
}
// getXBetweenIDs returns x amount of items somewhere between (not including) the given IDs.
//
// If frontToBack is true, items will be served paging down from behindID.
// This corresponds to an api call to /timelines/home?max_id=WHATEVER&since_id=WHATEVER
//
// If frontToBack is false, items will be served paging up from beforeID.
// This corresponds to an api call to /timelines/home?max_id=WHATEVER&min_id=WHATEVER
func (t *timeline) getXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) ([]Preparable, error) {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"amount", amount},
{"behindID", behindID},
{"beforeID", beforeID},
{"frontToBack", frontToBack},
}...)
l.Trace("entering getXBetweenID")
// Assume length we need to return.
items := make([]Preparable, 0, amount)
if beforeID >= behindID {
// This is an impossible situation, we
// can't serve anything between these.
return items, nil
}
// Try to ensure we have enough items prepared.
if err := t.prepareXBetweenIDs(ctx, amount, behindID, beforeID, frontToBack); err != nil {
// An error here doesn't necessarily mean we
// can't serve anything, so log + keep going.
l.Debugf("error calling prepareXBetweenIDs: %s", err)
}
var (
beforeIDMark *list.Element
served int
// Our behavior while ranging through the
// list changes depending on if we're
// going front-to-back or back-to-front.
//
// To avoid checking which one we're doing
// in each loop iteration, define our range
// function here outside the loop.
//
// The bool indicates to the caller whether
// iteration should continue (true) or stop
// (false).
rangeF func(e *list.Element) (bool, error)
// If we get certain errors on entries as we're
// looking through, we might want to cheekily
// remove their elements from the timeline.
// Everything added to this slice will be removed.
removeElements = []*list.Element{}
)
defer func() {
for _, e := range removeElements {
t.items.data.Remove(e)
}
}()
if frontToBack {
// We're going front-to-back, which means we
// don't need to look for a mark per se, we
// just keep serving items until we've reached
// a point where the items are out of the range
// we're interested in.
rangeF = func(e *list.Element) (bool, error) {
entry := e.Value.(*indexedItemsEntry)
if entry.itemID >= behindID {
// ID of this item is too high,
// just keep iterating.
l.Trace("item is too new, continuing")
return true, nil
}
if entry.itemID <= beforeID {
// We've gone as far as we can through
// the list and reached entries that are
// now too old for us, stop here.
l.Trace("reached older items, breaking")
return false, nil
}
l.Trace("entry is just right")
if entry.prepared == nil {
// Whoops, this entry isn't prepared yet; some
// race condition? That's OK, we can do it now.
prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID)
if err != nil {
if errors.Is(err, statusfilter.ErrHideStatus) {
// This item has been filtered out by the requesting user's filters.
// Remove it and skip past it.
removeElements = append(removeElements, e)
return true, nil
}
if errors.Is(err, db.ErrNoEntries) {
// ErrNoEntries means something has been deleted,
// so we'll likely not be able to ever prepare this.
// This means we can remove it and skip past it.
l.Debugf("db.ErrNoEntries while trying to prepare %s; will remove from timeline", entry.itemID)
removeElements = append(removeElements, e)
return true, nil
}
// We've got a proper db error.
err = gtserror.Newf("db error while trying to prepare %s: %w", entry.itemID, err)
return false, err
}
entry.prepared = prepared
}
items = append(items, entry.prepared)
served++
return served < amount, nil
}
} else {
// Iterate through the list from the top, until
// we reach an item with id smaller than beforeID;
// ie., an item OLDER than beforeID. At that point,
// we can stop looking because we're not interested
// in older entries.
rangeF = func(e *list.Element) (bool, error) {
// Move the mark back one place each loop.
beforeIDMark = e
if entry := e.Value.(*indexedItemsEntry); entry.itemID <= beforeID {
// We've gone as far as we can through
// the list and reached entries that are
// now too old for us, stop here.
l.Trace("reached older items, breaking")
return false, nil
}
return true, nil
}
}
// Iterate through the list until the function
// we defined above instructs us to stop.
for e := t.items.data.Front(); e != nil; e = e.Next() {
keepGoing, err := rangeF(e)
if err != nil {
return nil, err
}
if !keepGoing {
break
}
}
if frontToBack || beforeIDMark == nil {
// If we're serving front to back, then
// items should be populated by now. If
// we're serving back to front but didn't
// find any items newer than beforeID,
// we can just return empty items.
return items, nil
}
// We're serving back to front, so iterate upwards
// towards the front of the list from the mark we found,
// until we either get to the front, serve enough
// items, or reach behindID.
//
// To preserve ordering, we need to reverse the slice
// when we're finished.
for e := beforeIDMark; e != nil; e = e.Prev() {
entry := e.Value.(*indexedItemsEntry)
if entry.itemID == beforeID {
// Don't include the beforeID
// entry itself, just continue.
l.Trace("entry item ID is equal to beforeID, skipping")
continue
}
if entry.itemID >= behindID {
// We've reached items that are
// newer than what we're looking
// for, just stop here.
l.Trace("reached newer items, breaking")
break
}
if entry.prepared == nil {
// Whoops, this entry isn't prepared yet; some
// race condition? That's OK, we can do it now.
prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID)
if err != nil {
if errors.Is(err, statusfilter.ErrHideStatus) {
// This item has been filtered out by the requesting user's filters.
// Remove it and skip past it.
removeElements = append(removeElements, e)
continue
}
if errors.Is(err, db.ErrNoEntries) {
// ErrNoEntries means something has been deleted,
// so we'll likely not be able to ever prepare this.
// This means we can remove it and skip past it.
l.Debugf("db.ErrNoEntries while trying to prepare %s; will remove from timeline", entry.itemID)
removeElements = append(removeElements, e)
continue
}
// We've got a proper db error.
err = gtserror.Newf("db error while trying to prepare %s: %w", entry.itemID, err)
return nil, err
}
entry.prepared = prepared
}
items = append(items, entry.prepared)
served++
if served >= amount {
break
}
}
// Reverse order of items.
// https://zchee.github.io/golang-wiki/SliceTricks/#reversing
for l, r := 0, len(items)-1; l < r; l, r = l+1, r-1 {
items[l], items[r] = items[r], items[l]
}
return items, nil
}
func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, minID string) {
var (
// We explicitly use context.Background() rather than
// accepting a context param because we don't want this
// to stop/break when the calling context finishes.
ctx = context.Background()
err error
)
// Always perform this async so caller doesn't have to wait.
go func() {
switch {
case maxID == "" && sinceID == "" && minID == "":
err = t.prepareXBetweenIDs(ctx, amount, id.Highest, id.Lowest, true)
case maxID != "" && sinceID == "" && minID == "":
err = t.prepareXBetweenIDs(ctx, amount, maxID, id.Lowest, true)
case maxID != "" && sinceID != "":
err = t.prepareXBetweenIDs(ctx, amount, maxID, sinceID, true)
case maxID != "" && minID != "":
err = t.prepareXBetweenIDs(ctx, amount, maxID, minID, false)
case maxID == "" && sinceID != "":
err = t.prepareXBetweenIDs(ctx, amount, id.Highest, sinceID, true)
case maxID == "" && minID != "":
err = t.prepareXBetweenIDs(ctx, amount, id.Highest, minID, false)
default:
err = gtserror.New("switch statement exhausted with no results")
}
if err != nil {
log.
WithContext(ctx).
WithFields(kv.Fields{
{"amount", amount},
{"maxID", maxID},
{"sinceID", sinceID},
{"minID", minID},
}...).
Warnf("error preparing next query: %s", err)
}
}()
}

704
internal/timeline/get_test.go
View file

@ -1,704 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline_test
import (
"context"
"sync"
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
)
type GetTestSuite struct {
TimelineStandardTestSuite
}
func (suite *GetTestSuite) checkStatuses(statuses []timeline.Preparable, maxID string, minID string, expectedLength int) {
if l := len(statuses); l != expectedLength {
suite.FailNow("", "expected %d statuses in slice, got %d", expectedLength, l)
} else if l == 0 {
// Can't test empty slice.
return
}
// Check ordering + bounds of statuses.
highest := statuses[0].GetID()
for _, status := range statuses {
id := status.GetID()
if id >= maxID {
suite.FailNow("", "%s greater than maxID %s", id, maxID)
}
if id <= minID {
suite.FailNow("", "%s smaller than minID %s", id, minID)
}
if id > highest {
suite.FailNow("", "statuses in slice were not ordered highest -> lowest ID")
}
highest = id
}
}
func (suite *GetTestSuite) emptyAccountFollows(ctx context.Context, accountID string) {
// Get all of account's follows.
follows, err := suite.state.DB.GetAccountFollows(
gtscontext.SetBarebones(ctx),
accountID,
nil, // select all
)
if err != nil {
suite.FailNow(err.Error())
}
// Remove each follow.
for _, follow := range follows {
if err := suite.state.DB.DeleteFollowByID(ctx, follow.ID); err != nil {
suite.FailNow(err.Error())
}
}
// Ensure no follows left.
follows, err = suite.state.DB.GetAccountFollows(
gtscontext.SetBarebones(ctx),
accountID,
nil, // select all
)
if err != nil {
suite.FailNow(err.Error())
}
if len(follows) != 0 {
suite.FailNow("follows should be empty")
}
}
func (suite *GetTestSuite) emptyAccountStatuses(ctx context.Context, accountID string) {
// Get all of account's statuses.
statuses, err := suite.state.DB.GetAccountStatuses(
ctx,
accountID,
9999,
false,
false,
id.Highest,
id.Lowest,
false,
false,
)
if err != nil {
suite.FailNow(err.Error())
}
// Remove each status.
for _, status := range statuses {
if err := suite.state.DB.DeleteStatusByID(ctx, status.ID); err != nil {
suite.FailNow(err.Error())
}
}
}
func (suite *GetTestSuite) TestGetNewTimelinePageDown() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = ""
limit = 5
local = false
)
// Get 5 from the top.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, id.Lowest, 5)
// Get 5 from next maxID.
maxID = statuses[len(statuses)-1].GetID()
statuses, err = suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, maxID, id.Lowest, 5)
}
func (suite *GetTestSuite) TestGetNewTimelinePageUp() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = id.Lowest
limit = 5
local = false
)
// Get 5 from the back.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, minID, 5)
// Page up from next minID.
minID = statuses[0].GetID()
statuses, err = suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, minID, 5)
}
func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossible() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = ""
limit = 100
local = false
)
// Get 100 from the top.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, id.Lowest, 22)
}
func (suite *GetTestSuite) TestGetNewTimelineMoreThanPossiblePageUp() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = id.Lowest
limit = 100
local = false
)
// Get 100 from the back.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, id.Lowest, 22)
}
func (suite *GetTestSuite) TestGetNewTimelineNoFollowing() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = ""
limit = 10
local = false
)
suite.emptyAccountFollows(ctx, testAccount.ID)
// Try to get 10 from the top of the timeline.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, id.Lowest, 9)
for _, s := range statuses {
if s.GetAccountID() != testAccount.ID {
suite.FailNow("timeline with no follows should only contain posts by timeline owner account")
}
}
}
func (suite *GetTestSuite) TestGetNewTimelineNoFollowingNoStatuses() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = ""
limit = 5
local = false
)
suite.emptyAccountFollows(ctx, testAccount.ID)
suite.emptyAccountStatuses(ctx, testAccount.ID)
// Try to get 5 from the top of the timeline.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, id.Lowest, 0)
}
func (suite *GetTestSuite) TestGetNoParams() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = ""
limit = 10
local = false
)
suite.fillTimeline(testAccount.ID)
// Get 10 statuses from the top (no params).
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, id.Lowest, 10)
// First status should have the highest ID in the testrig.
suite.Equal(suite.highestStatusID, statuses[0].GetID())
}
func (suite *GetTestSuite) TestGetMaxID() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = "01F8MHBQCBTDKN6X5VHGMMN4MA"
sinceID = ""
minID = ""
limit = 10
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 10 with a max ID somewhere in the middle of the stack.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
// We'll only get 6 statuses back.
suite.checkStatuses(statuses, maxID, id.Lowest, 6)
}
func (suite *GetTestSuite) TestGetSinceID() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = "01F8MHBQCBTDKN6X5VHGMMN4MA"
minID = ""
limit = 10
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 10 with a since ID somewhere in the middle of the stack.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, sinceID, 10)
// The first status in the stack should have the highest ID of all
// in the testrig, because we're paging down.
suite.Equal(suite.highestStatusID, statuses[0].GetID())
}
func (suite *GetTestSuite) TestGetSinceIDOneOnly() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = "01F8MHBQCBTDKN6X5VHGMMN4MA"
minID = ""
limit = 1
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 1 with a since ID somewhere in the middle of the stack.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, sinceID, 1)
// The one status we got back should have the highest ID of all in
// the testrig, because using sinceID means we're paging down.
suite.Equal(suite.highestStatusID, statuses[0].GetID())
}
func (suite *GetTestSuite) TestGetMinID() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = "01F8MHBQCBTDKN6X5VHGMMN4MA"
limit = 5
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 5 with a min ID somewhere in the middle of the stack.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, minID, 5)
// We're paging up so even the highest status ID in the pile
// shouldn't be the highest ID we have.
suite.NotEqual(suite.highestStatusID, statuses[0])
}
func (suite *GetTestSuite) TestGetMinIDOneOnly() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = "01F8MHBQCBTDKN6X5VHGMMN4MA"
limit = 1
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 1 with a min ID somewhere in the middle of the stack.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, minID, 1)
// The one status we got back should have the an ID equal to the
// one ID immediately newer than it.
suite.Equal("01F8MHC0H0A7XHTVH5F596ZKBM", statuses[0].GetID())
}
func (suite *GetTestSuite) TestGetMinIDFromLowestInTestrig() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = suite.lowestStatusID
limit = 1
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 1 with minID equal to the lowest status in the testrig.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, minID, 1)
// The one status we got back should have an id higher than
// the lowest status in the testrig, since minID is not inclusive.
suite.Greater(statuses[0].GetID(), suite.lowestStatusID)
}
func (suite *GetTestSuite) TestGetMinIDFromLowestPossible() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = id.Lowest
limit = 1
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 1 with the lowest possible min ID.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(statuses, id.Highest, minID, 1)
// The one status we got back should have the an ID equal to the
// lowest ID status in the test rig.
suite.Equal(suite.lowestStatusID, statuses[0].GetID())
}
func (suite *GetTestSuite) TestGetBetweenID() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = "01F8MHCP5P2NWYQ416SBA0XSEV"
sinceID = ""
minID = "01F8MHBQCBTDKN6X5VHGMMN4MA"
limit = 10
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 10 between these two IDs
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
// There's only two statuses between these two IDs.
suite.checkStatuses(statuses, maxID, minID, 2)
}
func (suite *GetTestSuite) TestGetBetweenIDImpossible() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = id.Lowest
sinceID = ""
minID = id.Highest
limit = 10
local = false
)
suite.fillTimeline(testAccount.ID)
// Ask for 10 between these two IDs which present
// an impossible query.
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
// We should have nothing back.
suite.checkStatuses(statuses, maxID, minID, 0)
}
func (suite *GetTestSuite) TestGetTimelinesAsync() {
var (
ctx = context.Background()
accountToNuke = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = ""
limit = 5
local = false
multiplier = 5
)
// Nuke one account's statuses and follows,
// as though the account had just been created.
suite.emptyAccountFollows(ctx, accountToNuke.ID)
suite.emptyAccountStatuses(ctx, accountToNuke.ID)
// Get 5 statuses from each timeline in
// our testrig at the same time, five times.
wg := new(sync.WaitGroup)
wg.Add(len(suite.testAccounts) * multiplier)
for i := 0; i < multiplier; i++ {
go func() {
for _, testAccount := range suite.testAccounts {
if _, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
); err != nil {
suite.Fail(err.Error())
}
wg.Done()
}
}()
}
wg.Wait() // Wait until all get calls have returned.
}
func TestGetTestSuite(t *testing.T) {
suite.Run(t, new(GetTestSuite))
}

283
internal/timeline/index.go
View file

@ -1,283 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"container/list"
"context"
"errors"
"codeberg.org/gruf/go-kv"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
func (t *timeline) indexXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) error {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"amount", amount},
{"behindID", behindID},
{"beforeID", beforeID},
{"frontToBack", frontToBack},
}...)
l.Trace("entering indexXBetweenIDs")
if beforeID >= behindID {
// This is an impossible situation, we
// can't index anything between these.
return nil
}
t.Lock()
defer t.Unlock()
// Lazily init indexed items.
if t.items.data == nil {
t.items.data = &list.List{}
t.items.data.Init()
}
// Start by mapping out the list so we know what
// we have to do. Depending on the current state
// of the list we might not have to do *anything*.
var (
position int
listLen = t.items.data.Len()
behindIDPosition int
beforeIDPosition int
)
for e := t.items.data.Front(); e != nil; e = e.Next() {
entry := e.Value.(*indexedItemsEntry)
position++
if entry.itemID > behindID {
l.Trace("item is too new, continuing")
continue
}
if behindIDPosition == 0 {
// Gone far enough through the list
// and found our behindID mark.
// We only need to set this once.
l.Tracef("found behindID mark %s at position %d", entry.itemID, position)
behindIDPosition = position
}
if entry.itemID >= beforeID {
// Push the beforeID mark back
// one place every iteration.
l.Tracef("setting beforeID mark %s at position %d", entry.itemID, position)
beforeIDPosition = position
}
if entry.itemID <= beforeID {
// We've gone beyond the bounds of
// items we're interested in; stop.
l.Trace("reached older items, breaking")
break
}
}
// We can now figure out if we need to make db calls.
var grabMore bool
switch {
case listLen < amount:
// The whole list is shorter than the
// amount we're being asked to return,
// make up the difference.
grabMore = true
amount -= listLen
case beforeIDPosition-behindIDPosition < amount:
// Not enough items between behindID and
// beforeID to return amount required,
// try to get more.
grabMore = true
}
if !grabMore {
// We're good!
return nil
}
// Fetch additional items.
items, err := t.grab(ctx, amount, behindID, beforeID, frontToBack)
if err != nil {
return err
}
// Index all the items we got. We already have
// a lock on the timeline, so don't call IndexOne
// here, since that will also try to get a lock!
for _, item := range items {
entry := &indexedItemsEntry{
itemID: item.GetID(),
boostOfID: item.GetBoostOfID(),
accountID: item.GetAccountID(),
boostOfAccountID: item.GetBoostOfAccountID(),
}
if _, err := t.items.insertIndexed(ctx, entry); err != nil {
return gtserror.Newf("error inserting entry with itemID %s into index: %w", entry.itemID, err)
}
}
return nil
}
// grab wraps the timeline's grabFunction in paging + filtering logic.
func (t *timeline) grab(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) ([]Timelineable, error) {
var (
sinceID string
minID string
grabbed int
maxID = behindID
filtered = make([]Timelineable, 0, amount)
)
if frontToBack {
sinceID = beforeID
} else {
minID = beforeID
}
for attempts := 0; attempts < 5; attempts++ {
if grabbed >= amount {
// We got everything we needed.
break
}
items, stop, err := t.grabFunction(
ctx,
t.timelineID,
maxID,
sinceID,
minID,
// Don't grab more than we need to.
amount-grabbed,
)
if err != nil {
// Grab function already checks for
// db.ErrNoEntries, so if an error
// is returned then it's a real one.
return nil, err
}
if stop || len(items) == 0 {
// No items left.
break
}
// Set next query parameters.
if frontToBack {
// Page down.
maxID = items[len(items)-1].GetID()
if maxID <= beforeID {
// Can't go any further.
break
}
} else {
// Page up.
minID = items[0].GetID()
if minID >= behindID {
// Can't go any further.
break
}
}
for _, item := range items {
ok, err := t.filterFunction(ctx, t.timelineID, item)
if err != nil {
if !errors.Is(err, db.ErrNoEntries) {
// Real error here.
return nil, err
}
log.Warnf(ctx, "errNoEntries while filtering item %s: %s", item.GetID(), err)
continue
}
if ok {
filtered = append(filtered, item)
grabbed++ // count this as grabbed
}
}
}
return filtered, nil
}
func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
t.Lock()
defer t.Unlock()
postIndexEntry := &indexedItemsEntry{
itemID: statusID,
boostOfID: boostOfID,
accountID: accountID,
boostOfAccountID: boostOfAccountID,
}
if inserted, err := t.items.insertIndexed(ctx, postIndexEntry); err != nil {
return false, gtserror.Newf("error inserting indexed: %w", err)
} else if !inserted {
// Entry wasn't inserted, so
// don't bother preparing it.
return false, nil
}
preparable, err := t.prepareFunction(ctx, t.timelineID, statusID)
if err != nil {
return true, gtserror.Newf("error preparing: %w", err)
}
postIndexEntry.prepared = preparable
return true, nil
}
func (t *timeline) Len() int {
t.Lock()
defer t.Unlock()
if t.items == nil || t.items.data == nil {
// indexedItems hasnt been initialized yet.
return 0
}
return t.items.data.Len()
}
func (t *timeline) OldestIndexedItemID() string {
t.Lock()
defer t.Unlock()
if t.items == nil || t.items.data == nil {
// indexedItems hasnt been initialized yet.
return ""
}
e := t.items.data.Back()
if e == nil {
// List was empty.
return ""
}
return e.Value.(*indexedItemsEntry).itemID
}

92
internal/timeline/index_test.go
View file

@ -1,92 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
type IndexTestSuite struct {
TimelineStandardTestSuite
}
func (suite *IndexTestSuite) TestOldestIndexedItemIDEmpty() {
var (
ctx = context.Background()
testAccountID = suite.testAccounts["local_account_1"].ID
)
// the oldest indexed post should be an empty string since there's nothing indexed yet
postID := suite.state.Timelines.Home.GetOldestIndexedID(ctx, testAccountID)
suite.Empty(postID)
// indexLength should be 0
suite.Zero(0, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID))
}
func (suite *IndexTestSuite) TestIndexAlreadyIndexed() {
var (
ctx = context.Background()
testAccountID = suite.testAccounts["local_account_1"].ID
testStatus = suite.testStatuses["local_account_1_status_1"]
)
// index one post -- it should be indexed
indexed, err := suite.state.Timelines.Home.IngestOne(ctx, testAccountID, testStatus)
suite.NoError(err)
suite.True(indexed)
// try to index the same post again -- it should not be indexed
indexed, err = suite.state.Timelines.Home.IngestOne(ctx, testAccountID, testStatus)
suite.NoError(err)
suite.False(indexed)
}
func (suite *IndexTestSuite) TestIndexBoostOfAlreadyIndexed() {
var (
ctx = context.Background()
testAccountID = suite.testAccounts["local_account_1"].ID
testStatus = suite.testStatuses["local_account_1_status_1"]
boostOfTestStatus = &gtsmodel.Status{
CreatedAt: time.Now(),
ID: "01FD4TA6G2Z6M7W8NJQ3K5WXYD",
BoostOfID: testStatus.ID,
AccountID: "01FD4TAY1C0NGEJVE9CCCX7QKS",
BoostOfAccountID: testStatus.AccountID,
}
)
// index one post -- it should be indexed
indexed, err := suite.state.Timelines.Home.IngestOne(ctx, testAccountID, testStatus)
suite.NoError(err)
suite.True(indexed)
// try to index the a boost of that post -- it should not be indexed
indexed, err = suite.state.Timelines.Home.IngestOne(ctx, testAccountID, boostOfTestStatus)
suite.NoError(err)
suite.False(indexed)
}
func TestIndexTestSuite(t *testing.T) {
suite.Run(t, new(IndexTestSuite))
}

120
internal/timeline/indexeditems.go
View file

@ -1,120 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"container/list"
"context"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
type indexedItems struct {
data *list.List
skipInsert SkipInsertFunction
}
type indexedItemsEntry struct {
itemID string
boostOfID string
accountID string
boostOfAccountID string
prepared Preparable
}
// WARNING: ONLY CALL THIS FUNCTION IF YOU ALREADY HAVE
// A LOCK ON THE TIMELINE CONTAINING THIS INDEXEDITEMS!
func (i *indexedItems) insertIndexed(ctx context.Context, newEntry *indexedItemsEntry) (bool, error) {
// Lazily init indexed items.
if i.data == nil {
i.data = &list.List{}
i.data.Init()
}
if i.data.Len() == 0 {
// We have no entries yet, meaning this is both the
// newest + oldest entry, so just put it in the front.
i.data.PushFront(newEntry)
return true, nil
}
var (
insertMark *list.Element
currentPosition int
)
// We need to iterate through the index to make sure we put
// this item in the appropriate place according to its id.
// We also need to make sure we're not inserting a duplicate
// item -- this can happen sometimes and it's sucky UX.
for e := i.data.Front(); e != nil; e = e.Next() {
currentPosition++
currentEntry := e.Value.(*indexedItemsEntry)
// Check if we need to skip inserting this item based on
// the current item.
//
// For example, if the new item is a boost, and the current
// item is the original, we may not want to insert the boost
// if it would appear very shortly after the original.
if skip, err := i.skipInsert(
ctx,
newEntry.itemID,
newEntry.accountID,
newEntry.boostOfID,
newEntry.boostOfAccountID,
currentEntry.itemID,
currentEntry.accountID,
currentEntry.boostOfID,
currentEntry.boostOfAccountID,
currentPosition,
); err != nil {
return false, gtserror.Newf("error calling skipInsert: %w", err)
} else if skip {
// We don't need to insert this at all,
// so we can safely bail.
return false, nil
}
if insertMark != nil {
// We already found our mark.
continue
}
if currentEntry.itemID > newEntry.itemID {
// We're still in items newer than
// the one we're trying to insert.
continue
}
// We found our spot!
insertMark = e
}
if insertMark == nil {
// We looked through the whole timeline and didn't find
// a mark, so the new item is the oldest item we've seen;
// insert it at the back.
i.data.PushBack(newEntry)
return true, nil
}
i.data.InsertBefore(newEntry, insertMark)
return true, nil
}

259
internal/timeline/manager.go
View file

@ -1,259 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"context"
"sync"
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
const (
pruneLengthIndexed = 400
pruneLengthPrepared = 50
)
// Manager abstracts functions for creating multiple timelines, and adding, removing, and fetching entries from those timelines.
//
// By the time a timelineable hits the manager interface, it should already have been filtered and it should be established that the item indeed
// belongs in the given timeline.
//
// The manager makes a distinction between *indexed* items and *prepared* items.
//
// Indexed items consist of just that item's ID (in the database) and the time it was created. An indexed item takes up very little memory, so
// it's not a huge priority to keep trimming the indexed items list.
//
// Prepared items consist of the item's database ID, the time it was created, AND the apimodel representation of that item, for quick serialization.
// Prepared items of course take up more memory than indexed items, so they should be regularly pruned if they're not being actively served.
type Manager interface {
// IngestOne takes one timelineable and indexes it into the given timeline, and then immediately prepares it for serving.
// This is useful in cases where we know the item will need to be shown at the top of a user's timeline immediately (eg., a new status is created).
//
// It should already be established before calling this function that the item actually belongs in the timeline!
//
// The returned bool indicates whether the item was actually put in the timeline. This could be false in cases where
// a status is a boost, but a boost of the original status or the status itself already exists recently in the timeline.
IngestOne(ctx context.Context, timelineID string, item Timelineable) (bool, error)
// GetTimeline returns limit n amount of prepared entries from the given timeline, in descending chronological order.
GetTimeline(ctx context.Context, timelineID string, maxID string, sinceID string, minID string, limit int, local bool) ([]Preparable, error)
// GetIndexedLength returns the amount of items that have been indexed for the given account ID.
GetIndexedLength(ctx context.Context, timelineID string) int
// GetOldestIndexedID returns the id ID for the oldest item that we have indexed for the given timeline.
// Will be an empty string if nothing is (yet) indexed.
GetOldestIndexedID(ctx context.Context, timelineID string) string
// Remove removes one item from the given timeline.
Remove(ctx context.Context, timelineID string, itemID string) (int, error)
// RemoveTimeline completely removes one timeline.
RemoveTimeline(ctx context.Context, timelineID string) error
// WipeItemFromAllTimelines removes one item from the index and prepared items of all timelines
WipeItemFromAllTimelines(ctx context.Context, itemID string) error
// WipeStatusesFromAccountID removes all items by the given accountID from the given timeline.
WipeItemsFromAccountID(ctx context.Context, timelineID string, accountID string) error
// UnprepareItem unprepares/uncaches the prepared version fo the given itemID from the given timelineID.
// Use this for cache invalidation when the prepared representation of an item has changed.
UnprepareItem(ctx context.Context, timelineID string, itemID string) error
// UnprepareItemFromAllTimelines unprepares/uncaches the prepared version of the given itemID from all timelines.
// Use this for cache invalidation when the prepared representation of an item has changed.
UnprepareItemFromAllTimelines(ctx context.Context, itemID string) error
// Prune manually triggers a prune operation for the given timelineID.
Prune(ctx context.Context, timelineID string, desiredPreparedItemsLength int, desiredIndexedItemsLength int) (int, error)
// Start starts hourly cleanup jobs for this timeline manager.
Start() error
// Stop stops the timeline manager (currently a stub, doesn't do anything).
Stop() error
}
// NewManager returns a new timeline manager.
func NewManager(grabFunction GrabFunction, filterFunction FilterFunction, prepareFunction PrepareFunction, skipInsertFunction SkipInsertFunction) Manager {
return &manager{
timelines: sync.Map{},
grabFunction: grabFunction,
filterFunction: filterFunction,
prepareFunction: prepareFunction,
skipInsertFunction: skipInsertFunction,
}
}
type manager struct {
timelines sync.Map
grabFunction GrabFunction
filterFunction FilterFunction
prepareFunction PrepareFunction
skipInsertFunction SkipInsertFunction
}
func (m *manager) Start() error {
// Start a background goroutine which iterates
// through all stored timelines once per hour,
// and cleans up old entries if that timeline
// hasn't been accessed in the last hour.
go func() {
for now := range time.NewTicker(1 * time.Hour).C {
now := now // rescope
// Define the range function inside here,
// so that we can use the 'now' returned
// by the ticker, instead of having to call
// time.Now() multiple times.
//
// Unless it panics, this function always
// returns 'true', to continue the Range
// call through the sync.Map.
f := func(_ any, v any) bool {
timeline, ok := v.(Timeline)
if !ok {
log.Panic(nil, "couldn't parse timeline manager sync map value as Timeline, this should never happen so panic")
}
if now.Sub(timeline.LastGot()) < 1*time.Hour {
// Timeline has been fetched in the
// last hour, move on to the next one.
return true
}
if amountPruned := timeline.Prune(pruneLengthPrepared, pruneLengthIndexed); amountPruned > 0 {
log.WithField("accountID", timeline.TimelineID()).Infof("pruned %d indexed and prepared items from timeline", amountPruned)
}
return true
}
// Execute the function for each timeline.
m.timelines.Range(f)
}
}()
return nil
}
func (m *manager) Stop() error {
return nil
}
func (m *manager) IngestOne(ctx context.Context, timelineID string, item Timelineable) (bool, error) {
return m.getOrCreateTimeline(ctx, timelineID).IndexAndPrepareOne(
ctx,
item.GetID(),
item.GetBoostOfID(),
item.GetAccountID(),
item.GetBoostOfAccountID(),
)
}
func (m *manager) Remove(ctx context.Context, timelineID string, itemID string) (int, error) {
return m.getOrCreateTimeline(ctx, timelineID).Remove(ctx, itemID)
}
func (m *manager) RemoveTimeline(ctx context.Context, timelineID string) error {
m.timelines.Delete(timelineID)
return nil
}
func (m *manager) GetTimeline(ctx context.Context, timelineID string, maxID string, sinceID string, minID string, limit int, local bool) ([]Preparable, error) {
return m.getOrCreateTimeline(ctx, timelineID).Get(ctx, limit, maxID, sinceID, minID, true)
}
func (m *manager) GetIndexedLength(ctx context.Context, timelineID string) int {
return m.getOrCreateTimeline(ctx, timelineID).Len()
}
func (m *manager) GetOldestIndexedID(ctx context.Context, timelineID string) string {
return m.getOrCreateTimeline(ctx, timelineID).OldestIndexedItemID()
}
func (m *manager) WipeItemFromAllTimelines(ctx context.Context, itemID string) error {
errs := new(gtserror.MultiError)
m.timelines.Range(func(_ any, v any) bool {
if _, err := v.(Timeline).Remove(ctx, itemID); err != nil {
errs.Append(err)
}
return true // always continue range
})
if err := errs.Combine(); err != nil {
return gtserror.Newf("error(s) wiping status %s: %w", itemID, errs.Combine())
}
return nil
}
func (m *manager) WipeItemsFromAccountID(ctx context.Context, timelineID string, accountID string) error {
_, err := m.getOrCreateTimeline(ctx, timelineID).RemoveAllByOrBoosting(ctx, accountID)
return err
}
func (m *manager) UnprepareItemFromAllTimelines(ctx context.Context, itemID string) error {
errs := new(gtserror.MultiError)
// Work through all timelines held by this
// manager, and call Unprepare for each.
m.timelines.Range(func(_ any, v any) bool {
if err := v.(Timeline).Unprepare(ctx, itemID); err != nil {
errs.Append(err)
}
return true // always continue range
})
if err := errs.Combine(); err != nil {
return gtserror.Newf("error(s) unpreparing status %s: %w", itemID, errs.Combine())
}
return nil
}
func (m *manager) UnprepareItem(ctx context.Context, timelineID string, itemID string) error {
return m.getOrCreateTimeline(ctx, timelineID).Unprepare(ctx, itemID)
}
func (m *manager) Prune(ctx context.Context, timelineID string, desiredPreparedItemsLength int, desiredIndexedItemsLength int) (int, error) {
return m.getOrCreateTimeline(ctx, timelineID).Prune(desiredPreparedItemsLength, desiredIndexedItemsLength), nil
}
// getOrCreateTimeline returns a timeline with the given id,
// creating a new timeline with that id if necessary.
func (m *manager) getOrCreateTimeline(ctx context.Context, timelineID string) Timeline {
i, ok := m.timelines.Load(timelineID)
if ok {
// Timeline already existed in sync.Map.
return i.(Timeline)
}
// Timeline did not yet exist in sync.Map.
// Create + store it.
timeline := NewTimeline(ctx, timelineID, m.grabFunction, m.filterFunction, m.prepareFunction, m.skipInsertFunction)
m.timelines.Store(timelineID, timeline)
return timeline
}

146
internal/timeline/prepare.go
View file

@ -1,146 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"container/list"
"context"
"errors"
"codeberg.org/gruf/go-kv"
"github.com/superseriousbusiness/gotosocial/internal/db"
statusfilter "github.com/superseriousbusiness/gotosocial/internal/filter/status"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
func (t *timeline) prepareXBetweenIDs(ctx context.Context, amount int, behindID string, beforeID string, frontToBack bool) error {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"amount", amount},
{"behindID", behindID},
{"beforeID", beforeID},
{"frontToBack", frontToBack},
}...)
l.Trace("entering prepareXBetweenIDs")
if beforeID >= behindID {
// This is an impossible situation, we
// can't prepare anything between these.
return nil
}
if err := t.indexXBetweenIDs(ctx, amount, behindID, beforeID, frontToBack); err != nil {
// An error here doesn't necessarily mean we
// can't prepare anything, so log + keep going.
l.Debugf("error calling prepareXBetweenIDs: %s", err)
}
t.Lock()
defer t.Unlock()
// Try to prepare everything between (and including) the two points.
var (
toPrepare = make(map[*list.Element]*indexedItemsEntry)
foundToPrepare int
)
if frontToBack {
// Paging forwards / down.
for e := t.items.data.Front(); e != nil; e = e.Next() {
entry := e.Value.(*indexedItemsEntry)
if entry.itemID > behindID {
l.Trace("item is too new, continuing")
continue
}
if entry.itemID < beforeID {
// We've gone beyond the bounds of
// items we're interested in; stop.
l.Trace("reached older items, breaking")
break
}
// Only prepare entry if it's not
// already prepared, save db calls.
if entry.prepared == nil {
toPrepare[e] = entry
}
foundToPrepare++
if foundToPrepare >= amount {
break
}
}
} else {
// Paging backwards / up.
for e := t.items.data.Back(); e != nil; e = e.Prev() {
entry := e.Value.(*indexedItemsEntry)
if entry.itemID < beforeID {
l.Trace("item is too old, continuing")
continue
}
if entry.itemID > behindID {
// We've gone beyond the bounds of
// items we're interested in; stop.
l.Trace("reached newer items, breaking")
break
}
if entry.prepared == nil {
toPrepare[e] = entry
}
// Only prepare entry if it's not
// already prepared, save db calls.
foundToPrepare++
if foundToPrepare >= amount {
break
}
}
}
for e, entry := range toPrepare {
prepared, err := t.prepareFunction(ctx, t.timelineID, entry.itemID)
if err != nil {
if errors.Is(err, statusfilter.ErrHideStatus) {
// This item has been filtered out by the requesting user's filters.
// Remove it and skip past it.
t.items.data.Remove(e)
continue
}
if errors.Is(err, db.ErrNoEntries) {
// ErrNoEntries means something has been deleted,
// so we'll likely not be able to ever prepare this.
// This means we can remove it and skip past it.
l.Debugf("db.ErrNoEntries while trying to prepare %s; will remove from timeline", entry.itemID)
t.items.data.Remove(e)
continue
}
// We've got a proper db error.
return gtserror.Newf("db error while trying to prepare %s: %w", entry.itemID, err)
}
entry.prepared = prepared
}
return nil
}

83
internal/timeline/prune.go
View file

@ -1,83 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"container/list"
)
func (t *timeline) Prune(desiredPreparedItemsLength int, desiredIndexedItemsLength int) int {
t.Lock()
defer t.Unlock()
l := t.items.data
if l == nil {
// Nothing to prune.
return 0
}
var (
position int
totalPruned int
toRemove *[]*list.Element
)
// Only initialize toRemove if we know we're
// going to need it, otherwise skiperino.
if toRemoveLen := t.items.data.Len() - desiredIndexedItemsLength; toRemoveLen > 0 {
toRemove = func() *[]*list.Element { tr := make([]*list.Element, 0, toRemoveLen); return &tr }()
}
// Work from the front of the list until we get
// to the point where we need to start pruning.
for e := l.Front(); e != nil; e = e.Next() {
position++
if position <= desiredPreparedItemsLength {
// We're still within our allotted
// prepped length, nothing to do yet.
continue
}
// We need to *at least* unprepare this entry.
// If we're beyond our indexed length already,
// we can just remove the item completely.
if position > desiredIndexedItemsLength {
*toRemove = append(*toRemove, e)
totalPruned++
continue
}
entry := e.Value.(*indexedItemsEntry)
if entry.prepared == nil {
// It's already unprepared (mood).
continue
}
entry.prepared = nil // <- eat this up please garbage collector nom nom nom
totalPruned++
}
if toRemove != nil {
for _, e := range *toRemove {
l.Remove(e)
}
}
return totalPruned
}

103
internal/timeline/prune_test.go
View file

@ -1,103 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline_test
import (
"context"
"testing"
"github.com/stretchr/testify/suite"
)
type PruneTestSuite struct {
TimelineStandardTestSuite
}
func (suite *PruneTestSuite) TestPrune() {
var (
ctx = context.Background()
testAccountID = suite.testAccounts["local_account_1"].ID
desiredPreparedItemsLength = 5
desiredIndexedItemsLength = 5
)
suite.fillTimeline(testAccountID)
pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength)
suite.NoError(err)
suite.Equal(25, pruned)
suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID))
}
func (suite *PruneTestSuite) TestPruneTwice() {
var (
ctx = context.Background()
testAccountID = suite.testAccounts["local_account_1"].ID
desiredPreparedItemsLength = 5
desiredIndexedItemsLength = 5
)
suite.fillTimeline(testAccountID)
pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength)
suite.NoError(err)
suite.Equal(25, pruned)
suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID))
// Prune same again, nothing should be pruned this time.
pruned, err = suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength)
suite.NoError(err)
suite.Equal(0, pruned)
suite.Equal(5, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID))
}
func (suite *PruneTestSuite) TestPruneTo0() {
var (
ctx = context.Background()
testAccountID = suite.testAccounts["local_account_1"].ID
desiredPreparedItemsLength = 0
desiredIndexedItemsLength = 0
)
suite.fillTimeline(testAccountID)
pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength)
suite.NoError(err)
suite.Equal(30, pruned)
suite.Equal(0, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID))
}
func (suite *PruneTestSuite) TestPruneToInfinityAndBeyond() {
var (
ctx = context.Background()
testAccountID = suite.testAccounts["local_account_1"].ID
desiredPreparedItemsLength = 9999999
desiredIndexedItemsLength = 9999999
)
suite.fillTimeline(testAccountID)
pruned, err := suite.state.Timelines.Home.Prune(ctx, testAccountID, desiredPreparedItemsLength, desiredIndexedItemsLength)
suite.NoError(err)
suite.Equal(0, pruned)
suite.Equal(30, suite.state.Timelines.Home.GetIndexedLength(ctx, testAccountID))
}
func TestPruneTestSuite(t *testing.T) {
suite.Run(t, new(PruneTestSuite))
}

97
internal/timeline/remove.go
View file

@ -1,97 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"container/list"
"context"
"codeberg.org/gruf/go-kv"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
func (t *timeline) Remove(ctx context.Context, statusID string) (int, error) {
l := log.WithContext(ctx).
WithFields(kv.Fields{
{"accountTimeline", t.timelineID},
{"statusID", statusID},
}...)
t.Lock()
defer t.Unlock()
if t.items == nil || t.items.data == nil {
// Nothing to do.
return 0, nil
}
var toRemove []*list.Element
for e := t.items.data.Front(); e != nil; e = e.Next() {
entry := e.Value.(*indexedItemsEntry)
if entry.itemID != statusID {
// Not relevant.
continue
}
l.Debug("removing item")
toRemove = append(toRemove, e)
}
for _, e := range toRemove {
t.items.data.Remove(e)
}
return len(toRemove), nil
}
func (t *timeline) RemoveAllByOrBoosting(ctx context.Context, accountID string) (int, error) {
l := log.
WithContext(ctx).
WithFields(kv.Fields{
{"accountTimeline", t.timelineID},
{"accountID", accountID},
}...)
t.Lock()
defer t.Unlock()
if t.items == nil || t.items.data == nil {
// Nothing to do.
return 0, nil
}
var toRemove []*list.Element
for e := t.items.data.Front(); e != nil; e = e.Next() {
entry := e.Value.(*indexedItemsEntry)
if entry.accountID != accountID && entry.boostOfAccountID != accountID {
// Not relevant.
continue
}
l.Debug("removing item")
toRemove = append(toRemove, e)
}
for _, e := range toRemove {
t.items.data.Remove(e)
}
return len(toRemove), nil
}

172
internal/timeline/timeline.go
View file

@ -1,172 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"context"
"sync"
"time"
)
// GrabFunction is used by a Timeline to grab more items to index.
//
// It should be provided to NewTimeline when the caller is creating a timeline
// (of statuses, notifications, etc).
//
// - timelineID: ID of the timeline.
// - maxID: the maximum item ID desired.
// - sinceID: the minimum item ID desired.
// - minID: see sinceID
// - limit: the maximum amount of items to be returned
//
// If an error is returned, the timeline will stop processing whatever request called GrabFunction,
// and return the error. If no error is returned, but stop = true, this indicates to the caller of GrabFunction
// that there are no more items to return, and processing should continue with the items already grabbed.
type GrabFunction func(ctx context.Context, timelineID string, maxID string, sinceID string, minID string, limit int) (items []Timelineable, stop bool, err error)
// FilterFunction is used by a Timeline to filter whether or not a grabbed item should be indexed.
type FilterFunction func(ctx context.Context, timelineID string, item Timelineable) (shouldIndex bool, err error)
// PrepareFunction converts a Timelineable into a Preparable.
//
// For example, this might result in the converstion of a *gtsmodel.Status with the given itemID into a serializable *apimodel.Status.
type PrepareFunction func(ctx context.Context, timelineID string, itemID string) (Preparable, error)
// SkipInsertFunction indicates whether a new item about to be inserted in the prepared list should be skipped,
// based on the item itself, the next item in the timeline, and the depth at which nextItem has been found in the list.
//
// This will be called for every item found while iterating through a timeline, so callers should be very careful
// not to do anything expensive here.
type SkipInsertFunction func(ctx context.Context,
newItemID string,
newItemAccountID string,
newItemBoostOfID string,
newItemBoostOfAccountID string,
nextItemID string,
nextItemAccountID string,
nextItemBoostOfID string,
nextItemBoostOfAccountID string,
depth int) (bool, error)
// Timeline represents a timeline for one account, and contains indexed and prepared items.
type Timeline interface {
/*
RETRIEVAL FUNCTIONS
*/
// Get returns an amount of prepared items with the given parameters.
// If prepareNext is true, then the next predicted query will be prepared already in a goroutine,
// to make the next call to Get faster.
Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error)
/*
INDEXING + PREPARATION FUNCTIONS
*/
// IndexAndPrepareOne puts a item into the timeline at the appropriate place
// according to its id, and then immediately prepares it.
//
// The returned bool indicates whether or not the item was actually inserted
// into the timeline. This will be false if the item is a boost and the original
// item, or a boost of it, already exists recently in the timeline.
IndexAndPrepareOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// Unprepare clears the prepared version of the given item (and any boosts
// thereof) from the timeline, but leaves the indexed version in place.
//
// This is useful for cache invalidation when the prepared version of the
// item has changed for some reason (edits, updates, etc), but the item does
// not need to be removed: it will be prepared again next time Get is called.
Unprepare(ctx context.Context, itemID string) error
/*
INFO FUNCTIONS
*/
// TimelineID returns the id of this timeline.
TimelineID() string
// Len returns the length of the item index at this point in time.
Len() int
// OldestIndexedItemID returns the id of the rearmost (ie., the oldest) indexed item.
// If there's no oldest item, an empty string will be returned so make sure to check for this.
OldestIndexedItemID() string
/*
UTILITY FUNCTIONS
*/
// LastGot returns the time that Get was last called.
LastGot() time.Time
// Prune prunes prepared and indexed items in this timeline to the desired lengths.
// This will be a no-op if the lengths are already < the desired values.
//
// The returned int indicates the amount of entries that were removed or unprepared.
Prune(desiredPreparedItemsLength int, desiredIndexedItemsLength int) int
// Remove removes an item with the given ID.
//
// If a item has multiple entries in a timeline, they will all be removed.
//
// The returned int indicates the amount of entries that were removed.
Remove(ctx context.Context, itemID string) (int, error)
// RemoveAllByOrBoosting removes all items created by or boosting the given accountID.
//
// The returned int indicates the amount of entries that were removed.
RemoveAllByOrBoosting(ctx context.Context, accountID string) (int, error)
}
// timeline fulfils the Timeline interface
type timeline struct {
items *indexedItems
grabFunction GrabFunction
filterFunction FilterFunction
prepareFunction PrepareFunction
timelineID string
lastGot time.Time
sync.Mutex
}
func (t *timeline) TimelineID() string {
return t.timelineID
}
// NewTimeline returns a new Timeline with
// the given ID, using the given functions.
func NewTimeline(
ctx context.Context,
timelineID string,
grabFunction GrabFunction,
filterFunction FilterFunction,
prepareFunction PrepareFunction,
skipInsertFunction SkipInsertFunction,
) Timeline {
return &timeline{
items: &indexedItems{
skipInsert: skipInsertFunction,
},
grabFunction: grabFunction,
filterFunction: filterFunction,
prepareFunction: prepareFunction,
timelineID: timelineID,
lastGot: time.Time{},
}
}

98
internal/timeline/timeline_test.go
View file

@ -1,98 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline_test
import (
"context"
"sort"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type TimelineStandardTestSuite struct {
suite.Suite
state *state.State
testAccounts map[string]*gtsmodel.Account
testStatuses map[string]*gtsmodel.Status
highestStatusID string
lowestStatusID string
}
func (suite *TimelineStandardTestSuite) SetupSuite() {
suite.testAccounts = testrig.NewTestAccounts()
suite.testStatuses = testrig.NewTestStatuses()
}
func (suite *TimelineStandardTestSuite) SetupTest() {
suite.state = new(state.State)
suite.state.Caches.Init()
testrig.StartNoopWorkers(suite.state)
testrig.InitTestConfig()
testrig.InitTestLog()
suite.state.DB = testrig.NewTestDB(suite.state)
testrig.StartTimelines(
suite.state,
visibility.NewFilter(suite.state),
typeutils.NewConverter(suite.state),
)
testrig.StandardDBSetup(suite.state.DB, nil)
}
func (suite *TimelineStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.state.DB)
testrig.StopWorkers(suite.state)
}
func (suite *TimelineStandardTestSuite) fillTimeline(timelineID string) {
// Put testrig statuses in a determinate order
// since we can't trust a map to keep order.
statuses := []*gtsmodel.Status{}
for _, s := range suite.testStatuses {
statuses = append(statuses, s)
}
sort.Slice(statuses, func(i, j int) bool {
return statuses[i].ID > statuses[j].ID
})
// Statuses are now highest -> lowest.
suite.highestStatusID = statuses[0].ID
suite.lowestStatusID = statuses[len(statuses)-1].ID
if suite.highestStatusID < suite.lowestStatusID {
suite.FailNow("", "statuses weren't ordered properly by sort")
}
// Put all test statuses into the timeline; we don't
// need to be fussy about who sees what for these tests.
for _, status := range statuses {
if _, err := suite.state.Timelines.Home.IngestOne(context.Background(), timelineID, status); err != nil {
suite.FailNow(err.Error())
}
}
}

37
internal/timeline/timelines.go
View file

@ -1,37 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
type Timelines struct {
// Home provides access to account home timelines.
Home Manager
// List provides access to list timelines.
List Manager
// prevent pass-by-value.
_ nocopy
}
// nocopy when embedded will signal linter to
// error on pass-by-value of parent struct.
type nocopy struct{}
func (*nocopy) Lock() {}
func (*nocopy) Unlock() {}

34
internal/timeline/types.go
View file

@ -1,34 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
// Timelineable represents any item that can be indexed in a timeline.
type Timelineable interface {
GetID() string
GetAccountID() string
GetBoostOfID() string
GetBoostOfAccountID() string
}
// Preparable represents any item that can be prepared in a timeline.
type Preparable interface {
GetID() string
GetAccountID() string
GetBoostOfID() string
GetBoostOfAccountID() string
}

50
internal/timeline/unprepare.go
View file

@ -1,50 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"context"
)
func (t *timeline) Unprepare(ctx context.Context, itemID string) error {
t.Lock()
defer t.Unlock()
if t.items == nil || t.items.data == nil {
// Nothing to do.
return nil
}
for e := t.items.data.Front(); e != nil; e = e.Next() {
entry := e.Value.(*indexedItemsEntry)
if entry.itemID != itemID && entry.boostOfID != itemID {
// Not relevant.
continue
}
if entry.prepared == nil {
// It's already unprepared (mood).
continue
}
entry.prepared = nil // <- eat this up please garbage collector nom nom nom
}
return nil
}

142
internal/timeline/unprepare_test.go
View file

@ -1,142 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline_test
import (
"context"
"testing"
"github.com/stretchr/testify/suite"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
)
type UnprepareTestSuite struct {
TimelineStandardTestSuite
}
func (suite *UnprepareTestSuite) TestUnprepareFromFave() {
var (
ctx = context.Background()
testAccount = suite.testAccounts["local_account_1"]
maxID = ""
sinceID = ""
minID = ""
limit = 1
local = false
)
suite.fillTimeline(testAccount.ID)
// Get first status from the top (no params).
statuses, err := suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
if len(statuses) != 1 {
suite.FailNow("couldn't get top status")
}
targetStatus := statuses[0].(*apimodel.Status)
// Check fave stats of the top status.
suite.Equal(0, targetStatus.FavouritesCount)
suite.False(targetStatus.Favourited)
// Fave the top status from testAccount.
if err := suite.state.DB.PutStatusFave(ctx, &gtsmodel.StatusFave{
ID: id.NewULID(),
AccountID: testAccount.ID,
TargetAccountID: targetStatus.Account.ID,
StatusID: targetStatus.ID,
URI: "https://example.org/some/activity/path",
}); err != nil {
suite.FailNow(err.Error())
}
// Repeat call to get first status from the top.
// Get first status from the top (no params).
statuses, err = suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
if len(statuses) != 1 {
suite.FailNow("couldn't get top status")
}
targetStatus = statuses[0].(*apimodel.Status)
// We haven't yet uncached/unprepared the status,
// we've only inserted the fave, so counts should
// stay the same...
suite.Equal(0, targetStatus.FavouritesCount)
suite.False(targetStatus.Favourited)
// Now call unprepare.
suite.state.Timelines.Home.UnprepareItemFromAllTimelines(ctx, targetStatus.ID)
// Now a Get should trigger a fresh prepare of the
// target status, and the counts should be updated.
// Repeat call to get first status from the top.
// Get first status from the top (no params).
statuses, err = suite.state.Timelines.Home.GetTimeline(
ctx,
testAccount.ID,
maxID,
sinceID,
minID,
limit,
local,
)
if err != nil {
suite.FailNow(err.Error())
}
if len(statuses) != 1 {
suite.FailNow("couldn't get top status")
}
targetStatus = statuses[0].(*apimodel.Status)
suite.Equal(1, targetStatus.FavouritesCount)
suite.True(targetStatus.Favourited)
}
func TestUnprepareTestSuite(t *testing.T) {
suite.Run(t, new(UnprepareTestSuite))
}