mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 04:52:24 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Pg to bun (#148)

Browse source 
* start moving to bun

* changing more stuff

* more

* and yet more

* tests passing

* seems stable now

* more big changes

* small fix

* little fixes
This commit is contained in:
tobi 2021-08-25 15:34:33 +02:00 • committed by GitHub
commit 2dc9fc1626
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
713 changed files with 98694 additions and 22704 deletions
Download patch file Download diff file Expand all files Collapse all files

41
internal/timeline/get.go
View file

@ -20,6 +20,7 @@ package timeline
import (
"container/list"
"context"
"errors"
"fmt"
@ -29,7 +30,7 @@ import (
const retries = 5
func (t *timeline) Get(amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]*apimodel.Status, error) {
func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]*apimodel.Status, error) {
l := t.log.WithFields(logrus.Fields{
"func": "Get",
"accountID": t.accountID,
@ -46,14 +47,15 @@ func (t *timeline) Get(amount int, maxID string, sinceID string, minID string, p
// no params are defined to just fetch from the top
// this is equivalent to a user asking for the top x posts from their timeline
if maxID == "" && sinceID == "" && minID == "" {
statuses, err = t.GetXFromTop(amount)
statuses, err = t.GetXFromTop(ctx, amount)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
if len(statuses) != 0 {
nextMaxID := statuses[len(statuses)-1].ID
if prepareNext {
// already cache the next query to speed up scrolling
go func() {
if err := t.prepareNextQuery(amount, nextMaxID, "", ""); err != nil {
// use context.Background() because we don't want the query to abort when the request finishes
if err := t.prepareNextQuery(context.Background(), amount, nextMaxID, "", ""); err != nil {
l.Errorf("error preparing next query: %s", err)
}
}()
@ -65,14 +67,15 @@ func (t *timeline) Get(amount int, maxID string, sinceID string, minID string, p
// this is equivalent to a user asking for the next x posts from their timeline, starting from maxID
if maxID != "" && sinceID == "" {
attempts := 0
statuses, err = t.GetXBehindID(amount, maxID, &attempts)
statuses, err = t.GetXBehindID(ctx, amount, maxID, &attempts)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
if len(statuses) != 0 {
nextMaxID := statuses[len(statuses)-1].ID
if prepareNext {
// already cache the next query to speed up scrolling
go func() {
if err := t.prepareNextQuery(amount, nextMaxID, "", ""); err != nil {
// use context.Background() because we don't want the query to abort when the request finishes
if err := t.prepareNextQuery(context.Background(), amount, nextMaxID, "", ""); err != nil {
l.Errorf("error preparing next query: %s", err)
}
}()
@ -83,25 +86,25 @@ func (t *timeline) Get(amount int, maxID string, sinceID string, minID string, p
// maxID is defined and sinceID || minID are as well, so take a slice between them
// this is equivalent to a user asking for posts older than x but newer than y
if maxID != "" && sinceID != "" {
statuses, err = t.GetXBetweenID(amount, maxID, minID)
statuses, err = t.GetXBetweenID(ctx, amount, maxID, minID)
}
if maxID != "" && minID != "" {
statuses, err = t.GetXBetweenID(amount, maxID, minID)
statuses, err = t.GetXBetweenID(ctx, amount, maxID, minID)
}
// maxID isn't defined, but sinceID || minID are, so take x before
// this is equivalent to a user asking for posts newer than x (eg., refreshing the top of their timeline)
if maxID == "" && sinceID != "" {
statuses, err = t.GetXBeforeID(amount, sinceID, true)
statuses, err = t.GetXBeforeID(ctx, amount, sinceID, true)
}
if maxID == "" && minID != "" {
statuses, err = t.GetXBeforeID(amount, minID, true)
statuses, err = t.GetXBeforeID(ctx, amount, minID, true)
}
return statuses, err
}
func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) {
func (t *timeline) GetXFromTop(ctx context.Context, amount int) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
@ -111,7 +114,7 @@ func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) {
// make sure we have enough posts prepared to return
if t.preparedPosts.data.Len() < amount {
if err := t.PrepareFromTop(amount); err != nil {
if err := t.PrepareFromTop(ctx, amount); err != nil {
return nil, err
}
}
@ -133,7 +136,7 @@ func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) {
return statuses, nil
}
func (t *timeline) GetXBehindID(amount int, behindID string, attempts *int) ([]*apimodel.Status, error) {
func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string, attempts *int) ([]*apimodel.Status, error) {
l := t.log.WithFields(logrus.Fields{
"func": "GetXBehindID",
"amount": amount,
@ -174,10 +177,10 @@ findMarkLoop:
// we didn't find it, so we need to make sure it's indexed and prepared and then try again
// this can happen when a user asks for really old posts
if behindIDMark == nil {
if err := t.PrepareBehind(behindID, amount); err != nil {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID)
}
oldestID, err := t.OldestPreparedPostID()
oldestID, err := t.OldestPreparedPostID(ctx)
if err != nil {
return nil, err
}
@ -194,12 +197,12 @@ findMarkLoop:
return statuses, nil
}
l.Trace("trying GetXBehindID again")
return t.GetXBehindID(amount, behindID, attempts)
return t.GetXBehindID(ctx, amount, behindID, attempts)
}
// make sure we have enough posts prepared behind it to return what we're being asked for
if t.preparedPosts.data.Len() < amount+position {
if err := t.PrepareBehind(behindID, amount); err != nil {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, err
}
}
@ -224,7 +227,7 @@ serveloop:
return statuses, nil
}
func (t *timeline) GetXBeforeID(amount int, beforeID string, startFromTop bool) ([]*apimodel.Status, error) {
func (t *timeline) GetXBeforeID(ctx context.Context, amount int, beforeID string, startFromTop bool) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
@ -295,7 +298,7 @@ findMarkLoop:
return statuses, nil
}
func (t *timeline) GetXBetweenID(amount int, behindID string, beforeID string) ([]*apimodel.Status, error) {
func (t *timeline) GetXBetweenID(ctx context.Context, amount int, behindID string, beforeID string) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
@ -327,7 +330,7 @@ findMarkLoop:
// make sure we have enough posts prepared behind it to return what we're being asked for
if t.preparedPosts.data.Len() < amount+position {
if err := t.PrepareBehind(behindID, amount); err != nil {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, err
}
}

37
internal/timeline/get_test.go
View file

@ -19,6 +19,7 @@
package timeline_test
import (
"context"
"testing"
"time"
@ -45,14 +46,14 @@ func (suite *GetTestSuite) SetupTest() {
testrig.StandardDBSetup(suite.db, nil)
// let's take local_account_1 as the timeline owner
tl, err := timeline.NewTimeline(suite.testAccounts["local_account_1"].ID, suite.db, suite.tc, suite.log)
tl, err := timeline.NewTimeline(context.Background(), suite.testAccounts["local_account_1"].ID, suite.db, suite.tc, suite.log)
if err != nil {
suite.FailNow(err.Error())
}
// prepare the timeline by just shoving all test statuses in it -- let's not be fussy about who sees what
for _, s := range suite.testStatuses {
_, err := tl.IndexAndPrepareOne(s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID)
_, err := tl.IndexAndPrepareOne(context.Background(), s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID)
if err != nil {
suite.FailNow(err.Error())
}
@ -67,7 +68,7 @@ func (suite *GetTestSuite) TearDownTest() {
func (suite *GetTestSuite) TestGetDefault() {
// get 10 20 the top and don't prepare the next query
statuses, err := suite.timeline.Get(20, "", "", "", false)
statuses, err := suite.timeline.Get(context.Background(), 20, "", "", "", false)
if err != nil {
suite.FailNow(err.Error())
}
@ -89,7 +90,7 @@ func (suite *GetTestSuite) TestGetDefault() {
func (suite *GetTestSuite) TestGetDefaultPrepareNext() {
// get 10 from the top and prepare the next query
statuses, err := suite.timeline.Get(10, "", "", "", true)
statuses, err := suite.timeline.Get(context.Background(), 10, "", "", "", true)
if err != nil {
suite.FailNow(err.Error())
}
@ -113,7 +114,7 @@ func (suite *GetTestSuite) TestGetDefaultPrepareNext() {
func (suite *GetTestSuite) TestGetMaxID() {
// ask for 10 with a max ID somewhere in the middle of the stack
statuses, err := suite.timeline.Get(10, "01F8MHBQCBTDKN6X5VHGMMN4MA", "", "", false)
statuses, err := suite.timeline.Get(context.Background(), 10, "01F8MHBQCBTDKN6X5VHGMMN4MA", "", "", false)
if err != nil {
suite.FailNow(err.Error())
}
@ -135,7 +136,7 @@ func (suite *GetTestSuite) TestGetMaxID() {
func (suite *GetTestSuite) TestGetMaxIDPrepareNext() {
// ask for 10 with a max ID somewhere in the middle of the stack
statuses, err := suite.timeline.Get(10, "01F8MHBQCBTDKN6X5VHGMMN4MA", "", "", true)
statuses, err := suite.timeline.Get(context.Background(), 10, "01F8MHBQCBTDKN6X5VHGMMN4MA", "", "", true)
if err != nil {
suite.FailNow(err.Error())
}
@ -160,7 +161,7 @@ func (suite *GetTestSuite) TestGetMaxIDPrepareNext() {
func (suite *GetTestSuite) TestGetMinID() {
// ask for 10 with a min ID somewhere in the middle of the stack
statuses, err := suite.timeline.Get(10, "", "01F8MHBQCBTDKN6X5VHGMMN4MA", "", false)
statuses, err := suite.timeline.Get(context.Background(), 10, "", "01F8MHBQCBTDKN6X5VHGMMN4MA", "", false)
if err != nil {
suite.FailNow(err.Error())
}
@ -182,7 +183,7 @@ func (suite *GetTestSuite) TestGetMinID() {
func (suite *GetTestSuite) TestGetSinceID() {
// ask for 10 with a since ID somewhere in the middle of the stack
statuses, err := suite.timeline.Get(10, "", "", "01F8MHBQCBTDKN6X5VHGMMN4MA", false)
statuses, err := suite.timeline.Get(context.Background(), 10, "", "", "01F8MHBQCBTDKN6X5VHGMMN4MA", false)
if err != nil {
suite.FailNow(err.Error())
}
@ -204,7 +205,7 @@ func (suite *GetTestSuite) TestGetSinceID() {
func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
// ask for 10 with a since ID somewhere in the middle of the stack
statuses, err := suite.timeline.Get(10, "", "", "01F8MHBQCBTDKN6X5VHGMMN4MA", true)
statuses, err := suite.timeline.Get(context.Background(), 10, "", "", "01F8MHBQCBTDKN6X5VHGMMN4MA", true)
if err != nil {
suite.FailNow(err.Error())
}
@ -229,7 +230,7 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
func (suite *GetTestSuite) TestGetBetweenID() {
// ask for 10 between these two IDs
statuses, err := suite.timeline.Get(10, "01F8MHCP5P2NWYQ416SBA0XSEV", "", "01F8MHBQCBTDKN6X5VHGMMN4MA", false)
statuses, err := suite.timeline.Get(context.Background(), 10, "01F8MHCP5P2NWYQ416SBA0XSEV", "", "01F8MHBQCBTDKN6X5VHGMMN4MA", false)
if err != nil {
suite.FailNow(err.Error())
}
@ -251,7 +252,7 @@ func (suite *GetTestSuite) TestGetBetweenID() {
func (suite *GetTestSuite) TestGetBetweenIDPrepareNext() {
// ask for 10 between these two IDs
statuses, err := suite.timeline.Get(10, "01F8MHCP5P2NWYQ416SBA0XSEV", "", "01F8MHBQCBTDKN6X5VHGMMN4MA", true)
statuses, err := suite.timeline.Get(context.Background(), 10, "01F8MHCP5P2NWYQ416SBA0XSEV", "", "01F8MHBQCBTDKN6X5VHGMMN4MA", true)
if err != nil {
suite.FailNow(err.Error())
}
@ -276,7 +277,7 @@ func (suite *GetTestSuite) TestGetBetweenIDPrepareNext() {
func (suite *GetTestSuite) TestGetXFromTop() {
// get 5 from the top
statuses, err := suite.timeline.GetXFromTop(5)
statuses, err := suite.timeline.GetXFromTop(context.Background(), 5)
if err != nil {
suite.FailNow(err.Error())
}
@ -300,7 +301,7 @@ func (suite *GetTestSuite) TestGetXBehindID() {
var attempts *int
a := 0
attempts = &a
statuses, err := suite.timeline.GetXBehindID(3, "01F8MHBQCBTDKN6X5VHGMMN4MA", attempts)
statuses, err := suite.timeline.GetXBehindID(context.Background(), 3, "01F8MHBQCBTDKN6X5VHGMMN4MA", attempts)
if err != nil {
suite.FailNow(err.Error())
}
@ -326,7 +327,7 @@ func (suite *GetTestSuite) TestGetXBehindID0() {
var attempts *int
a := 0
attempts = &a
statuses, err := suite.timeline.GetXBehindID(3, "0", attempts)
statuses, err := suite.timeline.GetXBehindID(context.Background(), 3, "0", attempts)
if err != nil {
suite.FailNow(err.Error())
}
@ -340,7 +341,7 @@ func (suite *GetTestSuite) TestGetXBehindNonexistentReasonableID() {
var attempts *int
a := 0
attempts = &a
statuses, err := suite.timeline.GetXBehindID(3, "01F8MHBQCBTDKN6X5VHGMMN4MB", attempts) // change the last A to a B
statuses, err := suite.timeline.GetXBehindID(context.Background(), 3, "01F8MHBQCBTDKN6X5VHGMMN4MB", attempts) // change the last A to a B
if err != nil {
suite.FailNow(err.Error())
}
@ -365,7 +366,7 @@ func (suite *GetTestSuite) TestGetXBehindVeryHighID() {
var attempts *int
a := 0
attempts = &a
statuses, err := suite.timeline.GetXBehindID(7, "9998MHBQCBTDKN6X5VHGMMN4MA", attempts)
statuses, err := suite.timeline.GetXBehindID(context.Background(), 7, "9998MHBQCBTDKN6X5VHGMMN4MA", attempts)
if err != nil {
suite.FailNow(err.Error())
}
@ -389,7 +390,7 @@ func (suite *GetTestSuite) TestGetXBehindVeryHighID() {
func (suite *GetTestSuite) TestGetXBeforeID() {
// get 3 before the 'middle' id
statuses, err := suite.timeline.GetXBeforeID(3, "01F8MHBQCBTDKN6X5VHGMMN4MA", true)
statuses, err := suite.timeline.GetXBeforeID(context.Background(), 3, "01F8MHBQCBTDKN6X5VHGMMN4MA", true)
if err != nil {
suite.FailNow(err.Error())
}
@ -412,7 +413,7 @@ func (suite *GetTestSuite) TestGetXBeforeID() {
func (suite *GetTestSuite) TestGetXBeforeIDNoStartFromTop() {
// get 3 before the 'middle' id
statuses, err := suite.timeline.GetXBeforeID(3, "01F8MHBQCBTDKN6X5VHGMMN4MA", false)
statuses, err := suite.timeline.GetXBeforeID(context.Background(), 3, "01F8MHBQCBTDKN6X5VHGMMN4MA", false)
if err != nil {
suite.FailNow(err.Error())
}

31
internal/timeline/index.go
View file

@ -20,6 +20,7 @@ package timeline
import (
"container/list"
"context"
"errors"
"fmt"
"time"
@ -29,7 +30,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (t *timeline) IndexBefore(statusID string, include bool, amount int) error {
func (t *timeline) IndexBefore(ctx context.Context, statusID string, include bool, amount int) error {
// lazily initialize index if it hasn't been done already
if t.postIndex.data == nil {
t.postIndex.data = &list.List{}
@ -42,7 +43,7 @@ func (t *timeline) IndexBefore(statusID string, include bool, amount int) error
if include {
// if we have the status with given statusID in the database, include it in the results set as well
s := &gtsmodel.Status{}
if err := t.db.GetByID(statusID, s); err == nil {
if err := t.db.GetByID(ctx, statusID, s); err == nil {
filtered = append(filtered, s)
}
}
@ -50,7 +51,7 @@ func (t *timeline) IndexBefore(statusID string, include bool, amount int) error
i := 0
grabloop:
for ; len(filtered) < amount && i < 5; i = i + 1 { // try the grabloop 5 times only
statuses, err := t.db.GetHomeTimeline(t.accountID, "", "", offsetStatus, amount, false)
statuses, err := t.db.GetHomeTimeline(ctx, t.accountID, "", "", offsetStatus, amount, false)
if err != nil {
if err == db.ErrNoEntries {
break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
@ -59,7 +60,7 @@ grabloop:
}
for _, s := range statuses {
timelineable, err := t.filter.StatusHometimelineable(s, t.account)
timelineable, err := t.filter.StatusHometimelineable(ctx, s, t.account)
if err != nil {
continue
}
@ -71,7 +72,7 @@ grabloop:
}
for _, s := range filtered {
if _, err := t.IndexOne(s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
if _, err := t.IndexOne(ctx, s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
return fmt.Errorf("IndexBefore: error indexing status with id %s: %s", s.ID, err)
}
}
@ -79,7 +80,7 @@ grabloop:
return nil
}
func (t *timeline) IndexBehind(statusID string, include bool, amount int) error {
func (t *timeline) IndexBehind(ctx context.Context, statusID string, include bool, amount int) error {
l := t.log.WithFields(logrus.Fields{
"func": "IndexBehind",
"include": include,
@ -121,7 +122,7 @@ positionLoop:
if include {
// if we have the status with given statusID in the database, include it in the results set as well
s := &gtsmodel.Status{}
if err := t.db.GetByID(statusID, s); err == nil {
if err := t.db.GetByID(ctx, statusID, s); err == nil {
filtered = append(filtered, s)
}
}
@ -130,7 +131,7 @@ positionLoop:
grabloop:
for ; len(filtered) < amount && i < 5; i = i + 1 { // try the grabloop 5 times only
l.Tracef("entering grabloop; i is %d; len(filtered) is %d", i, len(filtered))
statuses, err := t.db.GetHomeTimeline(t.accountID, offsetStatus, "", "", amount, false)
statuses, err := t.db.GetHomeTimeline(ctx, t.accountID, offsetStatus, "", "", amount, false)
if err != nil {
if err == db.ErrNoEntries {
break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
@ -140,7 +141,7 @@ grabloop:
l.Tracef("got %d statuses", len(statuses))
for _, s := range statuses {
timelineable, err := t.filter.StatusHometimelineable(s, t.account)
timelineable, err := t.filter.StatusHometimelineable(ctx, s, t.account)
if err != nil {
l.Tracef("status was not hometimelineable: %s", err)
continue
@ -154,7 +155,7 @@ grabloop:
l.Trace("left grabloop")
for _, s := range filtered {
if _, err := t.IndexOne(s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
if _, err := t.IndexOne(ctx, s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
return fmt.Errorf("IndexBehind: error indexing status with id %s: %s", s.ID, err)
}
}
@ -163,7 +164,7 @@ grabloop:
return nil
}
func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
func (t *timeline) IndexOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
t.Lock()
defer t.Unlock()
@ -177,7 +178,7 @@ func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string, boostOfI
return t.postIndex.insertIndexed(postIndexEntry)
}
func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
t.Lock()
defer t.Unlock()
@ -194,7 +195,7 @@ func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string
}
if inserted {
if err := t.prepare(statusID); err != nil {
if err := t.prepare(ctx, statusID); err != nil {
return inserted, fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err)
}
}
@ -202,7 +203,7 @@ func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string
return inserted, nil
}
func (t *timeline) OldestIndexedPostID() (string, error) {
func (t *timeline) OldestIndexedPostID(ctx context.Context) (string, error) {
var id string
if t.postIndex == nil || t.postIndex.data == nil || t.postIndex.data.Back() == nil {
// return an empty string if postindex hasn't been initialized yet
@ -217,7 +218,7 @@ func (t *timeline) OldestIndexedPostID() (string, error) {
return entry.statusID, nil
}
func (t *timeline) NewestIndexedPostID() (string, error) {
func (t *timeline) NewestIndexedPostID(ctx context.Context) (string, error) {
var id string
if t.postIndex == nil || t.postIndex.data == nil || t.postIndex.data.Front() == nil {
// return an empty string if postindex hasn't been initialized yet

47
internal/timeline/index_test.go
View file

@ -19,6 +19,7 @@
package timeline_test
import (
"context"
"testing"
"time"
@ -46,7 +47,7 @@ func (suite *IndexTestSuite) SetupTest() {
testrig.StandardDBSetup(suite.db, nil)
// let's take local_account_1 as the timeline owner, and start with an empty timeline
tl, err := timeline.NewTimeline(suite.testAccounts["local_account_1"].ID, suite.db, suite.tc, suite.log)
tl, err := timeline.NewTimeline(context.Background(), suite.testAccounts["local_account_1"].ID, suite.db, suite.tc, suite.log)
if err != nil {
suite.FailNow(err.Error())
}
@ -59,82 +60,82 @@ func (suite *IndexTestSuite) TearDownTest() {
func (suite *IndexTestSuite) TestIndexBeforeLowID() {
// index 10 before the lowest status ID possible
err := suite.timeline.IndexBefore("00000000000000000000000000", true, 10)
err := suite.timeline.IndexBefore(context.Background(), "00000000000000000000000000", true, 10)
suite.NoError(err)
// the oldest indexed post should be the lowest one we have in our testrig
postID, err := suite.timeline.OldestIndexedPostID()
postID, err := suite.timeline.OldestIndexedPostID(context.Background())
suite.NoError(err)
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID)
indexLength := suite.timeline.PostIndexLength()
indexLength := suite.timeline.PostIndexLength(context.Background())
suite.Equal(10, indexLength)
}
func (suite *IndexTestSuite) TestIndexBeforeHighID() {
// index 10 before the highest status ID possible
err := suite.timeline.IndexBefore("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
err := suite.timeline.IndexBefore(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
suite.NoError(err)
// the oldest indexed post should be empty
postID, err := suite.timeline.OldestIndexedPostID()
postID, err := suite.timeline.OldestIndexedPostID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength()
indexLength := suite.timeline.PostIndexLength(context.Background())
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestIndexBehindHighID() {
// index 10 behind the highest status ID possible
err := suite.timeline.IndexBehind("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
err := suite.timeline.IndexBehind(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
suite.NoError(err)
// the newest indexed post should be the highest one we have in our testrig
postID, err := suite.timeline.NewestIndexedPostID()
postID, err := suite.timeline.NewestIndexedPostID(context.Background())
suite.NoError(err)
suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID)
// indexLength should be 10 because that's all this user has hometimelineable
indexLength := suite.timeline.PostIndexLength()
indexLength := suite.timeline.PostIndexLength(context.Background())
suite.Equal(10, indexLength)
}
func (suite *IndexTestSuite) TestIndexBehindLowID() {
// index 10 behind the lowest status ID possible
err := suite.timeline.IndexBehind("00000000000000000000000000", true, 10)
err := suite.timeline.IndexBehind(context.Background(), "00000000000000000000000000", true, 10)
suite.NoError(err)
// the newest indexed post should be empty
postID, err := suite.timeline.NewestIndexedPostID()
postID, err := suite.timeline.NewestIndexedPostID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength()
indexLength := suite.timeline.PostIndexLength(context.Background())
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestOldestIndexedPostIDEmpty() {
// the oldest indexed post should be an empty string since there's nothing indexed yet
postID, err := suite.timeline.OldestIndexedPostID()
postID, err := suite.timeline.OldestIndexedPostID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength()
indexLength := suite.timeline.PostIndexLength(context.Background())
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestNewestIndexedPostIDEmpty() {
// the newest indexed post should be an empty string since there's nothing indexed yet
postID, err := suite.timeline.NewestIndexedPostID()
postID, err := suite.timeline.NewestIndexedPostID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength()
indexLength := suite.timeline.PostIndexLength(context.Background())
suite.Equal(0, indexLength)
}
@ -142,12 +143,12 @@ func (suite *IndexTestSuite) TestIndexAlreadyIndexed() {
testStatus := suite.testStatuses["local_account_1_status_1"]
// index one post -- it should be indexed
indexed, err := suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index the same post again -- it should not be indexed
indexed, err = suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err = suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
@ -156,12 +157,12 @@ func (suite *IndexTestSuite) TestIndexAndPrepareAlreadyIndexedAndPrepared() {
testStatus := suite.testStatuses["local_account_1_status_1"]
// index and prepare one post -- it should be indexed
indexed, err := suite.timeline.IndexAndPrepareOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err := suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index and prepare the same post again -- it should not be indexed
indexed, err = suite.timeline.IndexAndPrepareOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err = suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
@ -177,12 +178,12 @@ func (suite *IndexTestSuite) TestIndexBoostOfAlreadyIndexed() {
}
// index one post -- it should be indexed
indexed, err := suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index the a boost of that post -- it should not be indexed
indexed, err = suite.timeline.IndexOne(boostOfTestStatus.CreatedAt, boostOfTestStatus.ID, boostOfTestStatus.BoostOfID, boostOfTestStatus.AccountID, boostOfTestStatus.BoostOfAccountID)
indexed, err = suite.timeline.IndexOne(context.Background(), boostOfTestStatus.CreatedAt, boostOfTestStatus.ID, boostOfTestStatus.BoostOfID, boostOfTestStatus.AccountID, boostOfTestStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}

79
internal/timeline/manager.go
View file

@ -19,6 +19,7 @@
package timeline
import (
"context"
"fmt"
"strings"
"sync"
@ -54,7 +55,7 @@ type Manager interface {
//
// The returned bool indicates whether the status was actually put in the timeline. This could be false in cases where
// the status is a boost, but a boost of the original post or the post itself already exists recently in the timeline.
Ingest(status *gtsmodel.Status, timelineAccountID string) (bool, error)
Ingest(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error)
// IngestAndPrepare takes one status and indexes it into the timeline for the given account ID, and then immediately prepares it for serving.
// This is useful in cases where we know the status will need to be shown at the top of a user's timeline immediately (eg., a new status is created).
//
@ -62,24 +63,24 @@ type Manager interface {
//
// The returned bool indicates whether the status was actually put in the timeline. This could be false in cases where
// the status is a boost, but a boost of the original post or the post itself already exists recently in the timeline.
IngestAndPrepare(status *gtsmodel.Status, timelineAccountID string) (bool, error)
IngestAndPrepare(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error)
// HomeTimeline returns limit n amount of entries from the home timeline of the given account ID, in descending chronological order.
// If maxID is provided, it will return entries from that maxID onwards, inclusive.
HomeTimeline(accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error)
HomeTimeline(ctx context.Context, accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error)
// GetIndexedLength returns the amount of posts/statuses that have been *indexed* for the given account ID.
GetIndexedLength(timelineAccountID string) int
GetIndexedLength(ctx context.Context, timelineAccountID string) int
// GetDesiredIndexLength returns the amount of posts that we, ideally, index for each user.
GetDesiredIndexLength() int
GetDesiredIndexLength(ctx context.Context) int
// GetOldestIndexedID returns the status ID for the oldest post that we have indexed for the given account.
GetOldestIndexedID(timelineAccountID string) (string, error)
GetOldestIndexedID(ctx context.Context, timelineAccountID string) (string, error)
// PrepareXFromTop prepares limit n amount of posts, based on their indexed representations, from the top of the index.
PrepareXFromTop(timelineAccountID string, limit int) error
PrepareXFromTop(ctx context.Context, timelineAccountID string, limit int) error
// Remove removes one status from the timeline of the given timelineAccountID
Remove(timelineAccountID string, statusID string) (int, error)
Remove(ctx context.Context, timelineAccountID string, statusID string) (int, error)
// WipeStatusFromAllTimelines removes one status from the index and prepared posts of all timelines
WipeStatusFromAllTimelines(statusID string) error
WipeStatusFromAllTimelines(ctx context.Context, statusID string) error
// WipeStatusesFromAccountID removes all statuses by the given accountID from the timelineAccountID's timelines.
WipeStatusesFromAccountID(timelineAccountID string, accountID string) error
WipeStatusesFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error
}
// NewManager returns a new timeline manager with the given database, typeconverter, config, and log.
@ -101,104 +102,104 @@ type manager struct {
log *logrus.Logger
}
func (m *manager) Ingest(status *gtsmodel.Status, timelineAccountID string) (bool, error) {
func (m *manager) Ingest(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error) {
l := m.log.WithFields(logrus.Fields{
"func": "Ingest",
"timelineAccountID": timelineAccountID,
"statusID": status.ID,
})
t, err := m.getOrCreateTimeline(timelineAccountID)
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return false, err
}
l.Trace("ingesting status")
return t.IndexOne(status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
return t.IndexOne(ctx, status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
}
func (m *manager) IngestAndPrepare(status *gtsmodel.Status, timelineAccountID string) (bool, error) {
func (m *manager) IngestAndPrepare(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error) {
l := m.log.WithFields(logrus.Fields{
"func": "IngestAndPrepare",
"timelineAccountID": timelineAccountID,
"statusID": status.ID,
})
t, err := m.getOrCreateTimeline(timelineAccountID)
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return false, err
}
l.Trace("ingesting status")
return t.IndexAndPrepareOne(status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
return t.IndexAndPrepareOne(ctx, status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
}
func (m *manager) Remove(timelineAccountID string, statusID string) (int, error) {
func (m *manager) Remove(ctx context.Context, timelineAccountID string, statusID string) (int, error) {
l := m.log.WithFields(logrus.Fields{
"func": "Remove",
"timelineAccountID": timelineAccountID,
"statusID": statusID,
})
t, err := m.getOrCreateTimeline(timelineAccountID)
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return 0, err
}
l.Trace("removing status")
return t.Remove(statusID)
return t.Remove(ctx, statusID)
}
func (m *manager) HomeTimeline(timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error) {
func (m *manager) HomeTimeline(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error) {
l := m.log.WithFields(logrus.Fields{
"func": "HomeTimelineGet",
"timelineAccountID": timelineAccountID,
})
t, err := m.getOrCreateTimeline(timelineAccountID)
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return nil, err
}
statuses, err := t.Get(limit, maxID, sinceID, minID, true)
statuses, err := t.Get(ctx, limit, maxID, sinceID, minID, true)
if err != nil {
l.Errorf("error getting statuses: %s", err)
}
return statuses, nil
}
func (m *manager) GetIndexedLength(timelineAccountID string) int {
t, err := m.getOrCreateTimeline(timelineAccountID)
func (m *manager) GetIndexedLength(ctx context.Context, timelineAccountID string) int {
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return 0
}
return t.PostIndexLength()
return t.PostIndexLength(ctx)
}
func (m *manager) GetDesiredIndexLength() int {
func (m *manager) GetDesiredIndexLength(ctx context.Context) int {
return desiredPostIndexLength
}
func (m *manager) GetOldestIndexedID(timelineAccountID string) (string, error) {
t, err := m.getOrCreateTimeline(timelineAccountID)
func (m *manager) GetOldestIndexedID(ctx context.Context, timelineAccountID string) (string, error) {
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return "", err
}
return t.OldestIndexedPostID()
return t.OldestIndexedPostID(ctx)
}
func (m *manager) PrepareXFromTop(timelineAccountID string, limit int) error {
t, err := m.getOrCreateTimeline(timelineAccountID)
func (m *manager) PrepareXFromTop(ctx context.Context, timelineAccountID string, limit int) error {
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return err
}
return t.PrepareFromTop(limit)
return t.PrepareFromTop(ctx, limit)
}
func (m *manager) WipeStatusFromAllTimelines(statusID string) error {
func (m *manager) WipeStatusFromAllTimelines(ctx context.Context, statusID string) error {
errors := []string{}
m.accountTimelines.Range(func(k interface{}, i interface{}) bool {
t, ok := i.(Timeline)
@ -206,7 +207,7 @@ func (m *manager) WipeStatusFromAllTimelines(statusID string) error {
panic("couldn't parse entry as Timeline, this should never happen so panic")
}
if _, err := t.Remove(statusID); err != nil {
if _, err := t.Remove(ctx, statusID); err != nil {
errors = append(errors, err.Error())
}
@ -221,22 +222,22 @@ func (m *manager) WipeStatusFromAllTimelines(statusID string) error {
return err
}
func (m *manager) WipeStatusesFromAccountID(timelineAccountID string, accountID string) error {
t, err := m.getOrCreateTimeline(timelineAccountID)
func (m *manager) WipeStatusesFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error {
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return err
}
_, err = t.RemoveAllBy(accountID)
_, err = t.RemoveAllBy(ctx, accountID)
return err
}
func (m *manager) getOrCreateTimeline(timelineAccountID string) (Timeline, error) {
func (m *manager) getOrCreateTimeline(ctx context.Context, timelineAccountID string) (Timeline, error) {
var t Timeline
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
var err error
t, err = NewTimeline(timelineAccountID, m.db, m.tc, m.log)
t, err = NewTimeline(ctx, timelineAccountID, m.db, m.tc, m.log)
if err != nil {
return nil, err
}

37
internal/timeline/manager_test.go
View file

@ -19,6 +19,7 @@
package timeline_test
import (
"context"
"testing"
"github.com/stretchr/testify/suite"
@ -54,85 +55,85 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
testAccount := suite.testAccounts["local_account_1"]
// should start at 0
indexedLen := suite.manager.GetIndexedLength(testAccount.ID)
indexedLen := suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(0, indexedLen)
// oldestIndexed should be empty string since there's nothing indexed
oldestIndexed, err := suite.manager.GetOldestIndexedID(testAccount.ID)
oldestIndexed, err := suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
suite.NoError(err)
suite.Empty(oldestIndexed)
// trigger status preparation
err = suite.manager.PrepareXFromTop(testAccount.ID, 20)
err = suite.manager.PrepareXFromTop(context.Background(), testAccount.ID, 20)
suite.NoError(err)
// local_account_1 can see 12 statuses out of the testrig statuses in its home timeline
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(12, indexedLen)
// oldest should now be set
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
suite.NoError(err)
suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", oldestIndexed)
// get hometimeline
statuses, err := suite.manager.HomeTimeline(testAccount.ID, "", "", "", 20, false)
statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false)
suite.NoError(err)
suite.Len(statuses, 12)
// now wipe the last status from all timelines, as though it had been deleted by the owner
err = suite.manager.WipeStatusFromAllTimelines("01F8MH75CBF9JFX4ZAD54N0W0R")
err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R")
suite.NoError(err)
// timeline should be shorter
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(11, indexedLen)
// oldest should now be different
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
suite.NoError(err)
suite.Equal("01F8MH82FYRXD2RC6108DAJ5HB", oldestIndexed)
// delete the new oldest status specifically from this timeline, as though local_account_1 had muted or blocked it
removed, err := suite.manager.Remove(testAccount.ID, "01F8MH82FYRXD2RC6108DAJ5HB")
removed, err := suite.manager.Remove(context.Background(), testAccount.ID, "01F8MH82FYRXD2RC6108DAJ5HB")
suite.NoError(err)
suite.Equal(2, removed) // 1 status should be removed, but from both indexed and prepared, so 2 removals total
// timeline should be shorter
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(10, indexedLen)
// oldest should now be different
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
suite.NoError(err)
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed)
// now remove all entries by local_account_2 from the timeline
err = suite.manager.WipeStatusesFromAccountID(testAccount.ID, suite.testAccounts["local_account_2"].ID)
err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID)
suite.NoError(err)
// timeline should be empty now
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(5, indexedLen)
// ingest 1 into the timeline
status1 := suite.testStatuses["admin_account_status_1"]
ingested, err := suite.manager.Ingest(status1, testAccount.ID)
ingested, err := suite.manager.Ingest(context.Background(), status1, testAccount.ID)
suite.NoError(err)
suite.True(ingested)
// ingest and prepare another one into the timeline
status2 := suite.testStatuses["local_account_2_status_1"]
ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID)
ingested, err = suite.manager.IngestAndPrepare(context.Background(), status2, testAccount.ID)
suite.NoError(err)
suite.True(ingested)
// timeline should be longer now
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
suite.Equal(7, indexedLen)
// try to ingest status 2 again
ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID)
ingested, err = suite.manager.IngestAndPrepare(context.Background(), status2, testAccount.ID)
suite.NoError(err)
suite.False(ingested) // should be false since it's a duplicate
}

35
internal/timeline/prepare.go
View file

@ -20,6 +20,7 @@ package timeline
import (
"container/list"
"context"
"errors"
"fmt"
@ -28,7 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, minID string) error {
func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID string, sinceID string, minID string) error {
l := t.log.WithFields(logrus.Fields{
"func": "prepareNextQuery",
"amount": amount,
@ -42,30 +43,30 @@ func (t *timeline) prepareNextQuery(amount int, maxID string, sinceID string, mi
// maxID is defined but sinceID isn't so take from behind
if maxID != "" && sinceID == "" {
l.Debug("preparing behind maxID")
err = t.PrepareBehind(maxID, amount)
err = t.PrepareBehind(ctx, maxID, amount)
}
// maxID isn't defined, but sinceID || minID are, so take x before
if maxID == "" && sinceID != "" {
l.Debug("preparing before sinceID")
err = t.PrepareBefore(sinceID, false, amount)
err = t.PrepareBefore(ctx, sinceID, false, amount)
}
if maxID == "" && minID != "" {
l.Debug("preparing before minID")
err = t.PrepareBefore(minID, false, amount)
err = t.PrepareBefore(ctx, minID, false, amount)
}
return err
}
func (t *timeline) PrepareBehind(statusID string, amount int) error {
func (t *timeline) PrepareBehind(ctx context.Context, statusID string, amount int) error {
// lazily initialize prepared posts if it hasn't been done already
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
t.preparedPosts.data.Init()
}
if err := t.IndexBehind(statusID, true, amount); err != nil {
if err := t.IndexBehind(ctx, statusID, true, amount); err != nil {
return fmt.Errorf("PrepareBehind: error indexing behind id %s: %s", statusID, err)
}
@ -93,7 +94,7 @@ prepareloop:
}
if preparing {
if err := t.prepare(entry.statusID); err != nil {
if err := t.prepare(ctx, entry.statusID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
@ -113,7 +114,7 @@ prepareloop:
return nil
}
func (t *timeline) PrepareBefore(statusID string, include bool, amount int) error {
func (t *timeline) PrepareBefore(ctx context.Context, statusID string, include bool, amount int) error {
t.Lock()
defer t.Unlock()
@ -148,7 +149,7 @@ prepareloop:
}
if preparing {
if err := t.prepare(entry.statusID); err != nil {
if err := t.prepare(ctx, entry.statusID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
@ -168,7 +169,7 @@ prepareloop:
return nil
}
func (t *timeline) PrepareFromTop(amount int) error {
func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
l := t.log.WithFields(logrus.Fields{
"func": "PrepareFromTop",
"amount": amount,
@ -183,7 +184,7 @@ func (t *timeline) PrepareFromTop(amount int) error {
// if the postindex is nil, nothing has been indexed yet so index from the highest ID possible
if t.postIndex.data == nil {
l.Debug("postindex.data was nil, indexing behind highest possible ID")
if err := t.IndexBehind("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", false, amount); err != nil {
if err := t.IndexBehind(ctx, "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", false, amount); err != nil {
return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", err)
}
}
@ -203,7 +204,7 @@ prepareloop:
return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
}
if err := t.prepare(entry.statusID); err != nil {
if err := t.prepare(ctx, entry.statusID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
@ -225,25 +226,25 @@ prepareloop:
return nil
}
func (t *timeline) prepare(statusID string) error {
func (t *timeline) prepare(ctx context.Context, statusID string) error {
// start by getting the status out of the database according to its indexed ID
gtsStatus := &gtsmodel.Status{}
if err := t.db.GetByID(statusID, gtsStatus); err != nil {
if err := t.db.GetByID(ctx, statusID, gtsStatus); err != nil {
return err
}
// if the account pointer hasn't been set on this timeline already, set it lazily here
if t.account == nil {
timelineOwnerAccount := &gtsmodel.Account{}
if err := t.db.GetByID(t.accountID, timelineOwnerAccount); err != nil {
if err := t.db.GetByID(ctx, t.accountID, timelineOwnerAccount); err != nil {
return err
}
t.account = timelineOwnerAccount
}
// serialize the status (or, at least, convert it to a form that's ready to be serialized)
apiModelStatus, err := t.tc.StatusToMasto(gtsStatus, t.account)
apiModelStatus, err := t.tc.StatusToMasto(ctx, gtsStatus, t.account)
if err != nil {
return err
}
@ -260,7 +261,7 @@ func (t *timeline) prepare(statusID string) error {
return t.preparedPosts.insertPrepared(preparedPostsEntry)
}
func (t *timeline) OldestPreparedPostID() (string, error) {
func (t *timeline) OldestPreparedPostID(ctx context.Context) (string, error) {
var id string
if t.preparedPosts == nil || t.preparedPosts.data == nil {
// return an empty string if prepared posts hasn't been initialized yet

5
internal/timeline/remove.go
View file

@ -20,12 +20,13 @@ package timeline
import (
"container/list"
"context"
"errors"
"github.com/sirupsen/logrus"
)
func (t *timeline) Remove(statusID string) (int, error) {
func (t *timeline) Remove(ctx context.Context, statusID string) (int, error) {
l := t.log.WithFields(logrus.Fields{
"func": "Remove",
"accountTimeline": t.accountID,
@ -77,7 +78,7 @@ func (t *timeline) Remove(statusID string) (int, error) {
return removed, nil
}
func (t *timeline) RemoveAllBy(accountID string) (int, error) {
func (t *timeline) RemoveAllBy(ctx context.Context, accountID string) (int, error) {
l := t.log.WithFields(logrus.Fields{
"func": "RemoveAllBy",
"accountTimeline": t.accountID,

41
internal/timeline/timeline.go
View file

@ -19,6 +19,7 @@
package timeline
import (
"context"
"sync"
"time"
@ -41,24 +42,24 @@ type Timeline interface {
// Get returns an amount of statuses with the given parameters.
// If prepareNext is true, then the next predicted query will be prepared already in a goroutine,
// to make the next call to Get faster.
Get(amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]*apimodel.Status, error)
Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]*apimodel.Status, error)
// GetXFromTop returns x amount of posts from the top of the timeline, from newest to oldest.
GetXFromTop(amount int) ([]*apimodel.Status, error)
GetXFromTop(ctx context.Context, amount int) ([]*apimodel.Status, error)
// GetXBehindID returns x amount of posts from the given id onwards, from newest to oldest.
// This will NOT include the status with the given ID.
//
// This corresponds to an api call to /timelines/home?max_id=WHATEVER
GetXBehindID(amount int, fromID string, attempts *int) ([]*apimodel.Status, error)
GetXBehindID(ctx context.Context, amount int, fromID string, attempts *int) ([]*apimodel.Status, error)
// GetXBeforeID returns x amount of posts up to the given id, from newest to oldest.
// This will NOT include the status with the given ID.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER
GetXBeforeID(amount int, sinceID string, startFromTop bool) ([]*apimodel.Status, error)
GetXBeforeID(ctx context.Context, amount int, sinceID string, startFromTop bool) ([]*apimodel.Status, error)
// GetXBetweenID returns x amount of posts from the given maxID, up to the given id, from newest to oldest.
// This will NOT include the status with the given IDs.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE
GetXBetweenID(amount int, maxID string, sinceID string) ([]*apimodel.Status, error)
GetXBetweenID(ctx context.Context, amount int, maxID string, sinceID string) ([]*apimodel.Status, error)
/*
INDEXING FUNCTIONS
@ -68,43 +69,43 @@ type Timeline interface {
//
// The returned bool indicates whether or not the status was actually inserted into the timeline. This will be false
// if the status is a boost and the original post or another boost of it already exists < boostReinsertionDepth back in the timeline.
IndexOne(statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
IndexOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// OldestIndexedPostID returns the id of the rearmost (ie., the oldest) indexed post, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
OldestIndexedPostID() (string, error)
OldestIndexedPostID(ctx context.Context) (string, error)
// NewestIndexedPostID returns the id of the frontmost (ie., the newest) indexed post, or an error if something goes wrong.
// If nothing goes wrong but there's no newest post, an empty string will be returned so make sure to check for this.
NewestIndexedPostID() (string, error)
NewestIndexedPostID(ctx context.Context) (string, error)
IndexBefore(statusID string, include bool, amount int) error
IndexBehind(statusID string, include bool, amount int) error
IndexBefore(ctx context.Context, statusID string, include bool, amount int) error
IndexBehind(ctx context.Context, statusID string, include bool, amount int) error
/*
PREPARATION FUNCTIONS
*/
// PrepareXFromTop instructs the timeline to prepare x amount of posts from the top of the timeline.
PrepareFromTop(amount int) error
PrepareFromTop(ctx context.Context, amount int) error
// PrepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
// If include is true, then the given status ID will also be prepared, otherwise only entries behind it will be prepared.
PrepareBehind(statusID string, amount int) error
PrepareBehind(ctx context.Context, statusID string, amount int) error
// IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property,
// and then immediately prepares it.
//
// The returned bool indicates whether or not the status was actually inserted into the timeline. This will be false
// if the status is a boost and the original post or another boost of it already exists < boostReinsertionDepth back in the timeline.
IndexAndPrepareOne(statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
IndexAndPrepareOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// OldestPreparedPostID returns the id of the rearmost (ie., the oldest) prepared post, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
OldestPreparedPostID() (string, error)
OldestPreparedPostID(ctx context.Context) (string, error)
/*
INFO FUNCTIONS
*/
// ActualPostIndexLength returns the actual length of the post index at this point in time.
PostIndexLength() int
PostIndexLength(ctx context.Context) int
/*
UTILITY FUNCTIONS
@ -117,11 +118,11 @@ type Timeline interface {
// If a status has multiple entries in a timeline, they will all be removed.
//
// The returned int indicates the amount of entries that were removed.
Remove(statusID string) (int, error)
Remove(ctx context.Context, statusID string) (int, error)
// RemoveAllBy removes all statuses by the given accountID, from both the index and prepared posts.
//
// The returned int indicates the amount of entries that were removed.
RemoveAllBy(accountID string) (int, error)
RemoveAllBy(ctx context.Context, accountID string) (int, error)
}
// timeline fulfils the Timeline interface
@ -138,9 +139,9 @@ type timeline struct {
}
// NewTimeline returns a new Timeline for the given account ID
func NewTimeline(accountID string, db db.DB, typeConverter typeutils.TypeConverter, log *logrus.Logger) (Timeline, error) {
func NewTimeline(ctx context.Context, accountID string, db db.DB, typeConverter typeutils.TypeConverter, log *logrus.Logger) (Timeline, error) {
timelineOwnerAccount := &gtsmodel.Account{}
if err := db.GetByID(accountID, timelineOwnerAccount); err != nil {
if err := db.GetByID(ctx, accountID, timelineOwnerAccount); err != nil {
return nil, err
}
@ -160,7 +161,7 @@ func (t *timeline) Reset() error {
return nil
}
func (t *timeline) PostIndexLength() int {
func (t *timeline) PostIndexLength(ctx context.Context) int {
if t.postIndex == nil || t.postIndex.data == nil {
return 0
}