diff --git a/internal/timeline/get.go b/internal/timeline/get.go index bf4c0dcd5..e8154797d 100644 --- a/internal/timeline/get.go +++ b/internal/timeline/get.go @@ -174,7 +174,7 @@ findMarkLoop: // we didn't find it, so we need to make sure it's indexed and prepared and then try again // this can happen when a user asks for really old posts if behindIDMark == nil { - if err := t.IndexBehind(behindID, amount); err != nil { + if err := t.IndexBehind(behindID, true, amount); err != nil { return nil, fmt.Errorf("GetXBehindID: error indexing behind and including ID %s", behindID) } if err := t.PrepareBehind(behindID, amount); err != nil { diff --git a/internal/timeline/get_test.go b/internal/timeline/get_test.go index a1cc4350c..0866f3bdd 100644 --- a/internal/timeline/get_test.go +++ b/internal/timeline/get_test.go @@ -66,13 +66,14 @@ func (suite *GetTestSuite) TearDownTest() { } func (suite *GetTestSuite) TestGetDefault() { - // get 10 from the top and don't prepare the next query - statuses, err := suite.timeline.Get(10, "", "", "", false) + // get 10 20 the top and don't prepare the next query + statuses, err := suite.timeline.Get(20, "", "", "", false) if err != nil { suite.FailNow(err.Error()) } - suite.Len(statuses, 10) + // we only have 12 statuses in the test suite + suite.Len(statuses, 12) // statuses should be sorted highest to lowest ID var highest string diff --git a/internal/timeline/index.go b/internal/timeline/index.go index c8894b284..b4c34051f 100644 --- a/internal/timeline/index.go +++ b/internal/timeline/index.go @@ -23,6 +23,7 @@ import ( "fmt" "time" + "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -32,17 +33,17 @@ func (t *timeline) IndexBefore(statusID string, include bool, amount int) error offsetStatus := statusID if include { + // if we have the status with given statusID in the database, include it in the results set as well s := >smodel.Status{} - if err := t.db.GetByID(statusID, s); err != nil { - return fmt.Errorf("IndexBefore: error getting initial status with id %s: %s", statusID, err) + if err := t.db.GetByID(statusID, s); err == nil { + filtered = append(filtered, s) } - filtered = append(filtered, s) } i := 0 grabloop: for ; len(filtered) < amount && i < 5; i = i + 1 { // try the grabloop 5 times only - statuses, err := t.db.GetHomeTimelineForAccount(t.accountID, "", offsetStatus, "", amount, false) + statuses, err := t.db.GetHomeTimelineForAccount(t.accountID, "", "", offsetStatus, amount, false) if err != nil { if _, ok := err.(db.ErrNoEntries); ok { break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail @@ -71,24 +72,41 @@ grabloop: return nil } -func (t *timeline) IndexBehind(statusID string, amount int) error { +func (t *timeline) IndexBehind(statusID string, include bool, amount int) error { + l := t.log.WithFields(logrus.Fields{ + "func": "IndexBehind", + "include": include, + "amount": amount, + }) + filtered := []*gtsmodel.Status{} offsetStatus := statusID + if include { + // if we have the status with given statusID in the database, include it in the results set as well + s := >smodel.Status{} + if err := t.db.GetByID(statusID, s); err == nil { + filtered = append(filtered, s) + } + } + i := 0 grabloop: for ; len(filtered) < amount && i < 5; i = i + 1 { // try the grabloop 5 times only + l.Tracef("entering grabloop; i is %d; len(filtered) is %d", i, len(filtered)) statuses, err := t.db.GetHomeTimelineForAccount(t.accountID, offsetStatus, "", "", amount, false) if err != nil { if _, ok := err.(db.ErrNoEntries); ok { break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail } - return fmt.Errorf("IndexBehindAndIncluding: error getting statuses from db: %s", err) + return fmt.Errorf("IndexBehind: error getting statuses from db: %s", err) } + l.Tracef("got %d statuses", len(statuses)) for _, s := range statuses { timelineable, err := t.filter.StatusHometimelineable(s, t.account) if err != nil { + l.Tracef("status was not hometimelineable: %s", err) continue } if timelineable { @@ -97,6 +115,7 @@ grabloop: offsetStatus = s.ID } } + l.Trace("left grabloop") for _, s := range filtered { if _, err := t.IndexOne(s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil { @@ -104,10 +123,7 @@ grabloop: } } - return nil -} - -func (t *timeline) IndexOneByID(statusID string) error { + l.Trace("exiting function") return nil } @@ -158,15 +174,24 @@ func (t *timeline) OldestIndexedPostID() (string, error) { } e := t.postIndex.data.Back() - - if e == nil { - // return an empty string if there's no back entry (ie., the index list hasn't been initialized yet) - return id, nil - } - entry, ok := e.Value.(*postIndexEntry) if !ok { return id, errors.New("OldestIndexedPostID: could not parse e as a postIndexEntry") } return entry.statusID, nil } + +func (t *timeline) NewestIndexedPostID() (string, error) { + var id string + if t.postIndex == nil || t.postIndex.data == nil { + // return an empty string if postindex hasn't been initialized yet + return id, nil + } + + e := t.postIndex.data.Front() + entry, ok := e.Value.(*postIndexEntry) + if !ok { + return id, errors.New("NewestIndexedPostID: could not parse e as a postIndexEntry") + } + return entry.statusID, nil +} diff --git a/internal/timeline/index_test.go b/internal/timeline/index_test.go new file mode 100644 index 000000000..e10337013 --- /dev/null +++ b/internal/timeline/index_test.go @@ -0,0 +1,193 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package timeline_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/timeline" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type IndexTestSuite struct { + TimelineStandardTestSuite +} + +func (suite *IndexTestSuite) SetupSuite() { + suite.testAccounts = testrig.NewTestAccounts() + suite.testStatuses = testrig.NewTestStatuses() +} + +func (suite *IndexTestSuite) SetupTest() { + suite.config = testrig.NewTestConfig() + suite.db = testrig.NewTestDB() + suite.log = testrig.NewTestLog() + suite.tc = testrig.NewTestTypeConverter(suite.db) + + testrig.StandardDBSetup(suite.db, nil) + + // let's take local_account_1 as the timeline owner, and start with an empty timeline + tl, err := timeline.NewTimeline(suite.testAccounts["local_account_1"].ID, suite.db, suite.tc, suite.log) + if err != nil { + suite.FailNow(err.Error()) + } + suite.timeline = tl +} + +func (suite *IndexTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) +} + +func (suite *IndexTestSuite) TestIndexBeforeLowID() { + // index 10 before the lowest status ID possible + err := suite.timeline.IndexBefore("00000000000000000000000000", true, 10) + suite.NoError(err) + + // the oldest indexed post should be the lowest one we have in our testrig + postID, err := suite.timeline.OldestIndexedPostID() + suite.NoError(err) + suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", postID) + + // indexLength should only be 6 because that's all this user has hometimelineable + indexLength := suite.timeline.PostIndexLength() + suite.Equal(6, indexLength) +} + +func (suite *IndexTestSuite) TestIndexBeforeHighID() { + // index 10 before the highest status ID possible + err := suite.timeline.IndexBefore("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10) + suite.NoError(err) + + // the oldest indexed post should be empty + postID, err := suite.timeline.OldestIndexedPostID() + suite.NoError(err) + suite.Empty(postID) + + // indexLength should be 0 + indexLength := suite.timeline.PostIndexLength() + suite.Equal(0, indexLength) +} + +func (suite *IndexTestSuite) TestIndexBehindHighID() { + // index 10 behind the highest status ID possible + err := suite.timeline.IndexBehind("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10) + suite.NoError(err) + + // the newest indexed post should be the highest one we have in our testrig + postID, err := suite.timeline.NewestIndexedPostID() + suite.NoError(err) + suite.Equal("01F8MHCP5P2NWYQ416SBA0XSEV", postID) + + // indexLength should only be 6 because that's all this user has hometimelineable + indexLength := suite.timeline.PostIndexLength() + suite.Equal(6, indexLength) +} + +func (suite *IndexTestSuite) TestIndexBehindLowID() { + // index 10 behind the lowest status ID possible + err := suite.timeline.IndexBehind("00000000000000000000000000", true, 10) + suite.NoError(err) + + // the newest indexed post should be empty + postID, err := suite.timeline.NewestIndexedPostID() + suite.NoError(err) + suite.Empty(postID) + + // indexLength should be 0 + indexLength := suite.timeline.PostIndexLength() + suite.Equal(0, indexLength) +} + +func (suite *IndexTestSuite) TestOldestIndexedPostIDEmpty() { + // the oldest indexed post should be an empty string since there's nothing indexed yet + postID, err := suite.timeline.OldestIndexedPostID() + suite.NoError(err) + suite.Empty(postID) + + // indexLength should be 0 + indexLength := suite.timeline.PostIndexLength() + suite.Equal(0, indexLength) +} + +func (suite *IndexTestSuite) TestNewestIndexedPostIDEmpty() { + // the newest indexed post should be an empty string since there's nothing indexed yet + postID, err := suite.timeline.NewestIndexedPostID() + suite.NoError(err) + suite.Empty(postID) + + // indexLength should be 0 + indexLength := suite.timeline.PostIndexLength() + suite.Equal(0, indexLength) +} + +func (suite *IndexTestSuite) TestIndexAlreadyIndexed() { + testStatus := suite.testStatuses["local_account_1_status_1"] + + // index one post -- it should be indexed + indexed, err := suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID) + suite.NoError(err) + suite.True(indexed) + + // try to index the same post again -- it should not be indexed + indexed, err = suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID) + suite.NoError(err) + suite.False(indexed) +} + +func (suite *IndexTestSuite) TestIndexAndPrepareAlreadyIndexedAndPrepared() { + testStatus := suite.testStatuses["local_account_1_status_1"] + + // index and prepare one post -- it should be indexed + indexed, err := suite.timeline.IndexAndPrepareOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID) + suite.NoError(err) + suite.True(indexed) + + // try to index and prepare the same post again -- it should not be indexed + indexed, err = suite.timeline.IndexAndPrepareOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID) + suite.NoError(err) + suite.False(indexed) +} + +func (suite *IndexTestSuite) TestIndexBoostOfAlreadyIndexed() { + testStatus := suite.testStatuses["local_account_1_status_1"] + boostOfTestStatus := >smodel.Status{ + CreatedAt: time.Now(), + ID: "01FD4TA6G2Z6M7W8NJQ3K5WXYD", + BoostOfID: testStatus.ID, + AccountID: "01FD4TAY1C0NGEJVE9CCCX7QKS", + BoostOfAccountID: testStatus.AccountID, + } + + // index one post -- it should be indexed + indexed, err := suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID) + suite.NoError(err) + suite.True(indexed) + + // try to index the a boost of that post -- it should not be indexed + indexed, err = suite.timeline.IndexOne(boostOfTestStatus.CreatedAt, boostOfTestStatus.ID, boostOfTestStatus.BoostOfID, boostOfTestStatus.AccountID, boostOfTestStatus.BoostOfAccountID) + suite.NoError(err) + suite.False(indexed) +} + +func TestIndexTestSuite(t *testing.T) { + suite.Run(t, new(IndexTestSuite)) +} diff --git a/internal/timeline/manager.go b/internal/timeline/manager.go index e2a98f623..a592670a8 100644 --- a/internal/timeline/manager.go +++ b/internal/timeline/manager.go @@ -75,11 +75,11 @@ type Manager interface { // PrepareXFromTop prepares limit n amount of posts, based on their indexed representations, from the top of the index. PrepareXFromTop(timelineAccountID string, limit int) error // Remove removes one status from the timeline of the given timelineAccountID - Remove(statusID string, timelineAccountID string) (int, error) + Remove(timelineAccountID string, statusID string) (int, error) // WipeStatusFromAllTimelines removes one status from the index and prepared posts of all timelines WipeStatusFromAllTimelines(statusID string) error // WipeStatusesFromAccountID removes all statuses by the given accountID from the timelineAccountID's timelines. - WipeStatusesFromAccountID(accountID string, timelineAccountID string) error + WipeStatusesFromAccountID(timelineAccountID string, accountID string) error } // NewManager returns a new timeline manager with the given database, typeconverter, config, and log. @@ -133,7 +133,7 @@ func (m *manager) IngestAndPrepare(status *gtsmodel.Status, timelineAccountID st return t.IndexAndPrepareOne(status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID) } -func (m *manager) Remove(statusID string, timelineAccountID string) (int, error) { +func (m *manager) Remove(timelineAccountID string, statusID string) (int, error) { l := m.log.WithFields(logrus.Fields{ "func": "Remove", "timelineAccountID": timelineAccountID, @@ -221,7 +221,7 @@ func (m *manager) WipeStatusFromAllTimelines(statusID string) error { return err } -func (m *manager) WipeStatusesFromAccountID(accountID string, timelineAccountID string) error { +func (m *manager) WipeStatusesFromAccountID(timelineAccountID string, accountID string) error { t, err := m.getOrCreateTimeline(timelineAccountID) if err != nil { return err diff --git a/internal/timeline/manager_test.go b/internal/timeline/manager_test.go new file mode 100644 index 000000000..20f295dd2 --- /dev/null +++ b/internal/timeline/manager_test.go @@ -0,0 +1,142 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package timeline_test + +import ( + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type ManagerTestSuite struct { + TimelineStandardTestSuite +} + +func (suite *ManagerTestSuite) SetupSuite() { + suite.testAccounts = testrig.NewTestAccounts() + suite.testStatuses = testrig.NewTestStatuses() +} + +func (suite *ManagerTestSuite) SetupTest() { + suite.config = testrig.NewTestConfig() + suite.db = testrig.NewTestDB() + suite.log = testrig.NewTestLog() + suite.tc = testrig.NewTestTypeConverter(suite.db) + + testrig.StandardDBSetup(suite.db, nil) + + manager := testrig.NewTestTimelineManager(suite.db) + suite.manager = manager +} + +func (suite *ManagerTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) +} + +func (suite *ManagerTestSuite) TestManagerIntegration() { + testAccount := suite.testAccounts["local_account_1"] + + // should start at 0 + indexedLen := suite.manager.GetIndexedLength(testAccount.ID) + suite.Equal(0, indexedLen) + + // oldestIndexed should be empty string since there's nothing indexed + oldestIndexed, err := suite.manager.GetOldestIndexedID(testAccount.ID) + suite.NoError(err) + suite.Empty(oldestIndexed) + + // trigger status preparation + err = suite.manager.PrepareXFromTop(testAccount.ID, 20) + suite.NoError(err) + + // local_account_1 can see 6 statuses out of the testrig statuses in its home timeline + indexedLen = suite.manager.GetIndexedLength(testAccount.ID) + suite.Equal(6, indexedLen) + + // oldest should now be set + oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID) + suite.NoError(err) + suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", oldestIndexed) + + // get hometimeline + statuses, err := suite.manager.HomeTimeline(testAccount.ID, "", "", "", 20, false) + suite.NoError(err) + suite.Len(statuses, 6) + + // now wipe the last status from all timelines, as though it had been deleted by the owner + err = suite.manager.WipeStatusFromAllTimelines("01F8MH75CBF9JFX4ZAD54N0W0R") + suite.NoError(err) + + // timeline should be shorter + indexedLen = suite.manager.GetIndexedLength(testAccount.ID) + suite.Equal(5, indexedLen) + + // oldest should now be different + oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID) + suite.NoError(err) + suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed) + + // delete the new oldest status specifically from this timeline, as though local_account_1 had muted or blocked it + removed, err := suite.manager.Remove(testAccount.ID, "01F8MHAAY43M6RJ473VQFCVH37") + suite.NoError(err) + suite.Equal(2, removed) // 1 status should be removed, but from both indexed and prepared, so 2 removals total + + // timeline should be shorter + indexedLen = suite.manager.GetIndexedLength(testAccount.ID) + suite.Equal(4, indexedLen) + + // oldest should now be different + oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID) + suite.NoError(err) + suite.Equal("01F8MHBQCBTDKN6X5VHGMMN4MA", oldestIndexed) + + // now remove all entries by local_account_2 from the timeline + err = suite.manager.WipeStatusesFromAccountID(testAccount.ID, suite.testAccounts["local_account_2"].ID) + suite.NoError(err) + + // timeline should be empty now + indexedLen = suite.manager.GetIndexedLength(testAccount.ID) + suite.Equal(0, indexedLen) + + // ingest 1 into the timeline + status1 := suite.testStatuses["admin_account_status_1"] + ingested, err := suite.manager.Ingest(status1, testAccount.ID) + suite.NoError(err) + suite.True(ingested) + + // ingest and prepare another one into the timeline + status2 := suite.testStatuses["admin_account_status_2"] + ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID) + suite.NoError(err) + suite.True(ingested) + + // timeline should be length 2 now + indexedLen = suite.manager.GetIndexedLength(testAccount.ID) + suite.Equal(2, indexedLen) + + // try to ingest status 2 again + ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID) + suite.NoError(err) + suite.False(ingested) // should be false since it's a duplicate +} + +func TestManagerTestSuite(t *testing.T) { + suite.Run(t, new(ManagerTestSuite)) +} diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go index 0fbd8ebba..06c1f98ec 100644 --- a/internal/timeline/prepare.go +++ b/internal/timeline/prepare.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" + "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -154,8 +155,10 @@ prepareloop: } func (t *timeline) PrepareFromTop(amount int) error { - t.Lock() - defer t.Unlock() + l := t.log.WithFields(logrus.Fields{ + "func": "PrepareFromTop", + "amount": amount, + }) // lazily initialize prepared posts if it hasn't been done already if t.preparedPosts.data == nil { @@ -163,11 +166,17 @@ func (t *timeline) PrepareFromTop(amount int) error { t.preparedPosts.data.Init() } - // if the postindex is nil, nothing has been indexed yet so there's nothing to prepare + // if the postindex is nil, nothing has been indexed yet so index from the highest ID possible if t.postIndex.data == nil { - return nil + l.Debug("postindex.data was nil, indexing behind highest possible ID") + if err := t.IndexBehind("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", false, amount); err != nil { + return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", err) + } } + l.Trace("entering prepareloop") + t.Lock() + defer t.Unlock() var prepared int prepareloop: for e := t.postIndex.data.Front(); e != nil; e = e.Next() { @@ -193,10 +202,12 @@ prepareloop: prepared = prepared + 1 if prepared == amount { // we're done + l.Trace("leaving prepareloop") break prepareloop } } + l.Trace("leaving function") return nil } diff --git a/internal/timeline/timeline.go b/internal/timeline/timeline.go index 9ef32401a..6274a86ac 100644 --- a/internal/timeline/timeline.go +++ b/internal/timeline/timeline.go @@ -73,6 +73,12 @@ type Timeline interface { // OldestIndexedPostID returns the id of the rearmost (ie., the oldest) indexed post, or an error if something goes wrong. // If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this. OldestIndexedPostID() (string, error) + // NewestIndexedPostID returns the id of the frontmost (ie., the newest) indexed post, or an error if something goes wrong. + // If nothing goes wrong but there's no newest post, an empty string will be returned so make sure to check for this. + NewestIndexedPostID() (string, error) + + IndexBefore(statusID string, include bool, amount int) error + IndexBehind(statusID string, include bool, amount int) error /* PREPARATION FUNCTIONS diff --git a/internal/timeline/timeline_test.go b/internal/timeline/timeline_test.go index 791a550eb..4f997cb1e 100644 --- a/internal/timeline/timeline_test.go +++ b/internal/timeline/timeline_test.go @@ -39,4 +39,5 @@ type TimelineStandardTestSuite struct { testStatuses map[string]*gtsmodel.Status timeline timeline.Timeline + manager timeline.Manager }