From 2b633bc9ff9af1c5406f3ee575ab52d12261b9a5 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sun, 15 Aug 2021 16:16:26 +0200 Subject: [PATCH] update some sturf --- internal/db/pg/timeline.go | 21 +++++- internal/processing/processor.go | 2 +- internal/processing/timeline.go | 102 ------------------------------ internal/timeline/index_test.go | 12 ++-- internal/timeline/manager_test.go | 24 +++---- 5 files changed, 37 insertions(+), 124 deletions(-) diff --git a/internal/db/pg/timeline.go b/internal/db/pg/timeline.go index 95acd4f38..edecc60b2 100644 --- a/internal/db/pg/timeline.go +++ b/internal/db/pg/timeline.go @@ -28,31 +28,46 @@ import ( func (ps *postgresService) GetHomeTimelineForAccount(accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*gtsmodel.Status, error) { statuses := []*gtsmodel.Status{} - q := ps.conn.Model(&statuses) - + q = q.ColumnExpr("status.*"). + // Find out who accountID follows. Join("LEFT JOIN follows AS f ON f.target_account_id = status.account_id"). - Where("f.account_id = ?", accountID). + // Use a WhereGroup here to specify that we want EITHER statuses posted by accounts that accountID follows, + // OR statuses posted by accountID itself (since a user should be able to see their own statuses). + // + // This is equivalent to something like WHERE ... AND (... OR ...) + // See: https://pg.uptrace.dev/queries/#select + WhereGroup(func(q *pg.Query) (*pg.Query, error) { + q = q.WhereOr("f.account_id = ?", accountID). + WhereOr("status.account_id = ?", accountID) + return q, nil + }). + // Sort by highest ID (newest) to lowest ID (oldest) Order("status.id DESC") if maxID != "" { + // return only statuses LOWER (ie., older) than maxID q = q.Where("status.id < ?", maxID) } if sinceID != "" { + // return only statuses HIGHER (ie., newer) than sinceID q = q.Where("status.id > ?", sinceID) } if minID != "" { + // return only statuses HIGHER (ie., newer) than minID q = q.Where("status.id > ?", minID) } if local { + // return only statuses posted by local account havers q = q.Where("status.local = ?", local) } if limit > 0 { + // limit amount of statuses returned q = q.Limit(limit) } diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 16f9ac2a3..48ed2a35f 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -304,7 +304,7 @@ func (p *processor) Start() error { } } }() - return p.initTimelines() + return nil } // Stop stops the processor cleanly, finishing handling any remaining messages before closing down. diff --git a/internal/processing/timeline.go b/internal/processing/timeline.go index b5cbf433a..18d0a6ac7 100644 --- a/internal/processing/timeline.go +++ b/internal/processing/timeline.go @@ -21,9 +21,7 @@ package processing import ( "fmt" "net/url" - "sync" - "github.com/sirupsen/logrus" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtserror" @@ -186,103 +184,3 @@ func (p *processor) filterFavedStatuses(authed *oauth.Auth, statuses []*gtsmodel return apiStatuses, nil } - -func (p *processor) initTimelines() error { - // get all local accounts (ie., domain = nil) that aren't suspended (suspended_at = nil) - localAccounts := []*gtsmodel.Account{} - where := []db.Where{ - { - Key: "domain", Value: nil, - }, - { - Key: "suspended_at", Value: nil, - }, - } - if err := p.db.GetWhere(where, &localAccounts); err != nil { - if _, ok := err.(db.ErrNoEntries); ok { - return nil - } - return fmt.Errorf("initTimelines: db error initializing timelines: %s", err) - } - - // we want to wait until all timelines are populated so created a waitgroup here - wg := &sync.WaitGroup{} - wg.Add(len(localAccounts)) - - for _, localAccount := range localAccounts { - // to save time we can populate the timelines asynchronously - // this will go heavy on the database, but since we're not actually serving yet it doesn't really matter - go p.initTimelineFor(localAccount, wg) - } - - // wait for all timelines to be populated before we exit - wg.Wait() - return nil -} - -func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGroup) { - defer wg.Done() - - l := p.log.WithFields(logrus.Fields{ - "func": "initTimelineFor", - "accountID": account.ID, - }) - - desiredIndexLength := p.timelineManager.GetDesiredIndexLength() - - statuses, err := p.db.GetHomeTimelineForAccount(account.ID, "", "", "", desiredIndexLength, false) - if err != nil { - if _, ok := err.(db.ErrNoEntries); !ok { - l.Error(fmt.Errorf("initTimelineFor: error getting statuses: %s", err)) - } - return - } - p.indexAndIngest(statuses, account, desiredIndexLength) - - lengthNow := p.timelineManager.GetIndexedLength(account.ID) - if lengthNow < desiredIndexLength { - // try and get more posts from the last ID onwards - rearmostStatusID, err := p.timelineManager.GetOldestIndexedID(account.ID) - if err != nil { - l.Error(fmt.Errorf("initTimelineFor: error getting id of rearmost status: %s", err)) - return - } - - if rearmostStatusID != "" { - moreStatuses, err := p.db.GetHomeTimelineForAccount(account.ID, rearmostStatusID, "", "", desiredIndexLength/2, false) - if err != nil { - l.Error(fmt.Errorf("initTimelineFor: error getting more statuses: %s", err)) - return - } - p.indexAndIngest(moreStatuses, account, desiredIndexLength) - } - } - - l.Debugf("prepared timeline of length %d for account %s", lengthNow, account.ID) -} - -func (p *processor) indexAndIngest(statuses []*gtsmodel.Status, timelineAccount *gtsmodel.Account, desiredIndexLength int) { - l := p.log.WithFields(logrus.Fields{ - "func": "indexAndIngest", - "accountID": timelineAccount.ID, - }) - - for _, s := range statuses { - timelineable, err := p.filter.StatusHometimelineable(s, timelineAccount) - if err != nil { - l.Error(fmt.Errorf("initTimelineFor: error checking home timelineability of status %s: %s", s.ID, err)) - continue - } - if timelineable { - if _, err := p.timelineManager.Ingest(s, timelineAccount.ID); err != nil { - l.Error(fmt.Errorf("initTimelineFor: error ingesting status %s: %s", s.ID, err)) - continue - } - - // check if we have enough posts now and return if we do - if p.timelineManager.GetIndexedLength(timelineAccount.ID) >= desiredIndexLength { - return - } - } - } -} diff --git a/internal/timeline/index_test.go b/internal/timeline/index_test.go index e10337013..f48b2691c 100644 --- a/internal/timeline/index_test.go +++ b/internal/timeline/index_test.go @@ -65,11 +65,11 @@ func (suite *IndexTestSuite) TestIndexBeforeLowID() { // the oldest indexed post should be the lowest one we have in our testrig postID, err := suite.timeline.OldestIndexedPostID() suite.NoError(err) - suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", postID) + suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID) - // indexLength should only be 6 because that's all this user has hometimelineable + // indexLength should only be 9 because that's all this user has hometimelineable indexLength := suite.timeline.PostIndexLength() - suite.Equal(6, indexLength) + suite.Equal(9, indexLength) } func (suite *IndexTestSuite) TestIndexBeforeHighID() { @@ -95,11 +95,11 @@ func (suite *IndexTestSuite) TestIndexBehindHighID() { // the newest indexed post should be the highest one we have in our testrig postID, err := suite.timeline.NewestIndexedPostID() suite.NoError(err) - suite.Equal("01F8MHCP5P2NWYQ416SBA0XSEV", postID) + suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID) - // indexLength should only be 6 because that's all this user has hometimelineable + // indexLength should only be 11 because that's all this user has hometimelineable indexLength := suite.timeline.PostIndexLength() - suite.Equal(6, indexLength) + suite.Equal(11, indexLength) } func (suite *IndexTestSuite) TestIndexBehindLowID() { diff --git a/internal/timeline/manager_test.go b/internal/timeline/manager_test.go index 20f295dd2..9b975a5ce 100644 --- a/internal/timeline/manager_test.go +++ b/internal/timeline/manager_test.go @@ -66,9 +66,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { err = suite.manager.PrepareXFromTop(testAccount.ID, 20) suite.NoError(err) - // local_account_1 can see 6 statuses out of the testrig statuses in its home timeline + // local_account_1 can see 11 statuses out of the testrig statuses in its home timeline indexedLen = suite.manager.GetIndexedLength(testAccount.ID) - suite.Equal(6, indexedLen) + suite.Equal(11, indexedLen) // oldest should now be set oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID) @@ -78,7 +78,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // get hometimeline statuses, err := suite.manager.HomeTimeline(testAccount.ID, "", "", "", 20, false) suite.NoError(err) - suite.Len(statuses, 6) + suite.Len(statuses, 11) // now wipe the last status from all timelines, as though it had been deleted by the owner err = suite.manager.WipeStatusFromAllTimelines("01F8MH75CBF9JFX4ZAD54N0W0R") @@ -86,26 +86,26 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(testAccount.ID) - suite.Equal(5, indexedLen) + suite.Equal(10, indexedLen) // oldest should now be different oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID) suite.NoError(err) - suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed) + suite.Equal("01F8MH82FYRXD2RC6108DAJ5HB", oldestIndexed) // delete the new oldest status specifically from this timeline, as though local_account_1 had muted or blocked it - removed, err := suite.manager.Remove(testAccount.ID, "01F8MHAAY43M6RJ473VQFCVH37") + removed, err := suite.manager.Remove(testAccount.ID, "01F8MH82FYRXD2RC6108DAJ5HB") suite.NoError(err) suite.Equal(2, removed) // 1 status should be removed, but from both indexed and prepared, so 2 removals total // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(testAccount.ID) - suite.Equal(4, indexedLen) + suite.Equal(9, indexedLen) // oldest should now be different oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID) suite.NoError(err) - suite.Equal("01F8MHBQCBTDKN6X5VHGMMN4MA", oldestIndexed) + suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed) // now remove all entries by local_account_2 from the timeline err = suite.manager.WipeStatusesFromAccountID(testAccount.ID, suite.testAccounts["local_account_2"].ID) @@ -113,7 +113,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be empty now indexedLen = suite.manager.GetIndexedLength(testAccount.ID) - suite.Equal(0, indexedLen) + suite.Equal(5, indexedLen) // ingest 1 into the timeline status1 := suite.testStatuses["admin_account_status_1"] @@ -122,14 +122,14 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { suite.True(ingested) // ingest and prepare another one into the timeline - status2 := suite.testStatuses["admin_account_status_2"] + status2 := suite.testStatuses["local_account_2_status_1"] ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID) suite.NoError(err) suite.True(ingested) - // timeline should be length 2 now + // timeline should be longer now indexedLen = suite.manager.GetIndexedLength(testAccount.ID) - suite.Equal(2, indexedLen) + suite.Equal(7, indexedLen) // try to ingest status 2 again ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID)