mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-12-29 15:56:16 -06:00
update some sturf
This commit is contained in:
tsmethurst
parent
6d07b414c8
commit
2b633bc9ff
5 changed files with 37 additions and 124 deletions
|
|
@ -28,31 +28,46 @@ import (
|
|||
|
||||
func (ps *postgresService) GetHomeTimelineForAccount(accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*gtsmodel.Status, error) {
|
||||
statuses := []*gtsmodel.Status{}
|
||||
|
||||
q := ps.conn.Model(&statuses)
|
||||
|
||||
|
||||
q = q.ColumnExpr("status.*").
|
||||
// Find out who accountID follows.
|
||||
Join("LEFT JOIN follows AS f ON f.target_account_id = status.account_id").
|
||||
Where("f.account_id = ?", accountID).
|
||||
// Use a WhereGroup here to specify that we want EITHER statuses posted by accounts that accountID follows,
|
||||
// OR statuses posted by accountID itself (since a user should be able to see their own statuses).
|
||||
//
|
||||
// This is equivalent to something like WHERE ... AND (... OR ...)
|
||||
// See: https://pg.uptrace.dev/queries/#select
|
||||
WhereGroup(func(q *pg.Query) (*pg.Query, error) {
|
||||
q = q.WhereOr("f.account_id = ?", accountID).
|
||||
WhereOr("status.account_id = ?", accountID)
|
||||
return q, nil
|
||||
}).
|
||||
// Sort by highest ID (newest) to lowest ID (oldest)
|
||||
Order("status.id DESC")
|
||||
|
||||
if maxID != "" {
|
||||
// return only statuses LOWER (ie., older) than maxID
|
||||
q = q.Where("status.id < ?", maxID)
|
||||
}
|
||||
|
||||
if sinceID != "" {
|
||||
// return only statuses HIGHER (ie., newer) than sinceID
|
||||
q = q.Where("status.id > ?", sinceID)
|
||||
}
|
||||
|
||||
if minID != "" {
|
||||
// return only statuses HIGHER (ie., newer) than minID
|
||||
q = q.Where("status.id > ?", minID)
|
||||
}
|
||||
|
||||
if local {
|
||||
// return only statuses posted by local account havers
|
||||
q = q.Where("status.local = ?", local)
|
||||
}
|
||||
|
||||
if limit > 0 {
|
||||
// limit amount of statuses returned
|
||||
q = q.Limit(limit)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -304,7 +304,7 @@ func (p *processor) Start() error {
|
|||
}
|
||||
}
|
||||
}()
|
||||
return p.initTimelines()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops the processor cleanly, finishing handling any remaining messages before closing down.
|
||||
|
|
|
|||
|
|
@ -21,9 +21,7 @@ package processing
|
|||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
|
||||
|
|
@ -186,103 +184,3 @@ func (p *processor) filterFavedStatuses(authed *oauth.Auth, statuses []*gtsmodel
|
|||
|
||||
return apiStatuses, nil
|
||||
}
|
||||
|
||||
func (p *processor) initTimelines() error {
|
||||
// get all local accounts (ie., domain = nil) that aren't suspended (suspended_at = nil)
|
||||
localAccounts := []*gtsmodel.Account{}
|
||||
where := []db.Where{
|
||||
{
|
||||
Key: "domain", Value: nil,
|
||||
},
|
||||
{
|
||||
Key: "suspended_at", Value: nil,
|
||||
},
|
||||
}
|
||||
if err := p.db.GetWhere(where, &localAccounts); err != nil {
|
||||
if _, ok := err.(db.ErrNoEntries); ok {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("initTimelines: db error initializing timelines: %s", err)
|
||||
}
|
||||
|
||||
// we want to wait until all timelines are populated so created a waitgroup here
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(len(localAccounts))
|
||||
|
||||
for _, localAccount := range localAccounts {
|
||||
// to save time we can populate the timelines asynchronously
|
||||
// this will go heavy on the database, but since we're not actually serving yet it doesn't really matter
|
||||
go p.initTimelineFor(localAccount, wg)
|
||||
}
|
||||
|
||||
// wait for all timelines to be populated before we exit
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
|
||||
l := p.log.WithFields(logrus.Fields{
|
||||
"func": "initTimelineFor",
|
||||
"accountID": account.ID,
|
||||
})
|
||||
|
||||
desiredIndexLength := p.timelineManager.GetDesiredIndexLength()
|
||||
|
||||
statuses, err := p.db.GetHomeTimelineForAccount(account.ID, "", "", "", desiredIndexLength, false)
|
||||
if err != nil {
|
||||
if _, ok := err.(db.ErrNoEntries); !ok {
|
||||
l.Error(fmt.Errorf("initTimelineFor: error getting statuses: %s", err))
|
||||
}
|
||||
return
|
||||
}
|
||||
p.indexAndIngest(statuses, account, desiredIndexLength)
|
||||
|
||||
lengthNow := p.timelineManager.GetIndexedLength(account.ID)
|
||||
if lengthNow < desiredIndexLength {
|
||||
// try and get more posts from the last ID onwards
|
||||
rearmostStatusID, err := p.timelineManager.GetOldestIndexedID(account.ID)
|
||||
if err != nil {
|
||||
l.Error(fmt.Errorf("initTimelineFor: error getting id of rearmost status: %s", err))
|
||||
return
|
||||
}
|
||||
|
||||
if rearmostStatusID != "" {
|
||||
moreStatuses, err := p.db.GetHomeTimelineForAccount(account.ID, rearmostStatusID, "", "", desiredIndexLength/2, false)
|
||||
if err != nil {
|
||||
l.Error(fmt.Errorf("initTimelineFor: error getting more statuses: %s", err))
|
||||
return
|
||||
}
|
||||
p.indexAndIngest(moreStatuses, account, desiredIndexLength)
|
||||
}
|
||||
}
|
||||
|
||||
l.Debugf("prepared timeline of length %d for account %s", lengthNow, account.ID)
|
||||
}
|
||||
|
||||
func (p *processor) indexAndIngest(statuses []*gtsmodel.Status, timelineAccount *gtsmodel.Account, desiredIndexLength int) {
|
||||
l := p.log.WithFields(logrus.Fields{
|
||||
"func": "indexAndIngest",
|
||||
"accountID": timelineAccount.ID,
|
||||
})
|
||||
|
||||
for _, s := range statuses {
|
||||
timelineable, err := p.filter.StatusHometimelineable(s, timelineAccount)
|
||||
if err != nil {
|
||||
l.Error(fmt.Errorf("initTimelineFor: error checking home timelineability of status %s: %s", s.ID, err))
|
||||
continue
|
||||
}
|
||||
if timelineable {
|
||||
if _, err := p.timelineManager.Ingest(s, timelineAccount.ID); err != nil {
|
||||
l.Error(fmt.Errorf("initTimelineFor: error ingesting status %s: %s", s.ID, err))
|
||||
continue
|
||||
}
|
||||
|
||||
// check if we have enough posts now and return if we do
|
||||
if p.timelineManager.GetIndexedLength(timelineAccount.ID) >= desiredIndexLength {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,11 +65,11 @@ func (suite *IndexTestSuite) TestIndexBeforeLowID() {
|
|||
// the oldest indexed post should be the lowest one we have in our testrig
|
||||
postID, err := suite.timeline.OldestIndexedPostID()
|
||||
suite.NoError(err)
|
||||
suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", postID)
|
||||
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID)
|
||||
|
||||
// indexLength should only be 6 because that's all this user has hometimelineable
|
||||
// indexLength should only be 9 because that's all this user has hometimelineable
|
||||
indexLength := suite.timeline.PostIndexLength()
|
||||
suite.Equal(6, indexLength)
|
||||
suite.Equal(9, indexLength)
|
||||
}
|
||||
|
||||
func (suite *IndexTestSuite) TestIndexBeforeHighID() {
|
||||
|
|
@ -95,11 +95,11 @@ func (suite *IndexTestSuite) TestIndexBehindHighID() {
|
|||
// the newest indexed post should be the highest one we have in our testrig
|
||||
postID, err := suite.timeline.NewestIndexedPostID()
|
||||
suite.NoError(err)
|
||||
suite.Equal("01F8MHCP5P2NWYQ416SBA0XSEV", postID)
|
||||
suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID)
|
||||
|
||||
// indexLength should only be 6 because that's all this user has hometimelineable
|
||||
// indexLength should only be 11 because that's all this user has hometimelineable
|
||||
indexLength := suite.timeline.PostIndexLength()
|
||||
suite.Equal(6, indexLength)
|
||||
suite.Equal(11, indexLength)
|
||||
}
|
||||
|
||||
func (suite *IndexTestSuite) TestIndexBehindLowID() {
|
||||
|
|
|
|||
|
|
@ -66,9 +66,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
|
|||
err = suite.manager.PrepareXFromTop(testAccount.ID, 20)
|
||||
suite.NoError(err)
|
||||
|
||||
// local_account_1 can see 6 statuses out of the testrig statuses in its home timeline
|
||||
// local_account_1 can see 11 statuses out of the testrig statuses in its home timeline
|
||||
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
|
||||
suite.Equal(6, indexedLen)
|
||||
suite.Equal(11, indexedLen)
|
||||
|
||||
// oldest should now be set
|
||||
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
|
||||
|
|
@ -78,7 +78,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
|
|||
// get hometimeline
|
||||
statuses, err := suite.manager.HomeTimeline(testAccount.ID, "", "", "", 20, false)
|
||||
suite.NoError(err)
|
||||
suite.Len(statuses, 6)
|
||||
suite.Len(statuses, 11)
|
||||
|
||||
// now wipe the last status from all timelines, as though it had been deleted by the owner
|
||||
err = suite.manager.WipeStatusFromAllTimelines("01F8MH75CBF9JFX4ZAD54N0W0R")
|
||||
|
|
@ -86,26 +86,26 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
|
|||
|
||||
// timeline should be shorter
|
||||
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
|
||||
suite.Equal(5, indexedLen)
|
||||
suite.Equal(10, indexedLen)
|
||||
|
||||
// oldest should now be different
|
||||
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
|
||||
suite.NoError(err)
|
||||
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed)
|
||||
suite.Equal("01F8MH82FYRXD2RC6108DAJ5HB", oldestIndexed)
|
||||
|
||||
// delete the new oldest status specifically from this timeline, as though local_account_1 had muted or blocked it
|
||||
removed, err := suite.manager.Remove(testAccount.ID, "01F8MHAAY43M6RJ473VQFCVH37")
|
||||
removed, err := suite.manager.Remove(testAccount.ID, "01F8MH82FYRXD2RC6108DAJ5HB")
|
||||
suite.NoError(err)
|
||||
suite.Equal(2, removed) // 1 status should be removed, but from both indexed and prepared, so 2 removals total
|
||||
|
||||
// timeline should be shorter
|
||||
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
|
||||
suite.Equal(4, indexedLen)
|
||||
suite.Equal(9, indexedLen)
|
||||
|
||||
// oldest should now be different
|
||||
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
|
||||
suite.NoError(err)
|
||||
suite.Equal("01F8MHBQCBTDKN6X5VHGMMN4MA", oldestIndexed)
|
||||
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed)
|
||||
|
||||
// now remove all entries by local_account_2 from the timeline
|
||||
err = suite.manager.WipeStatusesFromAccountID(testAccount.ID, suite.testAccounts["local_account_2"].ID)
|
||||
|
|
@ -113,7 +113,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
|
|||
|
||||
// timeline should be empty now
|
||||
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
|
||||
suite.Equal(0, indexedLen)
|
||||
suite.Equal(5, indexedLen)
|
||||
|
||||
// ingest 1 into the timeline
|
||||
status1 := suite.testStatuses["admin_account_status_1"]
|
||||
|
|
@ -122,14 +122,14 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
|
|||
suite.True(ingested)
|
||||
|
||||
// ingest and prepare another one into the timeline
|
||||
status2 := suite.testStatuses["admin_account_status_2"]
|
||||
status2 := suite.testStatuses["local_account_2_status_1"]
|
||||
ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID)
|
||||
suite.NoError(err)
|
||||
suite.True(ingested)
|
||||
|
||||
// timeline should be length 2 now
|
||||
// timeline should be longer now
|
||||
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
|
||||
suite.Equal(2, indexedLen)
|
||||
suite.Equal(7, indexedLen)
|
||||
|
||||
// try to ingest status 2 again
|
||||
ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue