it's coming along

internal/processing/timeline.go

@@ -20,28 +20,26 @@ package processing
 
 import (
 	"fmt"
+	"sync"
 
+	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 )
 
-func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]apimodel.Status, ErrorWithCode) {
-	statuses, err := p.db.GetHomeTimelineForAccount(authed.Account.ID, maxID, sinceID, minID, limit, local)
+func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, ErrorWithCode) {
+
+	statuses, err := p.timelineManager.HomeTimeline(authed.Account.ID, maxID, sinceID, minID, limit, local)
 	if err != nil {
 		return nil, NewErrorInternalError(err)
 	}
 
-	s, err := p.filterStatuses(authed, statuses)
-	if err != nil {
-		return nil, NewErrorInternalError(err)
-	}
-
-	return s, nil
+	return statuses, nil
 }
 
-func (p *processor) PublicTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]apimodel.Status, ErrorWithCode) {
+func (p *processor) PublicTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, ErrorWithCode) {
 	statuses, err := p.db.GetPublicTimelineForAccount(authed.Account.ID, maxID, sinceID, minID, limit, local)
 	if err != nil {
 		return nil, NewErrorInternalError(err)
@@ -55,10 +53,10 @@ func (p *processor) PublicTimelineGet(authed *oauth.Auth, maxID string, sinceID
 	return s, nil
 }
 
-func (p *processor) filterStatuses(authed *oauth.Auth, statuses []*gtsmodel.Status) ([]apimodel.Status, error) {
+func (p *processor) filterStatuses(authed *oauth.Auth, statuses []*gtsmodel.Status) ([]*apimodel.Status, error) {
 	l := p.log.WithField("func", "filterStatuses")
 
-	apiStatuses := []apimodel.Status{}
+	apiStatuses := []*apimodel.Status{}
 	for _, s := range statuses {
 		targetAccount := &gtsmodel.Account{}
 		if err := p.db.GetByID(s.AccountID, targetAccount); err != nil {
@@ -115,8 +113,111 @@ func (p *processor) filterStatuses(authed *oauth.Auth, statuses []*gtsmodel.Stat
 			continue
 		}
 
-		apiStatuses = append(apiStatuses, *apiStatus)
+		apiStatuses = append(apiStatuses, apiStatus)
 	}
 
 	return apiStatuses, nil
 }
+
+func (p *processor) initTimelines() error {
+	// get all local accounts (ie., domain = nil) that aren't suspended (suspended_at = nil)
+	localAccounts := []*gtsmodel.Account{}
+	where := []db.Where{
+		{
+			Key: "domain", Value: nil,
+		},
+		{
+			Key: "suspended_at", Value: nil,
+		},
+	}
+	if err := p.db.GetWhere(where, &localAccounts); err != nil {
+		if _, ok := err.(db.ErrNoEntries); ok {
+			return nil
+		}
+		return fmt.Errorf("initTimelines: db error initializing timelines: %s", err)
+	}
+
+	// we want to wait until all timelines are populated so created a waitgroup here
+	wg := &sync.WaitGroup{}
+	wg.Add(len(localAccounts))
+
+	for _, localAccount := range localAccounts {
+		// to save time we can populate the timelines asynchronously
+		// this will go heavy on the database, but since we're not actually serving yet it doesn't really matter
+		go p.initTimelineFor(localAccount, wg)
+	}
+
+	// wait for all timelines to be populated before we exit
+	wg.Wait()
+	return nil
+}
+
+func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	l := p.log.WithFields(logrus.Fields{
+		"func":      "initTimelineFor",
+		"accountID": account.ID,
+	})
+
+	desiredIndexLength := p.timelineManager.GetDesiredIndexLength()
+
+	statuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength, "")
+	if err != nil {
+		l.Error(fmt.Errorf("initTimelineFor: error getting statuses: %s", err))
+		return
+	}
+	p.indexAndIngest(statuses, account, desiredIndexLength)
+
+	lengthNow := p.timelineManager.GetIndexedLength(account.ID)
+	if lengthNow < desiredIndexLength {
+		// try and get more posts from the last ID onwards
+		rearmostStatusID, err := p.timelineManager.GetOldestIndexedID(account.ID)
+		if err != nil {
+			l.Error(fmt.Errorf("initTimelineFor: error getting id of rearmost status: %s", err))
+			return
+		}
+
+		if rearmostStatusID != "" {
+			moreStatuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength / 2, rearmostStatusID)
+			if err != nil {
+				l.Error(fmt.Errorf("initTimelineFor: error getting more statuses: %s", err))
+				return
+			}
+			p.indexAndIngest(moreStatuses, account, desiredIndexLength)
+		}
+	}
+
+	l.Debugf("prepared timeline of length %d for account %s", lengthNow, account.ID)
+}
+
+func (p *processor) indexAndIngest(statuses []*gtsmodel.Status, timelineAccount *gtsmodel.Account, desiredIndexLength int) {
+	l := p.log.WithFields(logrus.Fields{
+		"func":      "indexAndIngest",
+		"accountID": timelineAccount.ID,
+	})
+
+	for _, s := range statuses {
+		relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(s)
+		if err != nil {
+			l.Error(fmt.Errorf("initTimelineFor: error getting relevant accounts from status %s: %s", s.ID, err))
+			continue
+		}
+		visible, err := p.db.StatusVisible(s, relevantAccounts.StatusAuthor, timelineAccount, relevantAccounts)
+		if err != nil {
+			l.Error(fmt.Errorf("initTimelineFor: error checking visibility of status %s: %s", s.ID, err))
+			continue
+		}
+		if visible {
+			if err := p.timelineManager.Ingest(s, timelineAccount.ID); err != nil {
+				l.Error(fmt.Errorf("initTimelineFor: error ingesting status %s: %s", s.ID, err))
+				continue
+			}
+
+			// check if we have enough posts now and return if we do
+			if p.timelineManager.GetIndexedLength(timelineAccount.ID) >= desiredIndexLength {
+				return
+			}
+		}
+	}
+}