more tests

This commit is contained in:
tsmethurst 2021-08-15 15:46:04 +02:00
commit 24c5ddcced
9 changed files with 407 additions and 28 deletions

View file

@ -174,7 +174,7 @@ findMarkLoop:
// we didn't find it, so we need to make sure it's indexed and prepared and then try again
// this can happen when a user asks for really old posts
if behindIDMark == nil {
if err := t.IndexBehind(behindID, amount); err != nil {
if err := t.IndexBehind(behindID, true, amount); err != nil {
return nil, fmt.Errorf("GetXBehindID: error indexing behind and including ID %s", behindID)
}
if err := t.PrepareBehind(behindID, amount); err != nil {

View file

@ -66,13 +66,14 @@ func (suite *GetTestSuite) TearDownTest() {
}
func (suite *GetTestSuite) TestGetDefault() {
// get 10 from the top and don't prepare the next query
statuses, err := suite.timeline.Get(10, "", "", "", false)
// get 10 20 the top and don't prepare the next query
statuses, err := suite.timeline.Get(20, "", "", "", false)
if err != nil {
suite.FailNow(err.Error())
}
suite.Len(statuses, 10)
// we only have 12 statuses in the test suite
suite.Len(statuses, 12)
// statuses should be sorted highest to lowest ID
var highest string

View file

@ -23,6 +23,7 @@ import (
"fmt"
"time"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
@ -32,17 +33,17 @@ func (t *timeline) IndexBefore(statusID string, include bool, amount int) error
offsetStatus := statusID
if include {
// if we have the status with given statusID in the database, include it in the results set as well
s := &gtsmodel.Status{}
if err := t.db.GetByID(statusID, s); err != nil {
return fmt.Errorf("IndexBefore: error getting initial status with id %s: %s", statusID, err)
if err := t.db.GetByID(statusID, s); err == nil {
filtered = append(filtered, s)
}
filtered = append(filtered, s)
}
i := 0
grabloop:
for ; len(filtered) < amount && i < 5; i = i + 1 { // try the grabloop 5 times only
statuses, err := t.db.GetHomeTimelineForAccount(t.accountID, "", offsetStatus, "", amount, false)
statuses, err := t.db.GetHomeTimelineForAccount(t.accountID, "", "", offsetStatus, amount, false)
if err != nil {
if _, ok := err.(db.ErrNoEntries); ok {
break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
@ -71,24 +72,41 @@ grabloop:
return nil
}
func (t *timeline) IndexBehind(statusID string, amount int) error {
func (t *timeline) IndexBehind(statusID string, include bool, amount int) error {
l := t.log.WithFields(logrus.Fields{
"func": "IndexBehind",
"include": include,
"amount": amount,
})
filtered := []*gtsmodel.Status{}
offsetStatus := statusID
if include {
// if we have the status with given statusID in the database, include it in the results set as well
s := &gtsmodel.Status{}
if err := t.db.GetByID(statusID, s); err == nil {
filtered = append(filtered, s)
}
}
i := 0
grabloop:
for ; len(filtered) < amount && i < 5; i = i + 1 { // try the grabloop 5 times only
l.Tracef("entering grabloop; i is %d; len(filtered) is %d", i, len(filtered))
statuses, err := t.db.GetHomeTimelineForAccount(t.accountID, offsetStatus, "", "", amount, false)
if err != nil {
if _, ok := err.(db.ErrNoEntries); ok {
break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
}
return fmt.Errorf("IndexBehindAndIncluding: error getting statuses from db: %s", err)
return fmt.Errorf("IndexBehind: error getting statuses from db: %s", err)
}
l.Tracef("got %d statuses", len(statuses))
for _, s := range statuses {
timelineable, err := t.filter.StatusHometimelineable(s, t.account)
if err != nil {
l.Tracef("status was not hometimelineable: %s", err)
continue
}
if timelineable {
@ -97,6 +115,7 @@ grabloop:
offsetStatus = s.ID
}
}
l.Trace("left grabloop")
for _, s := range filtered {
if _, err := t.IndexOne(s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
@ -104,10 +123,7 @@ grabloop:
}
}
return nil
}
func (t *timeline) IndexOneByID(statusID string) error {
l.Trace("exiting function")
return nil
}
@ -158,15 +174,24 @@ func (t *timeline) OldestIndexedPostID() (string, error) {
}
e := t.postIndex.data.Back()
if e == nil {
// return an empty string if there's no back entry (ie., the index list hasn't been initialized yet)
return id, nil
}
entry, ok := e.Value.(*postIndexEntry)
if !ok {
return id, errors.New("OldestIndexedPostID: could not parse e as a postIndexEntry")
}
return entry.statusID, nil
}
func (t *timeline) NewestIndexedPostID() (string, error) {
var id string
if t.postIndex == nil || t.postIndex.data == nil {
// return an empty string if postindex hasn't been initialized yet
return id, nil
}
e := t.postIndex.data.Front()
entry, ok := e.Value.(*postIndexEntry)
if !ok {
return id, errors.New("NewestIndexedPostID: could not parse e as a postIndexEntry")
}
return entry.statusID, nil
}

View file

@ -0,0 +1,193 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline_test
import (
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type IndexTestSuite struct {
TimelineStandardTestSuite
}
func (suite *IndexTestSuite) SetupSuite() {
suite.testAccounts = testrig.NewTestAccounts()
suite.testStatuses = testrig.NewTestStatuses()
}
func (suite *IndexTestSuite) SetupTest() {
suite.config = testrig.NewTestConfig()
suite.db = testrig.NewTestDB()
suite.log = testrig.NewTestLog()
suite.tc = testrig.NewTestTypeConverter(suite.db)
testrig.StandardDBSetup(suite.db, nil)
// let's take local_account_1 as the timeline owner, and start with an empty timeline
tl, err := timeline.NewTimeline(suite.testAccounts["local_account_1"].ID, suite.db, suite.tc, suite.log)
if err != nil {
suite.FailNow(err.Error())
}
suite.timeline = tl
}
func (suite *IndexTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
}
func (suite *IndexTestSuite) TestIndexBeforeLowID() {
// index 10 before the lowest status ID possible
err := suite.timeline.IndexBefore("00000000000000000000000000", true, 10)
suite.NoError(err)
// the oldest indexed post should be the lowest one we have in our testrig
postID, err := suite.timeline.OldestIndexedPostID()
suite.NoError(err)
suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", postID)
// indexLength should only be 6 because that's all this user has hometimelineable
indexLength := suite.timeline.PostIndexLength()
suite.Equal(6, indexLength)
}
func (suite *IndexTestSuite) TestIndexBeforeHighID() {
// index 10 before the highest status ID possible
err := suite.timeline.IndexBefore("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
suite.NoError(err)
// the oldest indexed post should be empty
postID, err := suite.timeline.OldestIndexedPostID()
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength()
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestIndexBehindHighID() {
// index 10 behind the highest status ID possible
err := suite.timeline.IndexBehind("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
suite.NoError(err)
// the newest indexed post should be the highest one we have in our testrig
postID, err := suite.timeline.NewestIndexedPostID()
suite.NoError(err)
suite.Equal("01F8MHCP5P2NWYQ416SBA0XSEV", postID)
// indexLength should only be 6 because that's all this user has hometimelineable
indexLength := suite.timeline.PostIndexLength()
suite.Equal(6, indexLength)
}
func (suite *IndexTestSuite) TestIndexBehindLowID() {
// index 10 behind the lowest status ID possible
err := suite.timeline.IndexBehind("00000000000000000000000000", true, 10)
suite.NoError(err)
// the newest indexed post should be empty
postID, err := suite.timeline.NewestIndexedPostID()
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength()
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestOldestIndexedPostIDEmpty() {
// the oldest indexed post should be an empty string since there's nothing indexed yet
postID, err := suite.timeline.OldestIndexedPostID()
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength()
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestNewestIndexedPostIDEmpty() {
// the newest indexed post should be an empty string since there's nothing indexed yet
postID, err := suite.timeline.NewestIndexedPostID()
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength()
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestIndexAlreadyIndexed() {
testStatus := suite.testStatuses["local_account_1_status_1"]
// index one post -- it should be indexed
indexed, err := suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index the same post again -- it should not be indexed
indexed, err = suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
func (suite *IndexTestSuite) TestIndexAndPrepareAlreadyIndexedAndPrepared() {
testStatus := suite.testStatuses["local_account_1_status_1"]
// index and prepare one post -- it should be indexed
indexed, err := suite.timeline.IndexAndPrepareOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index and prepare the same post again -- it should not be indexed
indexed, err = suite.timeline.IndexAndPrepareOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
func (suite *IndexTestSuite) TestIndexBoostOfAlreadyIndexed() {
testStatus := suite.testStatuses["local_account_1_status_1"]
boostOfTestStatus := &gtsmodel.Status{
CreatedAt: time.Now(),
ID: "01FD4TA6G2Z6M7W8NJQ3K5WXYD",
BoostOfID: testStatus.ID,
AccountID: "01FD4TAY1C0NGEJVE9CCCX7QKS",
BoostOfAccountID: testStatus.AccountID,
}
// index one post -- it should be indexed
indexed, err := suite.timeline.IndexOne(testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index the a boost of that post -- it should not be indexed
indexed, err = suite.timeline.IndexOne(boostOfTestStatus.CreatedAt, boostOfTestStatus.ID, boostOfTestStatus.BoostOfID, boostOfTestStatus.AccountID, boostOfTestStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
func TestIndexTestSuite(t *testing.T) {
suite.Run(t, new(IndexTestSuite))
}

View file

@ -75,11 +75,11 @@ type Manager interface {
// PrepareXFromTop prepares limit n amount of posts, based on their indexed representations, from the top of the index.
PrepareXFromTop(timelineAccountID string, limit int) error
// Remove removes one status from the timeline of the given timelineAccountID
Remove(statusID string, timelineAccountID string) (int, error)
Remove(timelineAccountID string, statusID string) (int, error)
// WipeStatusFromAllTimelines removes one status from the index and prepared posts of all timelines
WipeStatusFromAllTimelines(statusID string) error
// WipeStatusesFromAccountID removes all statuses by the given accountID from the timelineAccountID's timelines.
WipeStatusesFromAccountID(accountID string, timelineAccountID string) error
WipeStatusesFromAccountID(timelineAccountID string, accountID string) error
}
// NewManager returns a new timeline manager with the given database, typeconverter, config, and log.
@ -133,7 +133,7 @@ func (m *manager) IngestAndPrepare(status *gtsmodel.Status, timelineAccountID st
return t.IndexAndPrepareOne(status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
}
func (m *manager) Remove(statusID string, timelineAccountID string) (int, error) {
func (m *manager) Remove(timelineAccountID string, statusID string) (int, error) {
l := m.log.WithFields(logrus.Fields{
"func": "Remove",
"timelineAccountID": timelineAccountID,
@ -221,7 +221,7 @@ func (m *manager) WipeStatusFromAllTimelines(statusID string) error {
return err
}
func (m *manager) WipeStatusesFromAccountID(accountID string, timelineAccountID string) error {
func (m *manager) WipeStatusesFromAccountID(timelineAccountID string, accountID string) error {
t, err := m.getOrCreateTimeline(timelineAccountID)
if err != nil {
return err

View file

@ -0,0 +1,142 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline_test
import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type ManagerTestSuite struct {
TimelineStandardTestSuite
}
func (suite *ManagerTestSuite) SetupSuite() {
suite.testAccounts = testrig.NewTestAccounts()
suite.testStatuses = testrig.NewTestStatuses()
}
func (suite *ManagerTestSuite) SetupTest() {
suite.config = testrig.NewTestConfig()
suite.db = testrig.NewTestDB()
suite.log = testrig.NewTestLog()
suite.tc = testrig.NewTestTypeConverter(suite.db)
testrig.StandardDBSetup(suite.db, nil)
manager := testrig.NewTestTimelineManager(suite.db)
suite.manager = manager
}
func (suite *ManagerTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
}
func (suite *ManagerTestSuite) TestManagerIntegration() {
testAccount := suite.testAccounts["local_account_1"]
// should start at 0
indexedLen := suite.manager.GetIndexedLength(testAccount.ID)
suite.Equal(0, indexedLen)
// oldestIndexed should be empty string since there's nothing indexed
oldestIndexed, err := suite.manager.GetOldestIndexedID(testAccount.ID)
suite.NoError(err)
suite.Empty(oldestIndexed)
// trigger status preparation
err = suite.manager.PrepareXFromTop(testAccount.ID, 20)
suite.NoError(err)
// local_account_1 can see 6 statuses out of the testrig statuses in its home timeline
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
suite.Equal(6, indexedLen)
// oldest should now be set
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
suite.NoError(err)
suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", oldestIndexed)
// get hometimeline
statuses, err := suite.manager.HomeTimeline(testAccount.ID, "", "", "", 20, false)
suite.NoError(err)
suite.Len(statuses, 6)
// now wipe the last status from all timelines, as though it had been deleted by the owner
err = suite.manager.WipeStatusFromAllTimelines("01F8MH75CBF9JFX4ZAD54N0W0R")
suite.NoError(err)
// timeline should be shorter
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
suite.Equal(5, indexedLen)
// oldest should now be different
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
suite.NoError(err)
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed)
// delete the new oldest status specifically from this timeline, as though local_account_1 had muted or blocked it
removed, err := suite.manager.Remove(testAccount.ID, "01F8MHAAY43M6RJ473VQFCVH37")
suite.NoError(err)
suite.Equal(2, removed) // 1 status should be removed, but from both indexed and prepared, so 2 removals total
// timeline should be shorter
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
suite.Equal(4, indexedLen)
// oldest should now be different
oldestIndexed, err = suite.manager.GetOldestIndexedID(testAccount.ID)
suite.NoError(err)
suite.Equal("01F8MHBQCBTDKN6X5VHGMMN4MA", oldestIndexed)
// now remove all entries by local_account_2 from the timeline
err = suite.manager.WipeStatusesFromAccountID(testAccount.ID, suite.testAccounts["local_account_2"].ID)
suite.NoError(err)
// timeline should be empty now
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
suite.Equal(0, indexedLen)
// ingest 1 into the timeline
status1 := suite.testStatuses["admin_account_status_1"]
ingested, err := suite.manager.Ingest(status1, testAccount.ID)
suite.NoError(err)
suite.True(ingested)
// ingest and prepare another one into the timeline
status2 := suite.testStatuses["admin_account_status_2"]
ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID)
suite.NoError(err)
suite.True(ingested)
// timeline should be length 2 now
indexedLen = suite.manager.GetIndexedLength(testAccount.ID)
suite.Equal(2, indexedLen)
// try to ingest status 2 again
ingested, err = suite.manager.IngestAndPrepare(status2, testAccount.ID)
suite.NoError(err)
suite.False(ingested) // should be false since it's a duplicate
}
func TestManagerTestSuite(t *testing.T) {
suite.Run(t, new(ManagerTestSuite))
}

View file

@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
@ -154,8 +155,10 @@ prepareloop:
}
func (t *timeline) PrepareFromTop(amount int) error {
t.Lock()
defer t.Unlock()
l := t.log.WithFields(logrus.Fields{
"func": "PrepareFromTop",
"amount": amount,
})
// lazily initialize prepared posts if it hasn't been done already
if t.preparedPosts.data == nil {
@ -163,11 +166,17 @@ func (t *timeline) PrepareFromTop(amount int) error {
t.preparedPosts.data.Init()
}
// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
// if the postindex is nil, nothing has been indexed yet so index from the highest ID possible
if t.postIndex.data == nil {
return nil
l.Debug("postindex.data was nil, indexing behind highest possible ID")
if err := t.IndexBehind("ZZZZZZZZZZZZZZZZZZZZZZZZZZ", false, amount); err != nil {
return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", err)
}
}
l.Trace("entering prepareloop")
t.Lock()
defer t.Unlock()
var prepared int
prepareloop:
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
@ -193,10 +202,12 @@ prepareloop:
prepared = prepared + 1
if prepared == amount {
// we're done
l.Trace("leaving prepareloop")
break prepareloop
}
}
l.Trace("leaving function")
return nil
}

View file

@ -73,6 +73,12 @@ type Timeline interface {
// OldestIndexedPostID returns the id of the rearmost (ie., the oldest) indexed post, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
OldestIndexedPostID() (string, error)
// NewestIndexedPostID returns the id of the frontmost (ie., the newest) indexed post, or an error if something goes wrong.
// If nothing goes wrong but there's no newest post, an empty string will be returned so make sure to check for this.
NewestIndexedPostID() (string, error)
IndexBefore(statusID string, include bool, amount int) error
IndexBehind(statusID string, include bool, amount int) error
/*
PREPARATION FUNCTIONS

View file

@ -39,4 +39,5 @@ type TimelineStandardTestSuite struct {
testStatuses map[string]*gtsmodel.Status
timeline timeline.Timeline
manager timeline.Manager
}