mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-12-02 16:08:08 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve GetRemoteStatus and db.GetStatus() logic (#174)

Browse source 
* only fetch status parents / children if explicity requested when dereferencing

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* Remove recursive DB GetStatus logic, don't fetch parent unless requested

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* StatusCache copies status so there are no thread-safety issues with modified status objects

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* remove sqlite test files

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* fix bugs introduced by previous commit

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* fix not continue on error in loop

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* use our own RunInTx implementation (possible fix for nested tx error)

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* fix cast statement to work with SQLite

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* be less strict about valid status in cache

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* add cache=shared ALWAYS for SQLite db instances

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* Fix EnrichRemoteAccount when updating account fails

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* add nolint tag

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* ensure file: prefixes the filename in sqlite addr

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* add an account cache, add status author account from db

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* Fix incompatible SQLite query

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* *actually* use the new getAccount() function in accountsDB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update cache tests to use test suite

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* add RelationshipTestSuite, add tests for methods with changed SQL

Signed-off-by: kim (grufwub) <grufwub@gmail.com>
This commit is contained in:
kim 2021-09-01 10:08:21 +01:00 • committed by GitHub
commit 7d193de25f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 660 additions and 234 deletions
Download patch file Download diff file Expand all files Collapse all files

157
internal/cache/account.go vendored Normal file
View file

@ -0,0 +1,157 @@
package cache
import (
"sync"
"github.com/ReneKroon/ttlcache"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// AccountCache is a wrapper around ttlcache.Cache to provide URL and URI lookups for gtsmodel.Account
type AccountCache struct {
cache *ttlcache.Cache // map of IDs -> cached accounts
urls map[string]string // map of account URLs -> IDs
uris map[string]string // map of account URIs -> IDs
mutex sync.Mutex
}
// NewAccountCache returns a new instantiated AccountCache object
func NewAccountCache() *AccountCache {
c := AccountCache{
cache: ttlcache.NewCache(),
urls: make(map[string]string, 100),
uris: make(map[string]string, 100),
mutex: sync.Mutex{},
}
// Set callback to purge lookup maps on expiration
c.cache.SetExpirationCallback(func(key string, value interface{}) {
account := value.(*gtsmodel.Account)
c.mutex.Lock()
delete(c.urls, account.URL)
delete(c.uris, account.URI)
c.mutex.Unlock()
})
return &c
}
// GetByID attempts to fetch a account from the cache by its ID, you will receive a copy for thread-safety
func (c *AccountCache) GetByID(id string) (*gtsmodel.Account, bool) {
c.mutex.Lock()
account, ok := c.getByID(id)
c.mutex.Unlock()
return account, ok
}
// GetByURL attempts to fetch a account from the cache by its URL, you will receive a copy for thread-safety
func (c *AccountCache) GetByURL(url string) (*gtsmodel.Account, bool) {
// Perform safe ID lookup
c.mutex.Lock()
id, ok := c.urls[url]
// Not found, unlock early
if !ok {
c.mutex.Unlock()
return nil, false
}
// Attempt account lookup
account, ok := c.getByID(id)
c.mutex.Unlock()
return account, ok
}
// GetByURI attempts to fetch a account from the cache by its URI, you will receive a copy for thread-safety
func (c *AccountCache) GetByURI(uri string) (*gtsmodel.Account, bool) {
// Perform safe ID lookup
c.mutex.Lock()
id, ok := c.uris[uri]
// Not found, unlock early
if !ok {
c.mutex.Unlock()
return nil, false
}
// Attempt account lookup
account, ok := c.getByID(id)
c.mutex.Unlock()
return account, ok
}
// getByID performs an unsafe (no mutex locks) lookup of account by ID, returning a copy of account in cache
func (c *AccountCache) getByID(id string) (*gtsmodel.Account, bool) {
v, ok := c.cache.Get(id)
if !ok {
return nil, false
}
return copyAccount(v.(*gtsmodel.Account)), true
}
// Put places a account in the cache, ensuring that the object place is a copy for thread-safety
func (c *AccountCache) Put(account *gtsmodel.Account) {
if account == nil || account.ID == "" {
panic("invalid account")
}
c.mutex.Lock()
c.cache.Set(account.ID, copyAccount(account))
if account.URL != "" {
c.urls[account.URL] = account.ID
}
if account.URI != "" {
c.uris[account.URI] = account.ID
}
c.mutex.Unlock()
}
// copyAccount performs a surface-level copy of account, only keeping attached IDs intact, not the objects.
// due to all the data being copied being 99% primitive types or strings (which are immutable and passed by ptr)
// this should be a relatively cheap process
func copyAccount(account *gtsmodel.Account) *gtsmodel.Account {
return &gtsmodel.Account{
ID: account.ID,
Username: account.Username,
Domain: account.Domain,
AvatarMediaAttachmentID: account.AvatarMediaAttachmentID,
AvatarMediaAttachment: nil,
AvatarRemoteURL: account.AvatarRemoteURL,
HeaderMediaAttachmentID: account.HeaderMediaAttachmentID,
HeaderMediaAttachment: nil,
HeaderRemoteURL: account.HeaderRemoteURL,
DisplayName: account.DisplayName,
Fields: account.Fields,
Note: account.Note,
Memorial: account.Memorial,
MovedToAccountID: account.MovedToAccountID,
CreatedAt: account.CreatedAt,
UpdatedAt: account.UpdatedAt,
Bot: account.Bot,
Reason: account.Reason,
Locked: account.Locked,
Discoverable: account.Discoverable,
Privacy: account.Privacy,
Sensitive: account.Sensitive,
Language: account.Language,
URI: account.URI,
URL: account.URL,
LastWebfingeredAt: account.LastWebfingeredAt,
InboxURI: account.InboxURI,
OutboxURI: account.OutboxURI,
FollowingURI: account.FollowingURI,
FollowersURI: account.FollowersURI,
FeaturedCollectionURI: account.FeaturedCollectionURI,
ActorType: account.ActorType,
AlsoKnownAs: account.AlsoKnownAs,
PrivateKey: account.PrivateKey,
PublicKey: account.PublicKey,
PublicKeyURI: account.PublicKeyURI,
SensitizedAt: account.SensitizedAt,
SilencedAt: account.SilencedAt,
SuspendedAt: account.SuspendedAt,
HideCollections: account.HideCollections,
SuspensionOrigin: account.SuspensionOrigin,
}
}

63
internal/cache/account_test.go vendored Normal file
View file

@ -0,0 +1,63 @@
package cache_test
import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/cache"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type AccountCacheTestSuite struct {
suite.Suite
data map[string]*gtsmodel.Account
cache *cache.AccountCache
}
func (suite *AccountCacheTestSuite) SetupSuite() {
suite.data = testrig.NewTestAccounts()
}
func (suite *AccountCacheTestSuite) SetupTest() {
suite.cache = cache.NewAccountCache()
}
func (suite *AccountCacheTestSuite) TearDownTest() {
suite.data = nil
suite.cache = nil
}
func (suite *AccountCacheTestSuite) TestAccountCache() {
for _, account := range suite.data {
// Place in the cache
suite.cache.Put(account)
}
for _, account := range suite.data {
var ok bool
var check *gtsmodel.Account
// Check we can retrieve
check, ok = suite.cache.GetByID(account.ID)
if !ok && !accountIs(account, check) {
suite.Fail("Failed to fetch expected account with ID: %s", account.ID)
}
check, ok = suite.cache.GetByURI(account.URI)
if account.URI != "" && !ok && !accountIs(account, check) {
suite.Fail("Failed to fetch expected account with URI: %s", account.URI)
}
check, ok = suite.cache.GetByURL(account.URL)
if account.URL != "" && !ok && !accountIs(account, check) {
suite.Fail("Failed to fetch expected account with URL: %s", account.URL)
}
}
}
func TestAccountCache(t *testing.T) {
suite.Run(t, &AccountCacheTestSuite{})
}
func accountIs(account1, account2 *gtsmodel.Account) bool {
return account1.ID == account2.ID && account1.URI == account2.URI && account1.URL == account2.URL
}

70
internal/cache/status.go vendored
View file

@ -37,7 +37,7 @@ func NewStatusCache() *StatusCache {
return &c
}
// GetByID attempts to fetch a status from the cache by its ID
// GetByID attempts to fetch a status from the cache by its ID, you will receive a copy for thread-safety
func (c *StatusCache) GetByID(id string) (*gtsmodel.Status, bool) {
c.mutex.Lock()
status, ok := c.getByID(id)
@ -45,7 +45,7 @@ func (c *StatusCache) GetByID(id string) (*gtsmodel.Status, bool) {
return status, ok
}
// GetByURL attempts to fetch a status from the cache by its URL
// GetByURL attempts to fetch a status from the cache by its URL, you will receive a copy for thread-safety
func (c *StatusCache) GetByURL(url string) (*gtsmodel.Status, bool) {
// Perform safe ID lookup
c.mutex.Lock()
@ -63,7 +63,7 @@ func (c *StatusCache) GetByURL(url string) (*gtsmodel.Status, bool) {
return status, ok
}
// GetByURI attempts to fetch a status from the cache by its URI
// GetByURI attempts to fetch a status from the cache by its URI, you will receive a copy for thread-safety
func (c *StatusCache) GetByURI(uri string) (*gtsmodel.Status, bool) {
// Perform safe ID lookup
c.mutex.Lock()
@ -81,26 +81,72 @@ func (c *StatusCache) GetByURI(uri string) (*gtsmodel.Status, bool) {
return status, ok
}
// getByID performs an unsafe (no mutex locks) lookup of status by ID
// getByID performs an unsafe (no mutex locks) lookup of status by ID, returning a copy of status in cache
func (c *StatusCache) getByID(id string) (*gtsmodel.Status, bool) {
v, ok := c.cache.Get(id)
if !ok {
return nil, false
}
return v.(*gtsmodel.Status), true
return copyStatus(v.(*gtsmodel.Status)), true
}
// Put places a status in the cache
// Put places a status in the cache, ensuring that the object place is a copy for thread-safety
func (c *StatusCache) Put(status *gtsmodel.Status) {
if status == nil || status.ID == "" ||
status.URL == "" ||
status.URI == "" {
if status == nil || status.ID == "" {
panic("invalid status")
}
c.mutex.Lock()
c.cache.Set(status.ID, status)
c.urls[status.URL] = status.ID
c.uris[status.URI] = status.ID
c.cache.Set(status.ID, copyStatus(status))
if status.URL != "" {
c.urls[status.URL] = status.ID
}
if status.URI != "" {
c.uris[status.URI] = status.ID
}
c.mutex.Unlock()
}
// copyStatus performs a surface-level copy of status, only keeping attached IDs intact, not the objects.
// due to all the data being copied being 99% primitive types or strings (which are immutable and passed by ptr)
// this should be a relatively cheap process
func copyStatus(status *gtsmodel.Status) *gtsmodel.Status {
return &gtsmodel.Status{
ID: status.ID,
URI: status.URI,
URL: status.URL,
Content: status.Content,
AttachmentIDs: status.AttachmentIDs,
Attachments: nil,
TagIDs: status.TagIDs,
Tags: nil,
MentionIDs: status.MentionIDs,
Mentions: nil,
EmojiIDs: status.EmojiIDs,
Emojis: nil,
CreatedAt: status.CreatedAt,
UpdatedAt: status.UpdatedAt,
Local: status.Local,
AccountID: status.AccountID,
Account: nil,
AccountURI: status.AccountURI,
InReplyToID: status.InReplyToID,
InReplyTo: nil,
InReplyToURI: status.InReplyToURI,
InReplyToAccountID: status.InReplyToAccountID,
InReplyToAccount: nil,
BoostOfID: status.BoostOfID,
BoostOf: nil,
BoostOfAccountID: status.BoostOfAccountID,
BoostOfAccount: nil,
ContentWarning: status.ContentWarning,
Visibility: status.Visibility,
Sensitive: status.Sensitive,
Language: status.Language,
CreatedWithApplicationID: status.CreatedWithApplicationID,
VisibilityAdvanced: status.VisibilityAdvanced,
ActivityStreamsType: status.ActivityStreamsType,
Text: status.Text,
Pinned: status.Pinned,
}
}

74
internal/cache/status_test.go vendored
View file

@ -3,37 +3,59 @@ package cache_test
import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/cache"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type StatusCacheTestSuite struct {
suite.Suite
data map[string]*gtsmodel.Status
cache *cache.StatusCache
}
func (suite *StatusCacheTestSuite) SetupSuite() {
suite.data = testrig.NewTestStatuses()
}
func (suite *StatusCacheTestSuite) SetupTest() {
suite.cache = cache.NewStatusCache()
}
func (suite *StatusCacheTestSuite) TearDownTest() {
suite.data = nil
suite.cache = nil
}
func (suite *StatusCacheTestSuite) TestStatusCache() {
for _, status := range suite.data {
// Place in the cache
suite.cache.Put(status)
}
for _, status := range suite.data {
var ok bool
var check *gtsmodel.Status
// Check we can retrieve
check, ok = suite.cache.GetByID(status.ID)
if !ok && !statusIs(status, check) {
suite.Fail("Failed to fetch expected account with ID: %s", status.ID)
}
check, ok = suite.cache.GetByURI(status.URI)
if status.URI != "" && !ok && !statusIs(status, check) {
suite.Fail("Failed to fetch expected account with URI: %s", status.URI)
}
check, ok = suite.cache.GetByURL(status.URL)
if status.URL != "" && !ok && !statusIs(status, check) {
suite.Fail("Failed to fetch expected account with URL: %s", status.URL)
}
}
}
func TestStatusCache(t *testing.T) {
cache := cache.NewStatusCache()
// Attempt to place a status
status := gtsmodel.Status{
ID: "id",
URI: "uri",
URL: "url",
}
cache.Put(&status)
var ok bool
var check *gtsmodel.Status
// Check we can retrieve
check, ok = cache.GetByID(status.ID)
if !ok || !statusIs(&status, check) {
t.Fatal("Could not find expected status")
}
check, ok = cache.GetByURI(status.URI)
if !ok || !statusIs(&status, check) {
t.Fatal("Could not find expected status")
}
check, ok = cache.GetByURL(status.URL)
if !ok || !statusIs(&status, check) {
t.Fatal("Could not find expected status")
}
suite.Run(t, &StatusCacheTestSuite{})
}
func statusIs(status1, status2 *gtsmodel.Status) bool {