diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 1d2d0533b..3f797beb6 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -18,8 +18,31 @@ package cache +import ( + "sync" + "time" +) + // Cache defines an in-memory cache that is safe to be wiped when the application is restarted type Cache interface { Store(k string, v interface{}) error Fetch(k string) (interface{}, error) } + +type cache struct { + stored *sync.Map +} + +// New returns a new in-memory cache. +func New() Cache { + cache := &cache{ + stored: &sync.Map{}, + } + go cache.sweep() + return cache +} + +type cacheEntry struct { + updated time.Time + value interface{} +} diff --git a/internal/cache/error.go b/internal/cache/error.go new file mode 100644 index 000000000..df7cd8710 --- /dev/null +++ b/internal/cache/error.go @@ -0,0 +1,27 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package cache + +import "errors" + +// CacheError models an error returned by the in-memory cache. +type CacheError error + +// ErrNotFound means that a value for the requested key was not found in the cache. +var ErrNotFound = errors.New("value not found in cache") diff --git a/internal/cache/fetch.go b/internal/cache/fetch.go new file mode 100644 index 000000000..540e9707e --- /dev/null +++ b/internal/cache/fetch.go @@ -0,0 +1,33 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package cache + +func (c *cache) Fetch(k string) (interface{}, error) { + ceI, stored := c.stored.Load(k) + if !stored { + return nil, ErrNotFound + } + + ce, ok := ceI.(*cacheEntry) + if !ok { + panic("cache entry was not a *cacheEntry -- this should never happen") + } + + return ce.value, nil +} diff --git a/internal/cache/store.go b/internal/cache/store.go new file mode 100644 index 000000000..50aa08cfc --- /dev/null +++ b/internal/cache/store.go @@ -0,0 +1,31 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package cache + +import "time" + +func (c *cache) Store(k string, v interface{}) error { + ce := &cacheEntry{ + updated: time.Now(), + value: v, + } + + c.stored.Store(k, ce) + return nil +} diff --git a/internal/cache/sweep.go b/internal/cache/sweep.go new file mode 100644 index 000000000..e3b19f6a4 --- /dev/null +++ b/internal/cache/sweep.go @@ -0,0 +1,42 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package cache + +import "time" + +// sweep removes all entries more than 5 minutes old, on a loop. +func (c *cache) sweep() { + t := time.NewTicker(5 * time.Minute) + for range t.C { + toRemove := []interface{}{} + c.stored.Range(func(key interface{}, value interface{}) bool { + ce, ok := value.(*cacheEntry) + if !ok { + panic("cache entry was not a *cacheEntry -- this should never happen") + } + if ce.updated.Add(5 * time.Minute).After(time.Now()) { + toRemove = append(toRemove, key) + } + return true + }) + for _, r := range toRemove { + c.stored.Delete(r) + } + } +} diff --git a/internal/db/db.go b/internal/db/db.go index b3d04939b..d74eb27ed 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -35,6 +35,7 @@ type DB interface { Admin Basic Instance + Media Mention Notification Relationship diff --git a/internal/db/media.go b/internal/db/media.go new file mode 100644 index 000000000..a677ad019 --- /dev/null +++ b/internal/db/media.go @@ -0,0 +1,26 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package db + +import "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + +type Media interface { + // GetAttachmentByID gets a single attachment by its ID + GetAttachmentByID(id string) (*gtsmodel.MediaAttachment, DBError) +} diff --git a/internal/db/pg/account_test.go b/internal/db/pg/account_test.go new file mode 100644 index 000000000..7ea5ff39a --- /dev/null +++ b/internal/db/pg/account_test.go @@ -0,0 +1,70 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package pg_test + +import ( + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/testrig" +) + +type AccountTestSuite struct { + PGStandardTestSuite +} + +func (suite *AccountTestSuite) SetupSuite() { + suite.testTokens = testrig.NewTestTokens() + suite.testClients = testrig.NewTestClients() + suite.testApplications = testrig.NewTestApplications() + suite.testUsers = testrig.NewTestUsers() + suite.testAccounts = testrig.NewTestAccounts() + suite.testAttachments = testrig.NewTestAttachments() + suite.testStatuses = testrig.NewTestStatuses() + suite.testTags = testrig.NewTestTags() + suite.testMentions = testrig.NewTestMentions() +} + +func (suite *AccountTestSuite) SetupTest() { + suite.config = testrig.NewTestConfig() + suite.db = testrig.NewTestDB() + suite.log = testrig.NewTestLog() + + testrig.StandardDBSetup(suite.db, suite.testAccounts) +} + +func (suite *AccountTestSuite) TearDownTest() { + testrig.StandardDBTeardown(suite.db) +} + +func (suite *AccountTestSuite) TestGetAccountByIDWithExtras() { + account, err := suite.db.GetAccountByID(suite.testAccounts["local_account_1"].ID) + if err != nil { + suite.FailNow(err.Error()) + } + suite.NotNil(account) + suite.NotNil(account.AvatarMediaAttachment) + suite.NotEmpty(account.AvatarMediaAttachment.URL) + suite.NotNil(account.HeaderMediaAttachment) + suite.NotEmpty(account.HeaderMediaAttachment.URL) +} + +func TestAccountTestSuite(t *testing.T) { + suite.Run(t, new(AccountTestSuite)) +} diff --git a/internal/db/pg/media.go b/internal/db/pg/media.go new file mode 100644 index 000000000..dff301fa5 --- /dev/null +++ b/internal/db/pg/media.go @@ -0,0 +1,53 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package pg + +import ( + "context" + + "github.com/go-pg/pg/v10" + "github.com/go-pg/pg/v10/orm" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type mediaDB struct { + config *config.Config + conn *pg.DB + log *logrus.Logger + cancel context.CancelFunc +} + +func (m *mediaDB) newMediaQ(i interface{}) *orm.Query { + return m.conn.Model(i). + Relation("Account") +} + +func (m *mediaDB) GetAttachmentByID(id string) (*gtsmodel.MediaAttachment, db.DBError) { + attachment := >smodel.MediaAttachment{} + + q := m.newMediaQ(attachment). + Where("media_attachment.id = ?", id) + + err := processErrorResponse(q.Select()) + + return attachment, err +} diff --git a/internal/db/pg/mention.go b/internal/db/pg/mention.go index e9a27c867..7ab395756 100644 --- a/internal/db/pg/mention.go +++ b/internal/db/pg/mention.go @@ -43,35 +43,28 @@ func (m *mentionDB) newMentionQ(i interface{}) *orm.Query { Relation("TargetAccount") } -func (m *mentionDB) processResponse(mention *gtsmodel.Mention, err error) (*gtsmodel.Mention, db.DBError) { - switch err { - case pg.ErrNoRows: - return nil, db.ErrNoEntries - case nil: - return mention, nil - default: - return nil, err - } -} - func (m *mentionDB) GetMention(id string) (*gtsmodel.Mention, db.DBError) { mention := >smodel.Mention{} q := m.newMentionQ(mention). Where("mention.id = ?", id) - return m.processResponse(mention, q.Select()) + err := processErrorResponse(q.Select()) + + return mention, err } func (m *mentionDB) GetMentions(ids []string) ([]*gtsmodel.Mention, db.DBError) { mentions := []*gtsmodel.Mention{} - q := m.newMentionQ(mentions). - Where("mention.id in (?)", pg.In(ids)) - - if err := q.Select(); err != nil { - return nil, err + if len(ids) == 0 { + return mentions, nil } - return mentions, nil + q := m.newMentionQ(&mentions). + Where("mention.id in (?)", pg.In(ids)) + + err := processErrorResponse(q.Select()) + + return mentions, err } diff --git a/internal/db/pg/pg.go b/internal/db/pg/pg.go index 098513b96..f63c62cf4 100644 --- a/internal/db/pg/pg.go +++ b/internal/db/pg/pg.go @@ -50,6 +50,7 @@ type postgresService struct { db.Admin db.Basic db.Instance + db.Media db.Mention db.Notification db.Relationship @@ -128,6 +129,12 @@ func NewPostgresService(ctx context.Context, c *config.Config, log *logrus.Logge log: log, cancel: cancel, }, + Media: &mediaDB{ + config: c, + conn: conn, + log: log, + cancel: cancel, + }, Mention: &mentionDB{ config: c, conn: conn, diff --git a/internal/db/pg/status_test.go b/internal/db/pg/status_test.go index da7299dfa..9d699f0b9 100644 --- a/internal/db/pg/status_test.go +++ b/internal/db/pg/status_test.go @@ -29,7 +29,7 @@ type StatusTestSuite struct { PGStandardTestSuite } -func (suite *PGStandardTestSuite) SetupSuite() { +func (suite *StatusTestSuite) SetupSuite() { suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() suite.testApplications = testrig.NewTestApplications() @@ -41,7 +41,7 @@ func (suite *PGStandardTestSuite) SetupSuite() { suite.testMentions = testrig.NewTestMentions() } -func (suite *PGStandardTestSuite) SetupTest() { +func (suite *StatusTestSuite) SetupTest() { suite.config = testrig.NewTestConfig() suite.db = testrig.NewTestDB() suite.log = testrig.NewTestLog() @@ -49,11 +49,11 @@ func (suite *PGStandardTestSuite) SetupTest() { testrig.StandardDBSetup(suite.db, suite.testAccounts) } -func (suite *PGStandardTestSuite) TearDownTest() { +func (suite *StatusTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) } -func (suite *PGStandardTestSuite) TestGetStatusByID() { +func (suite *StatusTestSuite) TestGetStatusByID() { status, err := suite.db.GetStatusByID(suite.testStatuses["local_account_1_status_1"].ID) if err != nil { suite.FailNow(err.Error()) @@ -67,7 +67,7 @@ func (suite *PGStandardTestSuite) TestGetStatusByID() { suite.Nil(status.InReplyToAccount) } -func (suite *PGStandardTestSuite) TestGetStatusByURI() { +func (suite *StatusTestSuite) TestGetStatusByURI() { status, err := suite.db.GetStatusByURI(suite.testStatuses["local_account_1_status_1"].URI) if err != nil { suite.FailNow(err.Error()) @@ -81,7 +81,7 @@ func (suite *PGStandardTestSuite) TestGetStatusByURI() { suite.Nil(status.InReplyToAccount) } -func (suite *PGStandardTestSuite) TestGetStatusWithExtras() { +func (suite *StatusTestSuite) TestGetStatusWithExtras() { status, err := suite.db.GetStatusByID(suite.testStatuses["admin_account_status_1"].ID) if err != nil { suite.FailNow(err.Error()) @@ -95,5 +95,5 @@ func (suite *PGStandardTestSuite) TestGetStatusWithExtras() { } func TestStatusTestSuite(t *testing.T) { - suite.Run(t, new(PGStandardTestSuite)) + suite.Run(t, new(StatusTestSuite)) } diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index c7af2bb76..354f37e04 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -34,16 +34,16 @@ type Status struct { Content string // Database IDs of any media attachments associated with this status AttachmentIDs []string `pg:"attachments,array"` - Attachments []*MediaAttachment `pg:"rel:has-many"` + Attachments []*MediaAttachment `pg:"attached_media,rel:has-many"` // Database IDs of any tags used in this status TagIDs []string `pg:"tags,array"` - Tags []*Tag `pg:"many2many:status_to_tags"` // https://pg.uptrace.dev/orm/many-to-many-relation/ + Tags []*Tag `pg:"attached_tags,many2many:status_to_tags"` // https://pg.uptrace.dev/orm/many-to-many-relation/ // Database IDs of any mentions in this status MentionIDs []string `pg:"mentions,array"` - Mentions []*Mention `pg:"rel:has-many"` + Mentions []*Mention `pg:"attached_mentions,rel:has-many"` // Database IDs of any emojis used in this status EmojiIDs []string `pg:"emojis,array"` - Emojis []*Emoji `pg:"many2many:status_to_emojis"` // https://pg.uptrace.dev/orm/many-to-many-relation/ + Emojis []*Emoji `pg:"attached_emojis,many2many:status_to_emojis"` // https://pg.uptrace.dev/orm/many-to-many-relation/ // when was this status created? CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"` // when was this status updated? diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go index 10d9a0f18..3a90cfa53 100644 --- a/internal/typeutils/converter.go +++ b/internal/typeutils/converter.go @@ -24,6 +24,7 @@ import ( "github.com/go-fed/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/cache" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -175,8 +176,9 @@ type TypeConverter interface { } type converter struct { - config *config.Config - db db.DB + config *config.Config + db db.DB + frontendCache cache.Cache } // NewConverter returns a new Converter @@ -184,5 +186,6 @@ func NewConverter(config *config.Config, db db.DB) TypeConverter { return &converter{ config: config, db: db, + frontendCache: cache.New(), } } diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index 552157d0c..cdf68730b 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -62,6 +62,14 @@ func (c *converter) AccountToMastoSensitive(a *gtsmodel.Account) (*model.Account } func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, error) { + // first check if we have this account in our frontEnd cache + if accountI, err := c.frontendCache.Fetch(a.ID); err == nil { + if account, ok := accountI.(*model.Account); ok { + // we have it, so just return it as-is + return account, nil + } + } + // count followers followersCount, err := c.db.CountAccountFollowers(a.ID, false) if err != nil { @@ -90,14 +98,30 @@ func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, e // build the avatar and header URLs var aviURL string var aviURLStatic string - if a.AvatarMediaAttachment != nil { + if a.AvatarMediaAttachmentID != "" { + // make sure avi is pinned to this account + if a.AvatarMediaAttachment == nil { + avi, err := c.db.GetAttachmentByID(a.AvatarMediaAttachmentID) + if err != nil { + return nil, fmt.Errorf("error retrieving avatar: %s", err) + } + a.AvatarMediaAttachment = avi + } aviURL = a.AvatarMediaAttachment.URL aviURLStatic = a.AvatarMediaAttachment.Thumbnail.URL } var headerURL string var headerURLStatic string - if a.HeaderMediaAttachment != nil { + if a.HeaderMediaAttachmentID != "" { + // make sure header is pinned to this account + if a.HeaderMediaAttachment == nil { + avi, err := c.db.GetAttachmentByID(a.HeaderMediaAttachmentID) + if err != nil { + return nil, fmt.Errorf("error retrieving avatar: %s", err) + } + a.HeaderMediaAttachment = avi + } headerURL = a.HeaderMediaAttachment.URL headerURLStatic = a.HeaderMediaAttachment.Thumbnail.URL } @@ -132,7 +156,7 @@ func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, e suspended = true } - return &model.Account{ + accountFrontend := &model.Account{ ID: a.ID, Username: a.Username, Acct: acct, @@ -153,7 +177,14 @@ func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, e Emojis: emojis, // TODO: implement this Fields: fields, Suspended: suspended, - }, nil + } + + // put the account in our cache in case we need it again soon + if err := c.frontendCache.Store(a.ID, accountFrontend); err != nil { + return nil, err + } + + return accountFrontend, nil } func (c *converter) AccountToMastoBlocked(a *gtsmodel.Account) (*model.Account, error) {