and yet more

internal/cache/cache.go

@@ -18,8 +18,31 @@
 
 package cache
 
+import (
+	"sync"
+	"time"
+)
+
 // Cache defines an in-memory cache that is safe to be wiped when the application is restarted
 type Cache interface {
 	Store(k string, v interface{}) error
 	Fetch(k string) (interface{}, error)
 }
+
+type cache struct {
+	stored *sync.Map
+}
+
+// New returns a new in-memory cache.
+func New() Cache {
+   cache := &cache{
+		stored: &sync.Map{},
+	}
+   go cache.sweep()
+   return cache
+}
+
+type cacheEntry struct {
+	updated time.Time
+	value   interface{}
+}

internal/cache/error.go

@@ -0,0 +1,27 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package cache
+
+import "errors"
+
+// CacheError models an error returned by the in-memory cache.
+type CacheError error
+
+// ErrNotFound means that a value for the requested key was not found in the cache.
+var ErrNotFound = errors.New("value not found in cache")

internal/cache/fetch.go

@@ -0,0 +1,33 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package cache
+
+func (c *cache) Fetch(k string) (interface{}, error) {
+	ceI, stored := c.stored.Load(k)
+	if !stored {
+		return nil, ErrNotFound
+	}
+
+	ce, ok := ceI.(*cacheEntry)
+	if !ok {
+		panic("cache entry was not a *cacheEntry -- this should never happen")
+	}
+
+	return ce.value, nil
+}

internal/cache/store.go

@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package cache
+
+import "time"
+
+func (c *cache) Store(k string, v interface{}) error {
+	ce := &cacheEntry{
+		updated: time.Now(),
+		value:   v,
+	}
+
+	c.stored.Store(k, ce)
+	return nil
+}

internal/cache/sweep.go

@@ -0,0 +1,42 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package cache
+
+import "time"
+
+// sweep removes all entries more than 5 minutes old, on a loop.
+func (c *cache) sweep() {
+	t := time.NewTicker(5 * time.Minute)
+	for range t.C {
+		toRemove := []interface{}{}
+		c.stored.Range(func(key interface{}, value interface{}) bool {
+			ce, ok := value.(*cacheEntry)
+			if !ok {
+				panic("cache entry was not a *cacheEntry -- this should never happen")
+			}
+			if ce.updated.Add(5 * time.Minute).After(time.Now()) {
+				toRemove = append(toRemove, key)
+			}
+			return true
+		})
+		for _, r := range toRemove {
+			c.stored.Delete(r)
+		}
+	}
+}

internal/db/db.go

@@ -35,6 +35,7 @@ type DB interface {
 	Admin
 	Basic
 	Instance
+	Media
 	Mention
 	Notification
 	Relationship

internal/db/media.go

@@ -0,0 +1,26 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package db
+
+import "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+
+type Media interface {
+	// GetAttachmentByID gets a single attachment by its ID
+	GetAttachmentByID(id string) (*gtsmodel.MediaAttachment, DBError)
+}

internal/db/pg/account_test.go

@@ -0,0 +1,70 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package pg_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type AccountTestSuite struct {
+	PGStandardTestSuite
+}
+
+func (suite *AccountTestSuite) SetupSuite() {
+	suite.testTokens = testrig.NewTestTokens()
+	suite.testClients = testrig.NewTestClients()
+	suite.testApplications = testrig.NewTestApplications()
+	suite.testUsers = testrig.NewTestUsers()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
+	suite.testStatuses = testrig.NewTestStatuses()
+	suite.testTags = testrig.NewTestTags()
+	suite.testMentions = testrig.NewTestMentions()
+}
+
+func (suite *AccountTestSuite) SetupTest() {
+	suite.config = testrig.NewTestConfig()
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+
+	testrig.StandardDBSetup(suite.db, suite.testAccounts)
+}
+
+func (suite *AccountTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+}
+
+func (suite *AccountTestSuite) TestGetAccountByIDWithExtras() {
+	account, err := suite.db.GetAccountByID(suite.testAccounts["local_account_1"].ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+	suite.NotNil(account)
+	suite.NotNil(account.AvatarMediaAttachment)
+	suite.NotEmpty(account.AvatarMediaAttachment.URL)
+	suite.NotNil(account.HeaderMediaAttachment)
+	suite.NotEmpty(account.HeaderMediaAttachment.URL)
+}
+
+func TestAccountTestSuite(t *testing.T) {
+	suite.Run(t, new(AccountTestSuite))
+}

internal/db/pg/media.go

@@ -0,0 +1,53 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package pg
+
+import (
+	"context"
+
+	"github.com/go-pg/pg/v10"
+	"github.com/go-pg/pg/v10/orm"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+type mediaDB struct {
+	config *config.Config
+	conn   *pg.DB
+	log    *logrus.Logger
+	cancel context.CancelFunc
+}
+
+func (m *mediaDB) newMediaQ(i interface{}) *orm.Query {
+	return m.conn.Model(i).
+		Relation("Account")
+}
+
+func (m *mediaDB) GetAttachmentByID(id string) (*gtsmodel.MediaAttachment, db.DBError) {
+	attachment := &gtsmodel.MediaAttachment{}
+
+	q := m.newMediaQ(attachment).
+		Where("media_attachment.id = ?", id)
+
+	err := processErrorResponse(q.Select())
+
+	return attachment, err
+}

internal/db/pg/mention.go

@@ -43,35 +43,28 @@ func (m *mentionDB) newMentionQ(i interface{}) *orm.Query {
 		Relation("TargetAccount")
 }
 
-func (m *mentionDB) processResponse(mention *gtsmodel.Mention, err error) (*gtsmodel.Mention, db.DBError) {
-	switch err {
-	case pg.ErrNoRows:
-		return nil, db.ErrNoEntries
-	case nil:
-		return mention, nil
-	default:
-		return nil, err
-	}
-}
-
 func (m *mentionDB) GetMention(id string) (*gtsmodel.Mention, db.DBError) {
 	mention := &gtsmodel.Mention{}
 
 	q := m.newMentionQ(mention).
 		Where("mention.id = ?", id)
 
-	return m.processResponse(mention, q.Select())
+	err := processErrorResponse(q.Select())
+
+	return mention, err
 }
 
 func (m *mentionDB) GetMentions(ids []string) ([]*gtsmodel.Mention, db.DBError) {
 	mentions := []*gtsmodel.Mention{}
 
-	q := m.newMentionQ(mentions).
-		Where("mention.id in (?)", pg.In(ids))
-
-	if err := q.Select(); err != nil {
-		return nil, err
+	if len(ids) == 0 {
+		return mentions, nil
 	}
 
-	return mentions, nil
+	q := m.newMentionQ(&mentions).
+		Where("mention.id in (?)", pg.In(ids))
+
+	err := processErrorResponse(q.Select())
+
+	return mentions, err
 }

internal/db/pg/pg.go

@@ -50,6 +50,7 @@ type postgresService struct {
 	db.Admin
 	db.Basic
 	db.Instance
+	db.Media
 	db.Mention
 	db.Notification
 	db.Relationship
@@ -128,6 +129,12 @@ func NewPostgresService(ctx context.Context, c *config.Config, log *logrus.Logge
 			log:    log,
 			cancel: cancel,
 		},
+		Media: &mediaDB{
+			config: c,
+			conn:   conn,
+			log:    log,
+			cancel: cancel,
+		},
 		Mention: &mentionDB{
 			config: c,
 			conn:   conn,

internal/db/pg/status_test.go

@@ -29,7 +29,7 @@ type StatusTestSuite struct {
 	PGStandardTestSuite
 }
 
-func (suite *PGStandardTestSuite) SetupSuite() {
+func (suite *StatusTestSuite) SetupSuite() {
 	suite.testTokens = testrig.NewTestTokens()
 	suite.testClients = testrig.NewTestClients()
 	suite.testApplications = testrig.NewTestApplications()
@@ -41,7 +41,7 @@ func (suite *PGStandardTestSuite) SetupSuite() {
 	suite.testMentions = testrig.NewTestMentions()
 }
 
-func (suite *PGStandardTestSuite) SetupTest() {
+func (suite *StatusTestSuite) SetupTest() {
 	suite.config = testrig.NewTestConfig()
 	suite.db = testrig.NewTestDB()
 	suite.log = testrig.NewTestLog()
@@ -49,11 +49,11 @@ func (suite *PGStandardTestSuite) SetupTest() {
 	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 }
 
-func (suite *PGStandardTestSuite) TearDownTest() {
+func (suite *StatusTestSuite) TearDownTest() {
 	testrig.StandardDBTeardown(suite.db)
 }
 
-func (suite *PGStandardTestSuite) TestGetStatusByID() {
+func (suite *StatusTestSuite) TestGetStatusByID() {
 	status, err := suite.db.GetStatusByID(suite.testStatuses["local_account_1_status_1"].ID)
 	if err != nil {
 		suite.FailNow(err.Error())
@@ -67,7 +67,7 @@ func (suite *PGStandardTestSuite) TestGetStatusByID() {
 	suite.Nil(status.InReplyToAccount)
 }
 
-func (suite *PGStandardTestSuite) TestGetStatusByURI() {
+func (suite *StatusTestSuite) TestGetStatusByURI() {
 	status, err := suite.db.GetStatusByURI(suite.testStatuses["local_account_1_status_1"].URI)
 	if err != nil {
 		suite.FailNow(err.Error())
@@ -81,7 +81,7 @@ func (suite *PGStandardTestSuite) TestGetStatusByURI() {
 	suite.Nil(status.InReplyToAccount)
 }
 
-func (suite *PGStandardTestSuite) TestGetStatusWithExtras() {
+func (suite *StatusTestSuite) TestGetStatusWithExtras() {
 	status, err := suite.db.GetStatusByID(suite.testStatuses["admin_account_status_1"].ID)
 	if err != nil {
 		suite.FailNow(err.Error())
@@ -95,5 +95,5 @@ func (suite *PGStandardTestSuite) TestGetStatusWithExtras() {
 }
 
 func TestStatusTestSuite(t *testing.T) {
-	suite.Run(t, new(PGStandardTestSuite))
+	suite.Run(t, new(StatusTestSuite))
 }

internal/gtsmodel/status.go

@@ -34,16 +34,16 @@ type Status struct {
 	Content string
 	// Database IDs of any media attachments associated with this status
 	AttachmentIDs []string           `pg:"attachments,array"`
-	Attachments   []*MediaAttachment `pg:"rel:has-many"`
+	Attachments   []*MediaAttachment `pg:"attached_media,rel:has-many"`
 	// Database IDs of any tags used in this status
 	TagIDs []string `pg:"tags,array"`
-	Tags   []*Tag   `pg:"many2many:status_to_tags"` // https://pg.uptrace.dev/orm/many-to-many-relation/
+	Tags   []*Tag   `pg:"attached_tags,many2many:status_to_tags"` // https://pg.uptrace.dev/orm/many-to-many-relation/
 	// Database IDs of any mentions in this status
 	MentionIDs []string   `pg:"mentions,array"`
-	Mentions   []*Mention `pg:"rel:has-many"`
+	Mentions   []*Mention `pg:"attached_mentions,rel:has-many"`
 	// Database IDs of any emojis used in this status
 	EmojiIDs []string `pg:"emojis,array"`
-	Emojis   []*Emoji `pg:"many2many:status_to_emojis"` // https://pg.uptrace.dev/orm/many-to-many-relation/
+	Emojis   []*Emoji `pg:"attached_emojis,many2many:status_to_emojis"` // https://pg.uptrace.dev/orm/many-to-many-relation/
 	// when was this status created?
 	CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
 	// when was this status updated?

internal/typeutils/converter.go

@@ -24,6 +24,7 @@ import (
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/cache"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -175,8 +176,9 @@ type TypeConverter interface {
 }
 
 type converter struct {
-	config *config.Config
-	db     db.DB
+	config         *config.Config
+	db             db.DB
+	frontendCache cache.Cache
 }
 
 // NewConverter returns a new Converter
@@ -184,5 +186,6 @@ func NewConverter(config *config.Config, db db.DB) TypeConverter {
 	return &converter{
 		config: config,
 		db:     db,
+		frontendCache: cache.New(),
 	}
 }

internal/typeutils/internaltofrontend.go

@@ -62,6 +62,14 @@ func (c *converter) AccountToMastoSensitive(a *gtsmodel.Account) (*model.Account
 }
 
 func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, error) {
+	// first check if we have this account in our frontEnd cache
+	if accountI, err := c.frontendCache.Fetch(a.ID); err == nil {
+		if account, ok := accountI.(*model.Account); ok {
+			// we have it, so just return it as-is
+			return account, nil
+		}
+	}
+
 	// count followers
 	followersCount, err := c.db.CountAccountFollowers(a.ID, false)
 	if err != nil {
@@ -90,14 +98,30 @@ func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, e
 	// build the avatar and header URLs
 	var aviURL string
 	var aviURLStatic string
-	if a.AvatarMediaAttachment != nil {
+	if a.AvatarMediaAttachmentID != "" {
+		// make sure avi is pinned to this account
+		if a.AvatarMediaAttachment == nil {
+			avi, err := c.db.GetAttachmentByID(a.AvatarMediaAttachmentID)
+			if err != nil {
+				return nil, fmt.Errorf("error retrieving avatar: %s", err)
+			}
+			a.AvatarMediaAttachment = avi
+		}
 		aviURL = a.AvatarMediaAttachment.URL
 		aviURLStatic = a.AvatarMediaAttachment.Thumbnail.URL
 	}
 
 	var headerURL string
 	var headerURLStatic string
-	if a.HeaderMediaAttachment != nil {
+	if a.HeaderMediaAttachmentID != "" {
+		// make sure header is pinned to this account
+		if a.HeaderMediaAttachment == nil {
+			avi, err := c.db.GetAttachmentByID(a.HeaderMediaAttachmentID)
+			if err != nil {
+				return nil, fmt.Errorf("error retrieving avatar: %s", err)
+			}
+			a.HeaderMediaAttachment = avi
+		}
 		headerURL = a.HeaderMediaAttachment.URL
 		headerURLStatic = a.HeaderMediaAttachment.Thumbnail.URL
 	}
@@ -132,7 +156,7 @@ func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, e
 		suspended = true
 	}
 
-	return &model.Account{
+	accountFrontend := &model.Account{
 		ID:             a.ID,
 		Username:       a.Username,
 		Acct:           acct,
@@ -153,7 +177,14 @@ func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, e
 		Emojis:         emojis, // TODO: implement this
 		Fields:         fields,
 		Suspended:      suspended,
-	}, nil
+	}
+
+	// put the account in our cache in case we need it again soon
+	if err := c.frontendCache.Store(a.ID, accountFrontend); err != nil {
+		return nil, err
+	}
+
+	return accountFrontend, nil
 }
 
 func (c *converter) AccountToMastoBlocked(a *gtsmodel.Account) (*model.Account, error) {