mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-12-07 18:38:08 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[bugfix/chore] oauth entropy fix + media cleanup tasks rewrite (#1853)

Browse source
This commit is contained in:
kim 2023-06-22 20:46:36 +01:00 • committed by GitHub
commit 9a22102fa8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 2076 additions and 1090 deletions
Download patch file Download diff file Expand all files Collapse all files

222
internal/db/bundb/emoji.go
View file

@ -19,6 +19,7 @@ package bundb
import (
"context"
"database/sql"
"errors"
"strings"
"time"
@ -37,19 +38,6 @@ type emojiDB struct {
state *state.State
}
func (e *emojiDB) newEmojiQ(emoji *gtsmodel.Emoji) *bun.SelectQuery {
return e.conn.
NewSelect().
Model(emoji).
Relation("Category")
}
func (e *emojiDB) newEmojiCategoryQ(emojiCategory *gtsmodel.EmojiCategory) *bun.SelectQuery {
return e.conn.
NewSelect().
Model(emojiCategory)
}
func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error {
return e.state.Caches.GTS.Emoji().Store(emoji, func() error {
_, err := e.conn.NewInsert().Model(emoji).Exec(ctx)
@ -57,14 +45,15 @@ func (e *emojiDB) PutEmoji(ctx context.Context, emoji *gtsmodel.Emoji) db.Error
})
}
func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) (*gtsmodel.Emoji, db.Error) {
func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, columns ...string) error {
emoji.UpdatedAt = time.Now()
if len(columns) > 0 {
// If we're updating by column, ensure "updated_at" is included.
columns = append(columns, "updated_at")
}
err := e.state.Caches.GTS.Emoji().Store(emoji, func() error {
// Update the emoji model in the database.
return e.state.Caches.GTS.Emoji().Store(emoji, func() error {
_, err := e.conn.
NewUpdate().
Model(emoji).
@ -73,15 +62,34 @@ func (e *emojiDB) UpdateEmoji(ctx context.Context, emoji *gtsmodel.Emoji, column
Exec(ctx)
return e.conn.ProcessError(err)
})
if err != nil {
return nil, err
}
return emoji, nil
}
func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error {
defer e.state.Caches.GTS.Emoji().Invalidate("ID", id)
var (
accountIDs []string
statusIDs []string
)
defer func() {
// Invalidate cached emoji.
e.state.Caches.GTS.
Emoji().
Invalidate("ID", id)
for _, id := range accountIDs {
// Invalidate cached account.
e.state.Caches.GTS.
Account().
Invalidate("ID", id)
}
for _, id := range statusIDs {
// Invalidate cached account.
e.state.Caches.GTS.
Status().
Invalidate("ID", id)
}
}()
// Load emoji into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
@ -99,6 +107,7 @@ func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error {
return e.conn.RunInTx(ctx, func(tx bun.Tx) error {
// delete links between this emoji and any statuses that use it
// TODO: remove when we delete this table
if _, err := tx.
NewDelete().
TableExpr("? AS ?", bun.Ident("status_to_emojis"), bun.Ident("status_to_emoji")).
@ -108,6 +117,7 @@ func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error {
}
// delete links between this emoji and any accounts that use it
// TODO: remove when we delete this table
if _, err := tx.
NewDelete().
TableExpr("? AS ?", bun.Ident("account_to_emojis"), bun.Ident("account_to_emoji")).
@ -116,19 +126,91 @@ func (e *emojiDB) DeleteEmojiByID(ctx context.Context, id string) db.Error {
return err
}
if _, err := tx.
NewDelete().
TableExpr("? AS ?", bun.Ident("emojis"), bun.Ident("emoji")).
Where("? = ?", bun.Ident("emoji.id"), id).
// Select all accounts using this emoji.
if _, err := tx.NewSelect().
Table("accounts").
Column("id").
Where("? IN (emojis)", id).
Exec(ctx, &accountIDs); err != nil {
return err
}
for _, id := range accountIDs {
var emojiIDs []string
// Select account with ID.
if _, err := tx.NewSelect().
Table("accounts").
Column("emojis").
Where("id = ?", id).
Exec(ctx); err != nil &&
err != sql.ErrNoRows {
return err
}
// Drop ID from account emojis.
emojiIDs = dropID(emojiIDs, id)
// Update account emoji IDs.
if _, err := tx.NewUpdate().
Table("accounts").
Where("id = ?", id).
Set("emojis = ?", emojiIDs).
Exec(ctx); err != nil &&
err != sql.ErrNoRows {
return err
}
}
// Select all statuses using this emoji.
if _, err := tx.NewSelect().
Table("statuses").
Column("id").
Where("? IN (emojis)", id).
Exec(ctx, &statusIDs); err != nil {
return err
}
for _, id := range statusIDs {
var emojiIDs []string
// Select statuses with ID.
if _, err := tx.NewSelect().
Table("statuses").
Column("emojis").
Where("id = ?", id).
Exec(ctx); err != nil &&
err != sql.ErrNoRows {
return err
}
// Drop ID from account emojis.
emojiIDs = dropID(emojiIDs, id)
// Update status emoji IDs.
if _, err := tx.NewUpdate().
Table("statuses").
Where("id = ?", id).
Set("emojis = ?", emojiIDs).
Exec(ctx); err != nil &&
err != sql.ErrNoRows {
return err
}
}
// Delete emoji from database.
if _, err := tx.NewDelete().
Table("emojis").
Where("id = ?", id).
Exec(ctx); err != nil {
return e.conn.ProcessError(err)
return err
}
return nil
})
}
func (e *emojiDB) GetEmojis(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, db.Error) {
func (e *emojiDB) GetEmojisBy(ctx context.Context, domain string, includeDisabled bool, includeEnabled bool, shortcode string, maxShortcodeDomain string, minShortcodeDomain string, limit int) ([]*gtsmodel.Emoji, error) {
emojiIDs := []string{}
subQuery := e.conn.
@ -245,6 +327,29 @@ func (e *emojiDB) GetEmojis(ctx context.Context, domain string, includeDisabled
return e.GetEmojisByIDs(ctx, emojiIDs)
}
func (e *emojiDB) GetEmojis(ctx context.Context, maxID string, limit int) ([]*gtsmodel.Emoji, error) {
emojiIDs := []string{}
q := e.conn.NewSelect().
Table("emojis").
Column("id").
Order("id DESC")
if maxID != "" {
q = q.Where("? < ?", bun.Ident("id"), maxID)
}
if limit != 0 {
q = q.Limit(limit)
}
if err := q.Scan(ctx, &emojiIDs); err != nil {
return nil, e.conn.ProcessError(err)
}
return e.GetEmojisByIDs(ctx, emojiIDs)
}
func (e *emojiDB) GetUseableEmojis(ctx context.Context) ([]*gtsmodel.Emoji, db.Error) {
emojiIDs := []string{}
@ -269,7 +374,10 @@ func (e *emojiDB) GetEmojiByID(ctx context.Context, id string) (*gtsmodel.Emoji,
ctx,
"ID",
func(emoji *gtsmodel.Emoji) error {
return e.newEmojiQ(emoji).Where("? = ?", bun.Ident("emoji.id"), id).Scan(ctx)
return e.conn.
NewSelect().
Model(emoji).
Where("? = ?", bun.Ident("emoji.id"), id).Scan(ctx)
},
id,
)
@ -280,7 +388,10 @@ func (e *emojiDB) GetEmojiByURI(ctx context.Context, uri string) (*gtsmodel.Emoj
ctx,
"URI",
func(emoji *gtsmodel.Emoji) error {
return e.newEmojiQ(emoji).Where("? = ?", bun.Ident("emoji.uri"), uri).Scan(ctx)
return e.conn.
NewSelect().
Model(emoji).
Where("? = ?", bun.Ident("emoji.uri"), uri).Scan(ctx)
},
uri,
)
@ -291,7 +402,9 @@ func (e *emojiDB) GetEmojiByShortcodeDomain(ctx context.Context, shortcode strin
ctx,
"Shortcode.Domain",
func(emoji *gtsmodel.Emoji) error {
q := e.newEmojiQ(emoji)
q := e.conn.
NewSelect().
Model(emoji)
if domain != "" {
q = q.Where("? = ?", bun.Ident("emoji.shortcode"), shortcode)
@ -313,8 +426,9 @@ func (e *emojiDB) GetEmojiByStaticURL(ctx context.Context, imageStaticURL string
ctx,
"ImageStaticURL",
func(emoji *gtsmodel.Emoji) error {
return e.
newEmojiQ(emoji).
return e.conn.
NewSelect().
Model(emoji).
Where("? = ?", bun.Ident("emoji.image_static_url"), imageStaticURL).
Scan(ctx)
},
@ -350,7 +464,10 @@ func (e *emojiDB) GetEmojiCategory(ctx context.Context, id string) (*gtsmodel.Em
ctx,
"ID",
func(emojiCategory *gtsmodel.EmojiCategory) error {
return e.newEmojiCategoryQ(emojiCategory).Where("? = ?", bun.Ident("emoji_category.id"), id).Scan(ctx)
return e.conn.
NewSelect().
Model(emojiCategory).
Where("? = ?", bun.Ident("emoji_category.id"), id).Scan(ctx)
},
id,
)
@ -361,14 +478,18 @@ func (e *emojiDB) GetEmojiCategoryByName(ctx context.Context, name string) (*gts
ctx,
"Name",
func(emojiCategory *gtsmodel.EmojiCategory) error {
return e.newEmojiCategoryQ(emojiCategory).Where("LOWER(?) = ?", bun.Ident("emoji_category.name"), strings.ToLower(name)).Scan(ctx)
return e.conn.
NewSelect().
Model(emojiCategory).
Where("LOWER(?) = ?", bun.Ident("emoji_category.name"), strings.ToLower(name)).Scan(ctx)
},
name,
)
}
func (e *emojiDB) getEmoji(ctx context.Context, lookup string, dbQuery func(*gtsmodel.Emoji) error, keyParts ...any) (*gtsmodel.Emoji, db.Error) {
return e.state.Caches.GTS.Emoji().Load(lookup, func() (*gtsmodel.Emoji, error) {
// Fetch emoji from database cache with loader callback
emoji, err := e.state.Caches.GTS.Emoji().Load(lookup, func() (*gtsmodel.Emoji, error) {
var emoji gtsmodel.Emoji
// Not cached! Perform database query
@ -378,6 +499,23 @@ func (e *emojiDB) getEmoji(ctx context.Context, lookup string, dbQuery func(*gts
return &emoji, nil
}, keyParts...)
if err != nil {
return nil, err
}
if gtscontext.Barebones(ctx) {
// no need to fully populate.
return emoji, nil
}
if emoji.CategoryID != "" {
emoji.Category, err = e.GetEmojiCategory(ctx, emoji.CategoryID)
if err != nil {
log.Errorf(ctx, "error getting emoji category %s: %v", emoji.CategoryID, err)
}
}
return emoji, nil
}
func (e *emojiDB) GetEmojisByIDs(ctx context.Context, emojiIDs []string) ([]*gtsmodel.Emoji, db.Error) {
@ -432,3 +570,17 @@ func (e *emojiDB) GetEmojiCategoriesByIDs(ctx context.Context, emojiCategoryIDs
return emojiCategories, nil
}
// dropIDs drops given ID string from IDs slice.
func dropID(ids []string, id string) []string {
for i := 0; i < len(ids); {
if ids[i] == id {
// Remove this reference.
copy(ids[i:], ids[i+1:])
ids = ids[:len(ids)-1]
continue
}
i++
}
return ids
}

22
internal/db/bundb/emoji_test.go
View file

@ -59,7 +59,7 @@ func (suite *EmojiTestSuite) TestGetEmojiByStaticURL() {
}
func (suite *EmojiTestSuite) TestGetAllEmojis() {
emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 0)
suite.NoError(err)
suite.Equal(2, len(emojis))
@ -68,7 +68,7 @@ func (suite *EmojiTestSuite) TestGetAllEmojis() {
}
func (suite *EmojiTestSuite) TestGetAllEmojisLimit1() {
emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 1)
emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, true, "", "", "", 1)
suite.NoError(err)
suite.Equal(1, len(emojis))
@ -76,7 +76,7 @@ func (suite *EmojiTestSuite) TestGetAllEmojisLimit1() {
}
func (suite *EmojiTestSuite) TestGetAllEmojisMaxID() {
emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "rainbow@", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, true, "", "rainbow@", "", 0)
suite.NoError(err)
suite.Equal(1, len(emojis))
@ -84,7 +84,7 @@ func (suite *EmojiTestSuite) TestGetAllEmojisMaxID() {
}
func (suite *EmojiTestSuite) TestGetAllEmojisMinID() {
emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, true, "", "", "yell@fossbros-anonymous.io", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, true, "", "", "yell@fossbros-anonymous.io", 0)
suite.NoError(err)
suite.Equal(1, len(emojis))
@ -92,14 +92,14 @@ func (suite *EmojiTestSuite) TestGetAllEmojisMinID() {
}
func (suite *EmojiTestSuite) TestGetAllDisabledEmojis() {
emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, true, false, "", "", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, true, false, "", "", "", 0)
suite.ErrorIs(err, db.ErrNoEntries)
suite.Equal(0, len(emojis))
}
func (suite *EmojiTestSuite) TestGetAllEnabledEmojis() {
emojis, err := suite.db.GetEmojis(context.Background(), db.EmojiAllDomains, false, true, "", "", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), db.EmojiAllDomains, false, true, "", "", "", 0)
suite.NoError(err)
suite.Equal(2, len(emojis))
@ -108,7 +108,7 @@ func (suite *EmojiTestSuite) TestGetAllEnabledEmojis() {
}
func (suite *EmojiTestSuite) TestGetLocalEnabledEmojis() {
emojis, err := suite.db.GetEmojis(context.Background(), "", false, true, "", "", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), "", false, true, "", "", "", 0)
suite.NoError(err)
suite.Equal(1, len(emojis))
@ -116,21 +116,21 @@ func (suite *EmojiTestSuite) TestGetLocalEnabledEmojis() {
}
func (suite *EmojiTestSuite) TestGetLocalDisabledEmojis() {
emojis, err := suite.db.GetEmojis(context.Background(), "", true, false, "", "", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), "", true, false, "", "", "", 0)
suite.ErrorIs(err, db.ErrNoEntries)
suite.Equal(0, len(emojis))
}
func (suite *EmojiTestSuite) TestGetAllEmojisFromDomain() {
emojis, err := suite.db.GetEmojis(context.Background(), "peepee.poopoo", true, true, "", "", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), "peepee.poopoo", true, true, "", "", "", 0)
suite.ErrorIs(err, db.ErrNoEntries)
suite.Equal(0, len(emojis))
}
func (suite *EmojiTestSuite) TestGetAllEmojisFromDomain2() {
emojis, err := suite.db.GetEmojis(context.Background(), "fossbros-anonymous.io", true, true, "", "", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), "fossbros-anonymous.io", true, true, "", "", "", 0)
suite.NoError(err)
suite.Equal(1, len(emojis))
@ -138,7 +138,7 @@ func (suite *EmojiTestSuite) TestGetAllEmojisFromDomain2() {
}
func (suite *EmojiTestSuite) TestGetSpecificEmojisFromDomain2() {
emojis, err := suite.db.GetEmojis(context.Background(), "fossbros-anonymous.io", true, true, "yell", "", "", 0)
emojis, err := suite.db.GetEmojisBy(context.Background(), "fossbros-anonymous.io", true, true, "yell", "", "", 0)
suite.NoError(err)
suite.Equal(1, len(emojis))

140
internal/db/bundb/media.go
View file

@ -24,6 +24,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
@ -110,7 +111,7 @@ func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error {
// Load media into cache before attempting a delete,
// as we need it cached in order to trigger the invalidate
// callback. This in turn invalidates others.
_, err := m.GetAttachmentByID(gtscontext.SetBarebones(ctx), id)
media, err := m.GetAttachmentByID(gtscontext.SetBarebones(ctx), id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// not an issue.
@ -119,11 +120,115 @@ func (m *mediaDB) DeleteAttachment(ctx context.Context, id string) error {
return err
}
// Finally delete media from DB.
_, err = m.conn.NewDelete().
TableExpr("? AS ?", bun.Ident("media_attachments"), bun.Ident("media_attachment")).
Where("? = ?", bun.Ident("media_attachment.id"), id).
Exec(ctx)
var (
invalidateAccount bool
invalidateStatus bool
)
// Delete media attachment in new transaction.
err = m.conn.RunInTx(ctx, func(tx bun.Tx) error {
if media.AccountID != "" {
var account gtsmodel.Account
// Get related account model.
if _, err := tx.NewSelect().
Model(&account).
Where("? = ?", bun.Ident("id"), media.AccountID).
Exec(ctx); err != nil && !errors.Is(err, db.ErrNoEntries) {
return gtserror.Newf("error selecting account: %w", err)
}
var set func(*bun.UpdateQuery) *bun.UpdateQuery
switch {
case *media.Avatar && account.AvatarMediaAttachmentID == id:
set = func(q *bun.UpdateQuery) *bun.UpdateQuery {
return q.Set("? = NULL", bun.Ident("avatar_media_attachment_id"))
}
case *media.Header && account.HeaderMediaAttachmentID == id:
set = func(q *bun.UpdateQuery) *bun.UpdateQuery {
return q.Set("? = NULL", bun.Ident("header_media_attachment_id"))
}
}
if set != nil {
// Note: this handles not found.
//
// Update the account model.
q := tx.NewUpdate().
Table("accounts").
Where("? = ?", bun.Ident("id"), account.ID)
if _, err := set(q).Exec(ctx); err != nil {
return gtserror.Newf("error updating account: %w", err)
}
// Mark as needing invalidate.
invalidateAccount = true
}
}
if media.StatusID != "" {
var status gtsmodel.Status
// Get related status model.
if _, err := tx.NewSelect().
Model(&status).
Where("? = ?", bun.Ident("id"), media.StatusID).
Exec(ctx); err != nil && !errors.Is(err, db.ErrNoEntries) {
return gtserror.Newf("error selecting status: %w", err)
}
// Get length of attachments beforehand.
before := len(status.AttachmentIDs)
for i := 0; i < len(status.AttachmentIDs); {
if status.AttachmentIDs[i] == id {
// Remove this reference to deleted attachment ID.
copy(status.AttachmentIDs[i:], status.AttachmentIDs[i+1:])
status.AttachmentIDs = status.AttachmentIDs[:len(status.AttachmentIDs)-1]
continue
}
i++
}
if before != len(status.AttachmentIDs) {
// Note: this accounts for status not found.
//
// Attachments changed, update the status.
if _, err := tx.NewUpdate().
Table("statuses").
Where("? = ?", bun.Ident("id"), status.ID).
Set("? = ?", bun.Ident("attachment_ids"), status.AttachmentIDs).
Exec(ctx); err != nil {
return gtserror.Newf("error updating status: %w", err)
}
// Mark as needing invalidate.
invalidateStatus = true
}
}
// Finally delete this media.
if _, err := tx.NewDelete().
Table("media_attachments").
Where("? = ?", bun.Ident("id"), id).
Exec(ctx); err != nil {
return gtserror.Newf("error deleting media: %w", err)
}
return nil
})
if invalidateAccount {
// The account for given ID will have been updated in transaction.
m.state.Caches.GTS.Account().Invalidate("ID", media.AccountID)
}
if invalidateStatus {
// The status for given ID will have been updated in transaction.
m.state.Caches.GTS.Status().Invalidate("ID", media.StatusID)
}
return m.conn.ProcessError(err)
}
@ -167,6 +272,29 @@ func (m *mediaDB) CountRemoteOlderThan(ctx context.Context, olderThan time.Time)
return count, nil
}
func (m *mediaDB) GetAttachments(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, error) {
attachmentIDs := []string{}
q := m.conn.NewSelect().
Table("media_attachments").
Column("id").
Order("id DESC")
if maxID != "" {
q = q.Where("? < ?", bun.Ident("id"), maxID)
}
if limit != 0 {
q = q.Limit(limit)
}
if err := q.Scan(ctx, &attachmentIDs); err != nil {
return nil, m.conn.ProcessError(err)
}
return m.GetAttachmentsByIDs(ctx, attachmentIDs)
}
func (m *mediaDB) GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
attachmentIDs := []string{}

11
internal/db/bundb/session.go
View file

@ -20,6 +20,7 @@ package bundb
import (
"context"
"crypto/rand"
"io"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -52,13 +53,11 @@ func (s *sessionDB) GetSession(ctx context.Context) (*gtsmodel.RouterSession, db
}
func (s *sessionDB) createSession(ctx context.Context) (*gtsmodel.RouterSession, db.Error) {
auth := make([]byte, 32)
crypt := make([]byte, 32)
buf := make([]byte, 64)
auth := buf[:32]
crypt := buf[32:64]
if _, err := rand.Read(auth); err != nil {
return nil, err
}
if _, err := rand.Read(crypt); err != nil {
if _, err := io.ReadFull(rand.Reader, buf); err != nil {
return nil, err
}