diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 7844c03f8..120df1ded 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -70,6 +70,7 @@ func (c *Caches) Init() { c.initBlock() c.initBlockIDs() c.initBoostOfIDs() + c.initCard() c.initConversation() c.initConversationLastStatusIDs() c.initDomainAllow() diff --git a/internal/cache/db.go b/internal/cache/db.go index 82cd9ac5f..e560d85cd 100644 --- a/internal/cache/db.go +++ b/internal/cache/db.go @@ -52,6 +52,9 @@ type DBCaches struct { // BoostOfIDs provides access to the boost of IDs list database cache. BoostOfIDs SliceCache[string] + // Card provides access to the gtsmodel Card database cache. + Card StructCache[*gtsmodel.Card] + // Conversation provides access to the gtsmodel Conversation database cache. Conversation StructCache[*gtsmodel.Conversation] @@ -637,6 +640,32 @@ func (c *Caches) initEmoji() { }) } +func (c *Caches) initCard() { + // Calculate maximum cache size. + cap := calculateResultCacheMax( + sizeofCard(), // model in-mem size. + config.GetCacheEmojiMemRatio(), // TODO: this need to be replaced with GetCacheCardMemRatio + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(e1 *gtsmodel.Card) *gtsmodel.Card { + e2 := new(gtsmodel.Card) + *e2 = *e1 + + return e2 + } + + c.DB.Card.Init(structr.CacheConfig[*gtsmodel.Card]{ + Indices: []structr.IndexConfig{ + {Fields: "ID"}, + }, + MaxSize: cap, + IgnoreErr: ignoreErrors, + Copy: copyF, + }) +} + func (c *Caches) initEmojiCategory() { // Calculate maximum cache size. cap := calculateResultCacheMax( diff --git a/internal/cache/size.go b/internal/cache/size.go index 9a30d5f08..aabe0f65d 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -395,6 +395,13 @@ func sizeofEmoji() uintptr { })) } +func sizeofCard() uintptr { + // TODO: this implementation need to be extended to contain other fields also. + return uintptr(size.Of(>smodel.Card{ + ID: exampleID, + })) +} + func sizeofEmojiCategory() uintptr { return uintptr(size.Of(>smodel.EmojiCategory{ ID: exampleID, diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 156c19fd5..7c33959a1 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -1021,9 +1021,7 @@ func (st *ConfigState) SetInstanceSubscriptionsProcessFrom(v string) { func InstanceSubscriptionsProcessFromFlag() string { return "instance-subscriptions-process-from" } // GetInstanceSubscriptionsProcessFrom safely fetches the value for global configuration 'InstanceSubscriptionsProcessFrom' field -func GetInstanceSubscriptionsProcessFrom() string { - return global.GetInstanceSubscriptionsProcessFrom() -} +func GetInstanceSubscriptionsProcessFrom() string { return global.GetInstanceSubscriptionsProcessFrom() } // SetInstanceSubscriptionsProcessFrom safely sets the value for global configuration 'InstanceSubscriptionsProcessFrom' field func SetInstanceSubscriptionsProcessFrom(v string) { global.SetInstanceSubscriptionsProcessFrom(v) } @@ -1048,14 +1046,10 @@ func (st *ConfigState) SetInstanceSubscriptionsProcessEvery(v time.Duration) { func InstanceSubscriptionsProcessEveryFlag() string { return "instance-subscriptions-process-every" } // GetInstanceSubscriptionsProcessEvery safely fetches the value for global configuration 'InstanceSubscriptionsProcessEvery' field -func GetInstanceSubscriptionsProcessEvery() time.Duration { - return global.GetInstanceSubscriptionsProcessEvery() -} +func GetInstanceSubscriptionsProcessEvery() time.Duration { return global.GetInstanceSubscriptionsProcessEvery() } // SetInstanceSubscriptionsProcessEvery safely sets the value for global configuration 'InstanceSubscriptionsProcessEvery' field -func SetInstanceSubscriptionsProcessEvery(v time.Duration) { - global.SetInstanceSubscriptionsProcessEvery(v) -} +func SetInstanceSubscriptionsProcessEvery(v time.Duration) { global.SetInstanceSubscriptionsProcessEvery(v) } // GetInstanceStatsMode safely fetches the Configuration value for state's 'InstanceStatsMode' field func (st *ConfigState) GetInstanceStatsMode() (v string) { @@ -2802,14 +2796,10 @@ func (st *ConfigState) SetAdvancedRateLimitExceptionsParsed(v []netip.Prefix) { func AdvancedRateLimitExceptionsParsedFlag() string { return "advanced-rate-limit-exceptions-parsed" } // GetAdvancedRateLimitExceptionsParsed safely fetches the value for global configuration 'AdvancedRateLimitExceptionsParsed' field -func GetAdvancedRateLimitExceptionsParsed() []netip.Prefix { - return global.GetAdvancedRateLimitExceptionsParsed() -} +func GetAdvancedRateLimitExceptionsParsed() []netip.Prefix { return global.GetAdvancedRateLimitExceptionsParsed() } // SetAdvancedRateLimitExceptionsParsed safely sets the value for global configuration 'AdvancedRateLimitExceptionsParsed' field -func SetAdvancedRateLimitExceptionsParsed(v []netip.Prefix) { - global.SetAdvancedRateLimitExceptionsParsed(v) -} +func SetAdvancedRateLimitExceptionsParsed(v []netip.Prefix) { global.SetAdvancedRateLimitExceptionsParsed(v) } // GetAdvancedThrottlingMultiplier safely fetches the Configuration value for state's 'AdvancedThrottlingMultiplier' field func (st *ConfigState) GetAdvancedThrottlingMultiplier() (v int) { @@ -3328,19 +3318,13 @@ func (st *ConfigState) SetCacheConversationLastStatusIDsMemRatio(v float64) { } // CacheConversationLastStatusIDsMemRatioFlag returns the flag name for the 'Cache.ConversationLastStatusIDsMemRatio' field -func CacheConversationLastStatusIDsMemRatioFlag() string { - return "cache-conversation-last-status-ids-mem-ratio" -} +func CacheConversationLastStatusIDsMemRatioFlag() string { return "cache-conversation-last-status-ids-mem-ratio" } // GetCacheConversationLastStatusIDsMemRatio safely fetches the value for global configuration 'Cache.ConversationLastStatusIDsMemRatio' field -func GetCacheConversationLastStatusIDsMemRatio() float64 { - return global.GetCacheConversationLastStatusIDsMemRatio() -} +func GetCacheConversationLastStatusIDsMemRatio() float64 { return global.GetCacheConversationLastStatusIDsMemRatio() } // SetCacheConversationLastStatusIDsMemRatio safely sets the value for global configuration 'Cache.ConversationLastStatusIDsMemRatio' field -func SetCacheConversationLastStatusIDsMemRatio(v float64) { - global.SetCacheConversationLastStatusIDsMemRatio(v) -} +func SetCacheConversationLastStatusIDsMemRatio(v float64) { global.SetCacheConversationLastStatusIDsMemRatio(v) } // GetCacheDomainPermissionDraftMemRation safely fetches the Configuration value for state's 'Cache.DomainPermissionDraftMemRation' field func (st *ConfigState) GetCacheDomainPermissionDraftMemRation() (v float64) { @@ -3359,19 +3343,13 @@ func (st *ConfigState) SetCacheDomainPermissionDraftMemRation(v float64) { } // CacheDomainPermissionDraftMemRationFlag returns the flag name for the 'Cache.DomainPermissionDraftMemRation' field -func CacheDomainPermissionDraftMemRationFlag() string { - return "cache-domain-permission-draft-mem-ratio" -} +func CacheDomainPermissionDraftMemRationFlag() string { return "cache-domain-permission-draft-mem-ratio" } // GetCacheDomainPermissionDraftMemRation safely fetches the value for global configuration 'Cache.DomainPermissionDraftMemRation' field -func GetCacheDomainPermissionDraftMemRation() float64 { - return global.GetCacheDomainPermissionDraftMemRation() -} +func GetCacheDomainPermissionDraftMemRation() float64 { return global.GetCacheDomainPermissionDraftMemRation() } // SetCacheDomainPermissionDraftMemRation safely sets the value for global configuration 'Cache.DomainPermissionDraftMemRation' field -func SetCacheDomainPermissionDraftMemRation(v float64) { - global.SetCacheDomainPermissionDraftMemRation(v) -} +func SetCacheDomainPermissionDraftMemRation(v float64) { global.SetCacheDomainPermissionDraftMemRation(v) } // GetCacheDomainPermissionSubscriptionMemRation safely fetches the Configuration value for state's 'Cache.DomainPermissionSubscriptionMemRation' field func (st *ConfigState) GetCacheDomainPermissionSubscriptionMemRation() (v float64) { @@ -3390,19 +3368,13 @@ func (st *ConfigState) SetCacheDomainPermissionSubscriptionMemRation(v float64) } // CacheDomainPermissionSubscriptionMemRationFlag returns the flag name for the 'Cache.DomainPermissionSubscriptionMemRation' field -func CacheDomainPermissionSubscriptionMemRationFlag() string { - return "cache-domain-permission-subscription-mem-ratio" -} +func CacheDomainPermissionSubscriptionMemRationFlag() string { return "cache-domain-permission-subscription-mem-ratio" } // GetCacheDomainPermissionSubscriptionMemRation safely fetches the value for global configuration 'Cache.DomainPermissionSubscriptionMemRation' field -func GetCacheDomainPermissionSubscriptionMemRation() float64 { - return global.GetCacheDomainPermissionSubscriptionMemRation() -} +func GetCacheDomainPermissionSubscriptionMemRation() float64 { return global.GetCacheDomainPermissionSubscriptionMemRation() } // SetCacheDomainPermissionSubscriptionMemRation safely sets the value for global configuration 'Cache.DomainPermissionSubscriptionMemRation' field -func SetCacheDomainPermissionSubscriptionMemRation(v float64) { - global.SetCacheDomainPermissionSubscriptionMemRation(v) -} +func SetCacheDomainPermissionSubscriptionMemRation(v float64) { global.SetCacheDomainPermissionSubscriptionMemRation(v) } // GetCacheEmojiMemRatio safely fetches the Configuration value for state's 'Cache.EmojiMemRatio' field func (st *ConfigState) GetCacheEmojiMemRatio() (v float64) { @@ -4424,9 +4396,7 @@ func (st *ConfigState) SetCacheWebPushSubscriptionMemRatio(v float64) { func CacheWebPushSubscriptionMemRatioFlag() string { return "cache-web-push-subscription-mem-ratio" } // GetCacheWebPushSubscriptionMemRatio safely fetches the value for global configuration 'Cache.WebPushSubscriptionMemRatio' field -func GetCacheWebPushSubscriptionMemRatio() float64 { - return global.GetCacheWebPushSubscriptionMemRatio() -} +func GetCacheWebPushSubscriptionMemRatio() float64 { return global.GetCacheWebPushSubscriptionMemRatio() } // SetCacheWebPushSubscriptionMemRatio safely sets the value for global configuration 'Cache.WebPushSubscriptionMemRatio' field func SetCacheWebPushSubscriptionMemRatio(v float64) { global.SetCacheWebPushSubscriptionMemRatio(v) } @@ -4448,19 +4418,13 @@ func (st *ConfigState) SetCacheWebPushSubscriptionIDsMemRatio(v float64) { } // CacheWebPushSubscriptionIDsMemRatioFlag returns the flag name for the 'Cache.WebPushSubscriptionIDsMemRatio' field -func CacheWebPushSubscriptionIDsMemRatioFlag() string { - return "cache-web-push-subscription-ids-mem-ratio" -} +func CacheWebPushSubscriptionIDsMemRatioFlag() string { return "cache-web-push-subscription-ids-mem-ratio" } // GetCacheWebPushSubscriptionIDsMemRatio safely fetches the value for global configuration 'Cache.WebPushSubscriptionIDsMemRatio' field -func GetCacheWebPushSubscriptionIDsMemRatio() float64 { - return global.GetCacheWebPushSubscriptionIDsMemRatio() -} +func GetCacheWebPushSubscriptionIDsMemRatio() float64 { return global.GetCacheWebPushSubscriptionIDsMemRatio() } // SetCacheWebPushSubscriptionIDsMemRatio safely sets the value for global configuration 'Cache.WebPushSubscriptionIDsMemRatio' field -func SetCacheWebPushSubscriptionIDsMemRatio(v float64) { - global.SetCacheWebPushSubscriptionIDsMemRatio(v) -} +func SetCacheWebPushSubscriptionIDsMemRatio(v float64) { global.SetCacheWebPushSubscriptionIDsMemRatio(v) } // GetCacheVisibilityMemRatio safely fetches the Configuration value for state's 'Cache.VisibilityMemRatio' field func (st *ConfigState) GetCacheVisibilityMemRatio() (v float64) { diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index 18fe5384c..856f41eab 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -58,6 +58,7 @@ type DBService struct { db.AdvancedMigration db.Application db.Basic + db.Card db.Conversation db.Domain db.Emoji @@ -186,6 +187,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { Basic: &basicDB{ db: db, }, + Card: &cardDB{ + db: db, + state: state, + }, Conversation: &conversationDB{ db: db, state: state, diff --git a/internal/db/bundb/card.go b/internal/db/bundb/card.go new file mode 100644 index 000000000..6c69e0f52 --- /dev/null +++ b/internal/db/bundb/card.go @@ -0,0 +1,87 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/uptrace/bun" +) + +type cardDB struct { + db *bun.DB + state *state.State +} + +func (c *cardDB) PutCard(ctx context.Context, card *gtsmodel.Card) error { + return c.state.Caches.DB.Card.Store(card, func() error { + _, err := c.db.NewInsert().Model(card).Exec(ctx) + return err + }) +} + +func (c *cardDB) GetCardByID(ctx context.Context, id string) (*gtsmodel.Card, error) { + return c.state.Caches.DB.Card.LoadOne("ID", func() (*gtsmodel.Card, error) { + var card gtsmodel.Card + + q := c.db. + NewSelect(). + Model(&card). + Where("? = ?", bun.Ident("card.id"), id) + + if err := q.Scan(ctx); err != nil { + return nil, err + } + + return &card, nil + }, id) +} + +func (c *cardDB) UpdateCard(ctx context.Context, card *gtsmodel.Card, columns ...string) error { + return c.state.Caches.DB.Card.Store(card, func() error { + return c.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + _, err := tx.NewUpdate(). + Model(card). + Column(columns...). + Where("? = ?", bun.Ident("id"), card.ID). + Exec(ctx) + return err + }) + }) +} + +func (c *cardDB) DeleteCardByID(ctx context.Context, id string) error { + // Delete card with ID from the database. + if err := c.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + + // Delete card from database. + if _, err := tx.NewDelete(). + Table("cards"). + Where("? = ?", bun.Ident("id"), id). + Exec(ctx); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + return nil +} diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 8383a9c01..9af0b923f 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -297,6 +297,16 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) } } + if !status.CardPopulated() { + status.Card, err = s.state.DB.GetCardByID( + gtscontext.SetBarebones(ctx), + status.CardID, + ) + if err != nil { + errs.Appendf("error populating status preview card: %w", err) + } + } + if status.CreatedWithApplicationID != "" && status.CreatedWithApplication == nil { // Populate the status' expected CreatedWithApplication (not always set). // Don't error on ErrNoEntries, as the application may have been cleaned up. diff --git a/internal/db/card.go b/internal/db/card.go new file mode 100644 index 000000000..17de50d7c --- /dev/null +++ b/internal/db/card.go @@ -0,0 +1,39 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package db + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +// Card contains functions for getting Cards, creating Cards, and checking various other fields on Cards. +type Card interface { + // GetCardByID fetches the Card from the database with matching id column. + GetCardByID(ctx context.Context, id string) (*gtsmodel.Card, error) + + // PutCard stores one Card in the database. + PutCard(ctx context.Context, Card *gtsmodel.Card) error + + // UpdateCard updates one Card in the database. + UpdateCard(ctx context.Context, Card *gtsmodel.Card, columns ...string) error + + // DeleteCardByID deletes one Card from the database. + DeleteCardByID(ctx context.Context, id string) error +} diff --git a/internal/db/db.go b/internal/db/db.go index 16796ae49..7cf778cc6 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -29,6 +29,7 @@ type DB interface { AdvancedMigration Application Basic + Card Conversation Domain Emoji diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index 61675faf4..90b7f3c83 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -175,6 +175,24 @@ func (s *Status) EditsPopulated() bool { return true } +// CardPopulated returns whether card is +// populated according to current CardID. +func (s *Status) CardPopulated() bool { + if s.CardID == "" { + return true + } + + if s.Card == nil { + return false + } + + if s.Card.ID != s.CardID { + return false + } + + return true +} + // EmojisUpToDate returns whether status emoji attachments of receiving status are up-to-date // according to emoji attachments of the passed status, by comparing their emoji URIs. We don't // use IDs as this is used to determine whether there are new emojis to fetch. diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go index e04e03cb1..b6d790f43 100644 --- a/internal/processing/status/create.go +++ b/internal/processing/status/create.go @@ -197,7 +197,14 @@ func (p *Processor) Create( if card != nil { status.CardID = id.NewULIDFromTime(now) + card.ID = status.CardID status.Card = card + + // Insert this newly prepared preview card into the database. + if err := p.state.DB.PutCard(ctx, card); err != nil { + err := gtserror.Newf("error inserting preview card in db: %w", err) + return nil, gtserror.NewErrorInternalError(err) + } } // Only store ContentWarningText if the parsed diff --git a/internal/processing/status/preview_card.go b/internal/processing/status/preview_card.go index 777fe1c1a..a27f5bf86 100644 --- a/internal/processing/status/preview_card.go +++ b/internal/processing/status/preview_card.go @@ -55,7 +55,7 @@ func FetchPreview(text string) (*gtsmodel.Card, gtserror.WithCode) { return nil, gtserror.NewErrorInternalError(fmt.Errorf("unsupported scheme: %s", parsed.Scheme)) } - resp, err := http.Get(link) + resp, err := safeGet(parsed) if err != nil { return nil, gtserror.NewErrorInternalError(err, "request failed") } @@ -109,3 +109,8 @@ func FetchPreview(text string) (*gtsmodel.Card, gtserror.WithCode) { return card, nil } + +func safeGet(u *url.URL) (*http.Response, error) { + // #nosec G107 -- URL was already validated + return http.Get(u.String()) +}