add support for status edits in the database, and update status dereferencer to handle them

This commit is contained in:
kim 2024-11-13 13:51:03 +00:00
commit 3e131f5da6
21 changed files with 623 additions and 172 deletions

View file

@ -187,6 +187,7 @@ type Accountable interface {
WithEndpoints
WithTag
WithPublished
WithUpdated
}
// Statusable represents the minimum activitypub interface for representing a 'status'.

View file

@ -105,6 +105,7 @@ func (c *Caches) Init() {
c.initStatus()
c.initStatusBookmark()
c.initStatusBookmarkIDs()
c.initStatusEdit()
c.initStatusFave()
c.initStatusFaveIDs()
c.initTag()

35
internal/cache/db.go vendored
View file

@ -226,6 +226,9 @@ type DBCaches struct {
// StatusBookmarkIDs provides access to the status bookmark IDs list database cache.
StatusBookmarkIDs SliceCache[string]
// StatusEdit provides access to the gtsmodel StatusEdit database cache.
StatusEdit StructCache[*gtsmodel.StatusEdit]
// StatusFave provides access to the gtsmodel StatusFave database cache.
StatusFave StructCache[*gtsmodel.StatusFave]
@ -1385,6 +1388,38 @@ func (c *Caches) initStatusBookmarkIDs() {
c.DB.StatusBookmarkIDs.Init(0, cap)
}
func (c *Caches) initStatusEdit() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(
sizeofStatusEdit(), // model in-mem size.
config.GetCacheStatusEditMemRatio(),
)
log.Infof(nil, "cache size = %d", cap)
copyF := func(s1 *gtsmodel.StatusEdit) *gtsmodel.StatusEdit {
s2 := new(gtsmodel.StatusEdit)
*s2 = *s1
// Don't include ptr fields that
// will be populated separately.
s2.Attachments = nil
return s2
}
c.DB.StatusEdit.Init(structr.CacheConfig[*gtsmodel.StatusEdit]{
Indices: []structr.IndexConfig{
{Fields: "ID"},
{Fields: "StatusID", Multiple: true},
},
MaxSize: cap,
IgnoreErr: ignoreErrors,
Copy: copyF,
Invalidate: c.OnInvalidateStatusEdit,
})
}
func (c *Caches) initStatusFave() {
// Calculate maximum cache size.
cap := calculateResultCacheMax(

View file

@ -273,6 +273,11 @@ func (c *Caches) OnInvalidateStatusBookmark(bookmark *gtsmodel.StatusBookmark) {
c.DB.StatusBookmarkIDs.Invalidate(bookmark.StatusID)
}
func (c *Caches) OnInvalidateStatusEdit(edit *gtsmodel.StatusEdit) {
// Invalidate cache of related status model.
c.DB.Status.Invalidate("ID", edit.StatusID)
}
func (c *Caches) OnInvalidateStatusFave(fave *gtsmodel.StatusFave) {
// Invalidate status fave ID list for this status.
c.DB.StatusFaveIDs.Invalidate(fave.StatusID)

View file

@ -674,6 +674,23 @@ func sizeofStatusBookmark() uintptr {
}))
}
func sizeofStatusEdit() uintptr {
return uintptr(size.Of(&gtsmodel.StatusEdit{
ID: exampleID,
Content: exampleText,
ContentWarning: exampleUsername, // similar length
Text: exampleText,
Language: "en",
Sensitive: func() *bool { ok := false; return &ok }(),
AttachmentIDs: []string{exampleID, exampleID, exampleID},
Attachments: nil,
PollOptions: []string{exampleTextSmall, exampleTextSmall, exampleTextSmall, exampleTextSmall},
PollVotes: []int{69, 420, 1337, 1969},
StatusID: exampleID,
CreatedAt: exampleTime,
}))
}
func sizeofStatusFave() uintptr {
return uintptr(size.Of(&gtsmodel.StatusFave{
ID: exampleID,

View file

@ -238,6 +238,7 @@ type CacheConfiguration struct {
StatusMemRatio float64 `name:"status-mem-ratio"`
StatusBookmarkMemRatio float64 `name:"status-bookmark-mem-ratio"`
StatusBookmarkIDsMemRatio float64 `name:"status-bookmark-ids-mem-ratio"`
StatusEditMemRatio float64 `name:"status-edit-mem-ratio"`
StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"`
StatusFaveIDsMemRatio float64 `name:"status-fave-ids-mem-ratio"`
TagMemRatio float64 `name:"tag-mem-ratio"`

View file

@ -3912,6 +3912,31 @@ func GetCacheStatusBookmarkIDsMemRatio() float64 { return global.GetCacheStatusB
// SetCacheStatusBookmarkIDsMemRatio safely sets the value for global configuration 'Cache.StatusBookmarkIDsMemRatio' field
func SetCacheStatusBookmarkIDsMemRatio(v float64) { global.SetCacheStatusBookmarkIDsMemRatio(v) }
// GetCacheStatusEditMemRatio safely fetches the Configuration value for state's 'Cache.StatusEditMemRatio' field
func (st *ConfigState) GetCacheStatusEditMemRatio() (v float64) {
st.mutex.RLock()
v = st.config.Cache.StatusEditMemRatio
st.mutex.RUnlock()
return
}
// SetCacheStatusEditMemRatio safely sets the Configuration value for state's 'Cache.StatusEditMemRatio' field
func (st *ConfigState) SetCacheStatusEditMemRatio(v float64) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.StatusEditMemRatio = v
st.reloadToViper()
}
// CacheStatusEditMemRatioFlag returns the flag name for the 'Cache.StatusEditMemRatio' field
func CacheStatusEditMemRatioFlag() string { return "cache-status-edit-mem-ratio" }
// GetCacheStatusEditMemRatio safely fetches the value for global configuration 'Cache.StatusEditMemRatio' field
func GetCacheStatusEditMemRatio() float64 { return global.GetCacheStatusEditMemRatio() }
// SetCacheStatusEditMemRatio safely sets the value for global configuration 'Cache.StatusEditMemRatio' field
func SetCacheStatusEditMemRatio(v float64) { global.SetCacheStatusEditMemRatio(v) }
// GetCacheStatusFaveMemRatio safely fetches the Configuration value for state's 'Cache.StatusFaveMemRatio' field
func (st *ConfigState) GetCacheStatusFaveMemRatio() (v float64) {
st.mutex.RLock()

View file

@ -81,6 +81,7 @@ type DBService struct {
db.SinBinStatus
db.Status
db.StatusBookmark
db.StatusEdit
db.StatusFave
db.Tag
db.Thread
@ -272,6 +273,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
db: db,
state: state,
},
StatusEdit: &statusEditDB{
db: db,
state: state,
},
StatusFave: &statusFaveDB{
db: db,
state: state,

View file

@ -181,13 +181,17 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g
func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) error {
var (
err error
errs = gtserror.NewMultiError(9)
errs gtserror.MultiError
)
// For sub-models we only want
// barebones versions of them.
ctx = gtscontext.SetBarebones(ctx)
if status.Account == nil {
// Status author is not set, fetch from database.
status.Account, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
ctx,
status.AccountID,
)
if err != nil {
@ -199,7 +203,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
if status.InReplyTo == nil {
// Status parent is not set, fetch from database.
status.InReplyTo, err = s.GetStatusByID(
gtscontext.SetBarebones(ctx),
ctx,
status.InReplyToID,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
@ -210,7 +214,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
if status.InReplyToAccount == nil {
// Status parent author is not set, fetch from database.
status.InReplyToAccount, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
ctx,
status.InReplyToAccountID,
)
if err != nil {
@ -223,7 +227,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
if status.BoostOf == nil {
// Status boost is not set, fetch from database.
status.BoostOf, err = s.GetStatusByID(
gtscontext.SetBarebones(ctx),
ctx,
status.BoostOfID,
)
if err != nil {
@ -234,7 +238,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
if status.BoostOfAccount == nil {
// Status boost author is not set, fetch from database.
status.BoostOfAccount, err = s.state.DB.GetAccountByID(
gtscontext.SetBarebones(ctx),
ctx,
status.BoostOfAccountID,
)
if err != nil {
@ -246,7 +250,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
if status.PollID != "" && status.Poll == nil {
// Status poll is not set, fetch from database.
status.Poll, err = s.state.DB.GetPollByID(
gtscontext.SetBarebones(ctx),
ctx,
status.PollID,
)
if err != nil {
@ -257,7 +261,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
if !status.AttachmentsPopulated() {
// Status attachments are out-of-date with IDs, repopulate.
status.Attachments, err = s.state.DB.GetAttachmentsByIDs(
ctx, // these are already barebones
ctx,
status.AttachmentIDs,
)
if err != nil {
@ -279,7 +283,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
if !status.MentionsPopulated() {
// Status mentions are out-of-date with IDs, repopulate.
status.Mentions, err = s.state.DB.GetMentions(
ctx, // leave fully populated for now
ctx,
status.MentionIDs,
)
if err != nil {
@ -290,7 +294,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
if !status.EmojisPopulated() {
// Status emojis are out-of-date with IDs, repopulate.
status.Emojis, err = s.state.DB.GetEmojisByIDs(
ctx, // these are already barebones
ctx,
status.EmojiIDs,
)
if err != nil {
@ -298,10 +302,21 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
}
}
if !status.EditsPopulated() {
// Status edits are out-of-date with IDs, repopulate.
status.Edits, err = s.state.DB.GetStatusEditsByIDs(
ctx,
status.EditIDs,
)
if err != nil {
errs.Appendf("error populating status edits: %w", err)
}
}
if status.CreatedWithApplicationID != "" && status.CreatedWithApplication == nil {
// Populate the status' expected CreatedWithApplication (not always set).
status.CreatedWithApplication, err = s.state.DB.GetApplicationByID(
ctx, // these are already barebones
ctx,
status.CreatedWithApplicationID,
)
if err != nil {

View file

@ -0,0 +1,147 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bundb
import (
"context"
"slices"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/util"
"github.com/uptrace/bun"
)
type statusEditDB struct {
db *bun.DB
state *state.State
}
func (s *statusEditDB) GetStatusEditByID(ctx context.Context, id string) (*gtsmodel.StatusEdit, error) {
// Fetch edit from database cache with loader callback.
edit, err := s.state.Caches.DB.StatusEdit.LoadOne("ID",
func() (*gtsmodel.StatusEdit, error) {
var edit gtsmodel.StatusEdit
// Not cached, load edit
// from database by its ID.
if err := s.db.NewSelect().
Model(&edit).
Where("? = ?", bun.Ident("id"), id).
Scan(ctx); err != nil {
return nil, err
}
return &edit, nil
},
)
if err != nil {
return nil, err
}
if gtscontext.Barebones(ctx) {
// no need to fully populate.
return edit, nil
}
// Further populate the edit fields where applicable.
if err := s.PopulateStatusEdit(ctx, edit); err != nil {
return nil, err
}
return edit, nil
}
func (s *statusEditDB) GetStatusEditsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.StatusEdit, error) {
// Load status edits for IDs via cache loader callbacks.
edits, err := s.state.Caches.DB.StatusEdit.LoadIDs("ID",
ids,
func(uncached []string) ([]*gtsmodel.StatusEdit, error) {
// Preallocate expected length of uncached edits.
edits := make([]*gtsmodel.StatusEdit, 0, len(uncached))
// Perform database query scanning
// the remaining (uncached) edit IDs.
if err := s.db.NewSelect().
Model(&edits).
Where("? IN (?)", bun.Ident("id"), bun.In(uncached)).
Scan(ctx); err != nil {
return nil, err
}
return edits, nil
},
)
if err != nil {
return nil, err
}
// Reorder the edits by their
// IDs to ensure in correct order.
getID := func(e *gtsmodel.StatusEdit) string { return e.ID }
util.OrderBy(edits, ids, getID)
if gtscontext.Barebones(ctx) {
// no need to fully populate.
return edits, nil
}
// Populate all loaded edits, removing those we fail to
// populate (removes needing so many nil checks everywhere).
edits = slices.DeleteFunc(edits, func(edit *gtsmodel.StatusEdit) bool {
if err := s.PopulateStatusEdit(ctx, edit); err != nil {
log.Errorf(ctx, "error populating edit %s: %v", edit.ID, err)
return true
}
return false
})
return edits, nil
}
func (s *statusEditDB) PopulateStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error {
var err error
var errs gtserror.MultiError
// For sub-models we only want
// barebones versions of them.
ctx = gtscontext.SetBarebones(ctx)
if !edit.AttachmentsPopulated() {
// Fetch all attachments for status edit's IDs.
edit.Attachments, err = s.state.DB.GetAttachmentsByIDs(
ctx,
edit.AttachmentIDs,
)
if err != nil {
errs.Appendf("error populating edit attachments: %w", err)
}
}
return errs.Combine()
}
func (s *statusEditDB) PutStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error {
return s.state.Caches.DB.StatusEdit.Store(edit, func() error {
_, err := s.db.NewInsert().Model(edit).Exec(ctx)
return err
})
}

View file

@ -51,6 +51,7 @@ type DB interface {
SinBinStatus
Status
StatusBookmark
StatusEdit
StatusFave
Tag
Thread

39
internal/db/statusedit.go Normal file
View file

@ -0,0 +1,39 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package db
import (
"context"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
type StatusEdit interface {
// GetStatusEditByID ...
GetStatusEditByID(ctx context.Context, id string) (*gtsmodel.StatusEdit, error)
// GetStatusEditsByIDs ...
GetStatusEditsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.StatusEdit, error)
// PopulateStatusEdit ...
PopulateStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error
// PutStatusEdit ...
PutStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error
}

View file

@ -87,7 +87,7 @@ func (d *Dereferencer) EnrichAnnounce(
boost.Federated = target.Federated
// Ensure this Announce is permitted by the Announcee.
permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost)
permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost, true)
if err != nil {
return nil, gtserror.Newf("error checking permitted status %s: %w", boost.URI, err)
}
@ -99,10 +99,7 @@ func (d *Dereferencer) EnrichAnnounce(
}
// Generate an ID for the boost wrapper status.
boost.ID, err = id.NewULIDFromTime(boost.CreatedAt)
if err != nil {
return nil, gtserror.Newf("error generating id: %w", err)
}
boost.ID = id.NewULIDFromTime(boost.CreatedAt)
// Store the boost wrapper status in database.
switch err = d.state.DB.PutStatus(ctx, boost); {

View file

@ -302,6 +302,7 @@ func (d *Dereferencer) enrichStatusSafely(
uri,
status,
statusable,
isNew,
)
// Check for a returned HTTP code via error.
@ -374,6 +375,7 @@ func (d *Dereferencer) enrichStatus(
uri *url.URL,
status *gtsmodel.Status,
statusable ap.Statusable,
isNew bool,
) (
*gtsmodel.Status,
ap.Statusable,
@ -476,8 +478,7 @@ func (d *Dereferencer) enrichStatus(
// Ensure the final parsed status URI or URL matches
// the input URI we fetched (or received) it as.
matches, err := util.URIMatches(
uri,
matches, err := util.URIMatches(uri,
append(
ap.GetURL(statusable), // status URL(s)
ap.GetJSONLDId(statusable), // status URI
@ -497,19 +498,10 @@ func (d *Dereferencer) enrichStatus(
)
}
var isNew bool
// Based on the original provided
// status model, determine whether
// this is a new insert / update.
if isNew = (status.ID == ""); isNew {
if isNew {
// Generate new status ID from the provided creation date.
latestStatus.ID, err = id.NewULIDFromTime(latestStatus.CreatedAt)
if err != nil {
log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err)
latestStatus.ID = id.NewULID() // just use "now"
}
latestStatus.ID = id.NewULIDFromTime(latestStatus.CreatedAt)
} else {
// Reuse existing status ID.
@ -538,8 +530,9 @@ func (d *Dereferencer) enrichStatus(
}
// Check if this is a permitted status we should accept.
// Function also sets "PendingApproval" bool as necessary.
permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus)
// Function also sets "PendingApproval" bool as necessary,
// and handles removal of existing statuses no longer permitted.
permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus, isNew)
if err != nil {
return nil, nil, gtserror.Newf("error checking permissibility for status %s: %w", uri, err)
}
@ -555,11 +548,6 @@ func (d *Dereferencer) enrichStatus(
return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err)
}
// Ensure the status' poll remains consistent, else reset the poll.
if err := d.fetchStatusPoll(ctx, status, latestStatus); err != nil {
return nil, nil, gtserror.Newf("error populating poll for status %s: %w", uri, err)
}
// Now that we know who this status replies to (handled by ASStatusToStatus)
// and who it mentions, we can add a ThreadID to it if necessary.
if err := d.threadStatus(ctx, latestStatus); err != nil {
@ -582,27 +570,28 @@ func (d *Dereferencer) enrichStatus(
}
if isNew {
// This is new, put the status in the database.
err := d.state.DB.PutStatus(ctx, latestStatus)
if err != nil {
return nil, nil, gtserror.Newf("error putting in database: %w", err)
// This is a new status, insert it into the database.
if err := d.insertStatus(ctx, latestStatus); err != nil {
return nil, nil, gtserror.Newf("error inserting new status %s: %w", uri, err)
}
} else {
// This is an existing status, update the model in the database.
if err := d.state.DB.UpdateStatus(ctx, latestStatus); err != nil {
return nil, nil, gtserror.Newf("error updating database: %w", err)
if err := d.updateStatus(ctx, status, latestStatus); err != nil {
return nil, nil, gtserror.Newf("error updating existing status %s: %w", uri, err)
}
}
return latestStatus, statusable, nil
}
// fetchStatusMentions ...
func (d *Dereferencer) fetchStatusMentions(
ctx context.Context,
requestUser string,
existing *gtsmodel.Status,
status *gtsmodel.Status,
) error {
// Allocate new slice to take the yet-to-be created mention IDs.
status.MentionIDs = make([]string, len(status.Mentions))
@ -637,11 +626,7 @@ func (d *Dereferencer) fetchStatusMentions(
// Generate new ID according to status creation.
// TODO: update this to use "edited_at" when we add
// support for edited status revision history.
mention.ID, err = id.NewULIDFromTime(status.CreatedAt)
if err != nil {
log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err)
mention.ID = id.NewULID() // just use "now"
}
mention.ID = id.NewULIDFromTime(status.CreatedAt)
// Set known further mention details.
mention.CreatedAt = status.CreatedAt
@ -681,7 +666,9 @@ func (d *Dereferencer) fetchStatusMentions(
return nil
}
// threadStatus ...
func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status) error {
if status.InReplyTo != nil {
if parentThreadID := status.InReplyTo.ThreadID; parentThreadID != "" {
// Simplest case: parent status
@ -719,24 +706,27 @@ func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status
// it to the status.
threadID := id.NewULID()
if err := d.state.DB.PutThread(
ctx,
&gtsmodel.Thread{
ID: threadID,
},
// ...
if err := d.state.DB.PutThread(ctx,
&gtsmodel.Thread{ID: threadID},
); err != nil {
return gtserror.Newf("error inserting new thread in db: %w", err)
}
// Set thread on latest status.
status.ThreadID = threadID
return nil
}
// fetchStatusTags populates the tags on 'status', fetching existing
// from the database and creating new where needed. 'existing' is used
// to fetch tags that have not changed since previous stored status.
func (d *Dereferencer) fetchStatusTags(
ctx context.Context,
existing *gtsmodel.Status,
status *gtsmodel.Status,
) error {
// Allocate new slice to take the yet-to-be determined tag IDs.
status.TagIDs = make([]string, len(status.Tags))
@ -791,103 +781,14 @@ func (d *Dereferencer) fetchStatusTags(
return nil
}
func (d *Dereferencer) fetchStatusPoll(
ctx context.Context,
existing *gtsmodel.Status,
status *gtsmodel.Status,
) error {
var (
// insertStatusPoll generates ID and inserts the poll attached to status into the database.
insertStatusPoll = func(ctx context.Context, status *gtsmodel.Status) error {
var err error
// Generate new ID for poll from the status CreatedAt.
// TODO: update this to use "edited_at" when we add
// support for edited status revision history.
status.Poll.ID, err = id.NewULIDFromTime(status.CreatedAt)
if err != nil {
log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err)
status.Poll.ID = id.NewULID() // just use "now"
}
// Update the status<->poll links.
status.PollID = status.Poll.ID
status.Poll.StatusID = status.ID
status.Poll.Status = status
// Insert this latest poll into the database.
err = d.state.DB.PutPoll(ctx, status.Poll)
if err != nil {
return gtserror.Newf("error putting in database: %w", err)
}
return nil
}
// deleteStatusPoll deletes the poll with ID, and all attached votes, from the database.
deleteStatusPoll = func(ctx context.Context, pollID string) error {
if err := d.state.DB.DeletePollByID(ctx, pollID); err != nil {
return gtserror.Newf("error deleting existing poll from database: %w", err)
}
return nil
}
)
switch {
case existing.Poll == nil && status.Poll == nil:
// no poll before or after, nothing to do.
return nil
case existing.Poll == nil && status.Poll != nil:
// no previous poll, insert new poll!
return insertStatusPoll(ctx, status)
case status.Poll == nil:
// existing poll has been deleted, remove this.
return deleteStatusPoll(ctx, existing.PollID)
case pollChanged(existing.Poll, status.Poll):
// poll has changed since original, delete and reinsert new.
if err := deleteStatusPoll(ctx, existing.PollID); err != nil {
return err
}
return insertStatusPoll(ctx, status)
case pollUpdated(existing.Poll, status.Poll):
// Since we last saw it, the poll has updated!
// Whether that be stats, or close time.
poll := existing.Poll
poll.Closing = pollJustClosed(existing.Poll, status.Poll)
poll.ClosedAt = status.Poll.ClosedAt
poll.Voters = status.Poll.Voters
poll.Votes = status.Poll.Votes
// Update poll model in the database (specifically only the possible changed columns).
if err := d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil {
return gtserror.Newf("error updating poll: %w", err)
}
// Update poll on status.
status.PollID = poll.ID
status.Poll = poll
return nil
default:
// latest and existing
// polls are up to date.
poll := existing.Poll
status.PollID = poll.ID
status.Poll = poll
return nil
}
}
// fetchStatusAttachments ...
func (d *Dereferencer) fetchStatusAttachments(
ctx context.Context,
requestUser string,
existing *gtsmodel.Status,
status *gtsmodel.Status,
) error {
// Allocate new slice to take the yet-to-be fetched attachment IDs.
status.AttachmentIDs = make([]string, len(status.Attachments))
@ -916,8 +817,7 @@ func (d *Dereferencer) fetchStatusAttachments(
}
// Load this new media attachment.
attachment, err := d.GetMedia(
ctx,
attachment, err := d.GetMedia(ctx,
requestUser,
status.AccountID,
placeholder.RemoteURL,
@ -958,11 +858,13 @@ func (d *Dereferencer) fetchStatusAttachments(
return nil
}
// fetchStatusEmojis ...
func (d *Dereferencer) fetchStatusEmojis(
ctx context.Context,
existing *gtsmodel.Status,
status *gtsmodel.Status,
) error {
// Fetch the updated emojis for our status.
emojis, changed, err := d.fetchEmojis(ctx,
existing.Emojis,
@ -991,6 +893,175 @@ func (d *Dereferencer) fetchStatusEmojis(
return nil
}
// insertStatus handles the insert of a new status into the database, inserting new poll if necessary.
func (d *Dereferencer) insertStatus(ctx context.Context, status *gtsmodel.Status) error {
if status.Poll != nil {
// Insert this poll attached to status in the database.
if err := d.insertStatusPoll(ctx, status); err != nil {
return err
}
}
// Insert new status into the database.
err := d.state.DB.PutStatus(ctx, status)
if err != nil {
return gtserror.Newf("error putting status in database: %w", err)
}
return nil
}
// updateStatus handles the updating of an existing status in the
// database, handling any required poll changes and / or staus edits.
func (d *Dereferencer) updateStatus(
ctx context.Context,
existing *gtsmodel.Status,
status *gtsmodel.Status,
) error {
// Handle any changes in status poll from existing to new.
pollChanged, err := d.updateStatusPoll(ctx, existing, status)
if err != nil {
return err
}
// If there was no change to poll, look for
// any other changes in status content itself.
if pollChanged || statusChanged(existing, status) {
// Status has been editted since last
// we saw it, take snapshot of existing.
var edit gtsmodel.StatusEdit
edit.ID = id.NewULIDFromTime(status.UpdatedAt)
edit.Content = existing.Content
edit.ContentWarning = existing.ContentWarning
edit.Text = existing.Text
edit.Language = existing.Language
edit.Sensitive = existing.Sensitive
edit.AttachmentIDs = existing.AttachmentIDs
edit.Attachments = existing.Attachments
// Edit creation is last update time.
edit.CreatedAt = existing.UpdatedAt
if existing.Poll != nil {
// Poll only set if existing contained them.
edit.PollOptions = existing.Poll.Options
if pollChanged {
// Votes are only set if the poll
// itself changed during this edit.
edit.PollVotes = existing.Poll.Votes
}
}
// Insert this new edit of existing status into database.
if err := d.state.DB.PutStatusEdit(ctx, &edit); err != nil {
return gtserror.Newf("error putting edit in database: %w", err)
}
// Add edit to list of edits on the status.
status.EditIDs = append(status.EditIDs, edit.ID)
status.Edits = append(status.Edits, &edit)
}
// Update the existing status in database with new details.
if err := d.state.DB.UpdateStatus(ctx, status); err != nil {
return gtserror.Newf("error updating status in database: %w", err)
}
return nil
}
// updateStatusPoll handles updates of a status poll from an existing
// stored status to latest model. this handles the case of simple vote
// count updates (without being classified as a change of the poll
// itself), as well as full poll changes that delete existing instance.
// 'changed' indicates whether the entire poll was changed.
func (d *Dereferencer) updateStatusPoll(
ctx context.Context,
existing *gtsmodel.Status,
status *gtsmodel.Status,
) (
changed bool,
err error,
) {
switch {
case existing.Poll == nil && status.Poll == nil:
// no poll before or after, nothing to do.
return false, nil
case existing.Poll == nil && status.Poll != nil:
// no previous poll, insert new status poll!
return true, d.insertStatusPoll(ctx, status)
case status.Poll == nil:
// existing status poll has been deleted, remove this from the database.
if err = d.state.DB.DeletePollByID(ctx, existing.Poll.ID); err != nil {
err = gtserror.Newf("error deleting poll from database: %w", err)
}
return true, err
case pollChanged(existing.Poll, status.Poll):
// existing status poll has been changed, remove this from the database.
if err = d.state.DB.DeletePollByID(ctx, existing.Poll.ID); err != nil {
return true, gtserror.Newf("error deleting poll from database: %w", err)
}
// insert latest poll version into database.
return true, d.insertStatusPoll(ctx, status)
case pollUpdated(existing.Poll, status.Poll):
// Since we last saw it, the poll has updated!
// Whether that be stats, or close time.
poll := existing.Poll
poll.Closing = pollJustClosed(existing.Poll, status.Poll)
poll.ClosedAt = status.Poll.ClosedAt
poll.Voters = status.Poll.Voters
poll.Votes = status.Poll.Votes
// Update poll model in the database (specifically only the possible changed columns).
if err = d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil {
return false, gtserror.Newf("error updating poll: %w", err)
}
// Update poll on status.
status.PollID = poll.ID
status.Poll = poll
return false, nil
default:
// latest and existing
// polls are up to date.
poll := existing.Poll
status.PollID = poll.ID
status.Poll = poll
return false, nil
}
}
// insertStatusPoll ...
func (d *Dereferencer) insertStatusPoll(ctx context.Context, status *gtsmodel.Status) error {
var err error
// Generate new ID for poll from latest updated time.
status.Poll.ID = id.NewULIDFromTime(status.UpdatedAt)
// Update the status<->poll links.
status.PollID = status.Poll.ID
status.Poll.StatusID = status.ID
status.Poll.Status = status
// Insert this latest poll into the database.
err = d.state.DB.PutPoll(ctx, status.Poll)
if err != nil {
return gtserror.Newf("error putting poll in database: %w", err)
}
return nil
}
// getPopulatedMention tries to populate the given
// mention with the correct TargetAccount and (if not
// yet set) TargetAccountURI, returning the populated

View file

@ -62,6 +62,7 @@ func (d *Dereferencer) isPermittedStatus(
requestUser string,
existing *gtsmodel.Status,
status *gtsmodel.Status,
isNew bool,
) (
permitted bool, // is permitted?
err error,
@ -98,7 +99,7 @@ func (d *Dereferencer) isPermittedStatus(
permitted = true
}
if !permitted && existing != nil {
if !permitted && !isNew {
log.Infof(ctx, "deleting unpermitted: %s", existing.URI)
// Delete existing status from database as it's no longer permitted.
@ -110,11 +111,13 @@ func (d *Dereferencer) isPermittedStatus(
return
}
// isPermittedReply ...
func (d *Dereferencer) isPermittedReply(
ctx context.Context,
requestUser string,
reply *gtsmodel.Status,
) (bool, error) {
var (
replyURI = reply.URI // Definitely set.
inReplyToURI = reply.InReplyToURI // Definitely set.
@ -149,8 +152,7 @@ func (d *Dereferencer) isPermittedReply(
// If this status's parent was rejected,
// implicitly this reply should be too;
// there's nothing more to check here.
return false, d.unpermittedByParent(
ctx,
return false, d.unpermittedByParent(ctx,
reply,
thisReq,
parentReq,
@ -164,6 +166,7 @@ func (d *Dereferencer) isPermittedReply(
// be approved, then we should just reject it
// again, as nothing's changed since last time.
if thisRejected && acceptIRI == "" {
// Nothing changed,
// still rejected.
return false, nil
@ -174,6 +177,7 @@ func (d *Dereferencer) isPermittedReply(
// to be approved. Continue permission checks.
if inReplyTo == nil {
// If we didn't have the replied-to status
// in our database (yet), we can't check
// right now if this reply is permitted.

View file

@ -52,7 +52,7 @@ func emojiChanged(existing, latest *gtsmodel.Emoji) bool {
// pollChanged returns whether a poll has changed in way that
// indicates that this should be an entirely new poll. i.e. if
// the available options have changed, or the expiry has increased.
// the available options have changed, or the expiry has changed.
func pollChanged(existing, latest *gtsmodel.Poll) bool {
return !slices.Equal(existing.Options, latest.Options) ||
!existing.ExpiresAt.Equal(latest.ExpiresAt)
@ -70,3 +70,12 @@ func pollUpdated(existing, latest *gtsmodel.Poll) bool {
func pollJustClosed(existing, latest *gtsmodel.Poll) bool {
return existing.ClosedAt.IsZero() && latest.Closed()
}
// statusChanged returns whether a status has changed in a way that
// indicates that existing should be snapshotted for version history.
func statusChanged(existing, latest *gtsmodel.Status) bool {
return !existing.UpdatedAt.Equal(latest.UpdatedAt) ||
existing.Content != latest.Content ||
existing.ContentWarning != latest.ContentWarning ||
!slices.Equal(existing.AttachmentIDs, latest.AttachmentIDs)
}

View file

@ -55,6 +55,8 @@ type Status struct {
BoostOf *Status `bun:"-"` // status that corresponds to boostOfID
BoostOfAccount *Account `bun:"rel:belongs-to"` // account that corresponds to boostOfAccountID
ThreadID string `bun:"type:CHAR(26),nullzero"` // id of the thread to which this status belongs; only set for remote statuses if a local account is involved at some point in the thread, otherwise null
EditIDs []string `bun:",array"` //
Edits []*StatusEdit `bun:"-"` //
PollID string `bun:"type:CHAR(26),nullzero"` //
Poll *Poll `bun:"-"` //
ContentWarning string `bun:",nullzero"` // cw string for this status
@ -92,7 +94,8 @@ func (s *Status) GetBoostOfAccountID() string {
return s.BoostOfAccountID
}
// AttachmentsPopulated returns whether media attachments are populated according to current AttachmentIDs.
// AttachmentsPopulated returns whether media attachments
// are populated according to current AttachmentIDs.
func (s *Status) AttachmentsPopulated() bool {
if len(s.AttachmentIDs) != len(s.Attachments) {
// this is the quickest indicator.
@ -106,7 +109,8 @@ func (s *Status) AttachmentsPopulated() bool {
return true
}
// TagsPopulated returns whether tags are populated according to current TagIDs.
// TagsPopulated returns whether tags are
// populated according to current TagIDs.
func (s *Status) TagsPopulated() bool {
if len(s.TagIDs) != len(s.Tags) {
// this is the quickest indicator.
@ -120,7 +124,8 @@ func (s *Status) TagsPopulated() bool {
return true
}
// MentionsPopulated returns whether mentions are populated according to current MentionIDs.
// MentionsPopulated returns whether mentions are
// populated according to current MentionIDs.
func (s *Status) MentionsPopulated() bool {
if len(s.MentionIDs) != len(s.Mentions) {
// this is the quickest indicator.
@ -134,7 +139,8 @@ func (s *Status) MentionsPopulated() bool {
return true
}
// EmojisPopulated returns whether emojis are populated according to current EmojiIDs.
// EmojisPopulated returns whether emojis are
// populated according to current EmojiIDs.
func (s *Status) EmojisPopulated() bool {
if len(s.EmojiIDs) != len(s.Emojis) {
// this is the quickest indicator.
@ -148,6 +154,21 @@ func (s *Status) EmojisPopulated() bool {
return true
}
// EditsPopulated returns whether edits are
// populated according to current EditIDs.
func (s *Status) EditsPopulated() bool {
if len(s.EditIDs) != len(s.Edits) {
// this is quickest indicator.
return false
}
for i, id := range s.EditIDs {
if s.Edits[i].ID != id {
return false
}
}
return true
}
// EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date
// according to emoji attachments of the passed status, by comparing their emoji URIs. We don't
// use IDs as this is used to determine whether there are new emojis to fetch.

View file

@ -0,0 +1,61 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package gtsmodel
import "time"
// StatusEdit represents a **historical** view of a Status
// after a received edit. The Status itself will always
// contain the latest up-to-date information.
//
// Note that stored status edits may not exactly match that
// of the origin server, they are a best-effort by receiver
// to store version history. There is no AP history endpoint.
type StatusEdit struct {
ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database.
Content string `bun:""` // Content of status at time of edit; likely html-formatted but not guaranteed.
ContentWarning string `bun:",nullzero"` // Content warning of status at time of edit.
Text string `bun:""` // Original status text, without formatting, at time of edit.
Language string `bun:",nullzero"` // Status language at time of edit.
Sensitive *bool `bun:",nullzero,notnull,default:false"` // Status sensitive flag at time of edit.
AttachmentIDs []string `bun:",array"` // Database IDs of media attachments associated with status at time of edit.
Attachments []*MediaAttachment `bun:"-"` // Media attachments relating to .AttachmentIDs field (not always populated).
PollOptions []string `bun:",array"` // Poll options of status at time of edit, only set if status contains a poll.
PollVotes []int `bun:",array"` // Poll vote count at time of status edit, only set if poll votes were reset.
StatusID string `bun:"type:CHAR(26),nullzero,notnull"` // The originating status ID this is a historical edit of.
CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // The creation time of this version of the status content (according to receiving server).
// We don't bother having a *gtsmodel.Status model here
// as the StatusEdit is always just attached to a Status,
// so it doesn't need a self-reference back to it.
}
// AttachmentsPopulated returns whether media attachments
// are populated according to current AttachmentIDs.
func (e *StatusEdit) AttachmentsPopulated() bool {
if len(e.AttachmentIDs) != len(e.Attachments) {
// this is the quickest indicator.
return false
}
for i, id := range e.AttachmentIDs {
if e.Attachments[i].ID != id {
return false
}
}
return true
}

View file

@ -45,13 +45,14 @@ func NewULID() string {
return ulid.String()
}
// NewULIDFromTime returns a new ULID string using the given time, or an error if something goes wrong.
func NewULIDFromTime(t time.Time) (string, error) {
// NewULIDFromTime returns a new ULID string using
// given time, or from current time on any error.
func NewULIDFromTime(t time.Time) string {
newUlid, err := ulid.New(ulid.Timestamp(t), rand.Reader)
if err != nil {
return "", err
return NewULID()
}
return newUlid.String(), nil
return newUlid.String()
}
// NewRandomULID returns a new ULID string using a random time in an ~80 year range around the current datetime, or an error if something goes wrong.

View file

@ -537,11 +537,7 @@ func (u *utils) requestFave(
}
// Create + store new interaction request.
req, err = typeutils.StatusFaveToInteractionRequest(ctx, fave)
if err != nil {
return gtserror.Newf("error creating interaction request: %w", err)
}
req = typeutils.StatusFaveToInteractionRequest(fave)
if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil {
return gtserror.Newf("db error storing interaction request: %w", err)
}
@ -584,11 +580,7 @@ func (u *utils) requestReply(
}
// Create + store interaction request.
req, err = typeutils.StatusToInteractionRequest(ctx, reply)
if err != nil {
return gtserror.Newf("error creating interaction request: %w", err)
}
req = typeutils.StatusToInteractionRequest(reply)
if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil {
return gtserror.Newf("db error storing interaction request: %w", err)
}
@ -631,11 +623,7 @@ func (u *utils) requestAnnounce(
}
// Create + store interaction request.
req, err = typeutils.StatusToInteractionRequest(ctx, boost)
if err != nil {
return gtserror.Newf("error creating interaction request: %w", err)
}
req = typeutils.StatusToInteractionRequest(boost)
if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil {
return gtserror.Newf("db error storing interaction request: %w", err)
}

View file

@ -111,6 +111,13 @@ func (c *Converter) ASRepresentationToAccount(
acct.UpdatedAt = pub
}
// Extract updated time if possible, i.e. last edited.
if upd := ap.GetUpdated(accountable); !upd.IsZero() {
acct.UpdatedAt = upd
} else {
acct.UpdatedAt = acct.CreatedAt
}
// Extract a preferred name (display name), fallback to username.
if displayName := ap.ExtractName(accountable); displayName != "" {
acct.DisplayName = displayName