From 3e131f5da6176c0364741ad1ffa1af203f9724f0 Mon Sep 17 00:00:00 2001 From: kim Date: Wed, 13 Nov 2024 13:51:03 +0000 Subject: [PATCH] add support for status edits in the database, and update status dereferencer to handle them --- internal/ap/interfaces.go | 1 + internal/cache/cache.go | 1 + internal/cache/db.go | 35 ++ internal/cache/invalidate.go | 5 + internal/cache/size.go | 17 + internal/config/config.go | 1 + internal/config/helpers.gen.go | 25 ++ internal/db/bundb/bundb.go | 5 + internal/db/bundb/status.go | 37 +- internal/db/bundb/statusedit.go | 147 ++++++++ internal/db/db.go | 1 + internal/db/statusedit.go | 39 +++ internal/federation/dereferencing/announce.go | 7 +- internal/federation/dereferencing/status.go | 329 +++++++++++------- .../dereferencing/status_permitted.go | 10 +- internal/federation/dereferencing/util.go | 11 +- internal/gtsmodel/status.go | 29 +- internal/gtsmodel/statusedit.go | 61 ++++ internal/id/ulid.go | 9 +- internal/processing/workers/util.go | 18 +- internal/typeutils/astointernal.go | 7 + 21 files changed, 623 insertions(+), 172 deletions(-) create mode 100644 internal/db/bundb/statusedit.go create mode 100644 internal/db/statusedit.go create mode 100644 internal/gtsmodel/statusedit.go diff --git a/internal/ap/interfaces.go b/internal/ap/interfaces.go index cbe95aa17..1f08fde37 100644 --- a/internal/ap/interfaces.go +++ b/internal/ap/interfaces.go @@ -187,6 +187,7 @@ type Accountable interface { WithEndpoints WithTag WithPublished + WithUpdated } // Statusable represents the minimum activitypub interface for representing a 'status'. diff --git a/internal/cache/cache.go b/internal/cache/cache.go index a4f9f2044..1a66fcd6b 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -105,6 +105,7 @@ func (c *Caches) Init() { c.initStatus() c.initStatusBookmark() c.initStatusBookmarkIDs() + c.initStatusEdit() c.initStatusFave() c.initStatusFaveIDs() c.initTag() diff --git a/internal/cache/db.go b/internal/cache/db.go index aac11236a..dc47bc31c 100644 --- a/internal/cache/db.go +++ b/internal/cache/db.go @@ -226,6 +226,9 @@ type DBCaches struct { // StatusBookmarkIDs provides access to the status bookmark IDs list database cache. StatusBookmarkIDs SliceCache[string] + // StatusEdit provides access to the gtsmodel StatusEdit database cache. + StatusEdit StructCache[*gtsmodel.StatusEdit] + // StatusFave provides access to the gtsmodel StatusFave database cache. StatusFave StructCache[*gtsmodel.StatusFave] @@ -1385,6 +1388,38 @@ func (c *Caches) initStatusBookmarkIDs() { c.DB.StatusBookmarkIDs.Init(0, cap) } +func (c *Caches) initStatusEdit() { + // Calculate maximum cache size. + cap := calculateResultCacheMax( + sizeofStatusEdit(), // model in-mem size. + config.GetCacheStatusEditMemRatio(), + ) + + log.Infof(nil, "cache size = %d", cap) + + copyF := func(s1 *gtsmodel.StatusEdit) *gtsmodel.StatusEdit { + s2 := new(gtsmodel.StatusEdit) + *s2 = *s1 + + // Don't include ptr fields that + // will be populated separately. + s2.Attachments = nil + + return s2 + } + + c.DB.StatusEdit.Init(structr.CacheConfig[*gtsmodel.StatusEdit]{ + Indices: []structr.IndexConfig{ + {Fields: "ID"}, + {Fields: "StatusID", Multiple: true}, + }, + MaxSize: cap, + IgnoreErr: ignoreErrors, + Copy: copyF, + Invalidate: c.OnInvalidateStatusEdit, + }) +} + func (c *Caches) initStatusFave() { // Calculate maximum cache size. cap := calculateResultCacheMax( diff --git a/internal/cache/invalidate.go b/internal/cache/invalidate.go index 9b42e88f6..42d7b7399 100644 --- a/internal/cache/invalidate.go +++ b/internal/cache/invalidate.go @@ -273,6 +273,11 @@ func (c *Caches) OnInvalidateStatusBookmark(bookmark *gtsmodel.StatusBookmark) { c.DB.StatusBookmarkIDs.Invalidate(bookmark.StatusID) } +func (c *Caches) OnInvalidateStatusEdit(edit *gtsmodel.StatusEdit) { + // Invalidate cache of related status model. + c.DB.Status.Invalidate("ID", edit.StatusID) +} + func (c *Caches) OnInvalidateStatusFave(fave *gtsmodel.StatusFave) { // Invalidate status fave ID list for this status. c.DB.StatusFaveIDs.Invalidate(fave.StatusID) diff --git a/internal/cache/size.go b/internal/cache/size.go index 26f4096ed..81a6e2cec 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -674,6 +674,23 @@ func sizeofStatusBookmark() uintptr { })) } +func sizeofStatusEdit() uintptr { + return uintptr(size.Of(>smodel.StatusEdit{ + ID: exampleID, + Content: exampleText, + ContentWarning: exampleUsername, // similar length + Text: exampleText, + Language: "en", + Sensitive: func() *bool { ok := false; return &ok }(), + AttachmentIDs: []string{exampleID, exampleID, exampleID}, + Attachments: nil, + PollOptions: []string{exampleTextSmall, exampleTextSmall, exampleTextSmall, exampleTextSmall}, + PollVotes: []int{69, 420, 1337, 1969}, + StatusID: exampleID, + CreatedAt: exampleTime, + })) +} + func sizeofStatusFave() uintptr { return uintptr(size.Of(>smodel.StatusFave{ ID: exampleID, diff --git a/internal/config/config.go b/internal/config/config.go index 2e3ad8ec1..413743409 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -238,6 +238,7 @@ type CacheConfiguration struct { StatusMemRatio float64 `name:"status-mem-ratio"` StatusBookmarkMemRatio float64 `name:"status-bookmark-mem-ratio"` StatusBookmarkIDsMemRatio float64 `name:"status-bookmark-ids-mem-ratio"` + StatusEditMemRatio float64 `name:"status-edit-mem-ratio"` StatusFaveMemRatio float64 `name:"status-fave-mem-ratio"` StatusFaveIDsMemRatio float64 `name:"status-fave-ids-mem-ratio"` TagMemRatio float64 `name:"tag-mem-ratio"` diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index a35622f8e..543292ebe 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -3912,6 +3912,31 @@ func GetCacheStatusBookmarkIDsMemRatio() float64 { return global.GetCacheStatusB // SetCacheStatusBookmarkIDsMemRatio safely sets the value for global configuration 'Cache.StatusBookmarkIDsMemRatio' field func SetCacheStatusBookmarkIDsMemRatio(v float64) { global.SetCacheStatusBookmarkIDsMemRatio(v) } +// GetCacheStatusEditMemRatio safely fetches the Configuration value for state's 'Cache.StatusEditMemRatio' field +func (st *ConfigState) GetCacheStatusEditMemRatio() (v float64) { + st.mutex.RLock() + v = st.config.Cache.StatusEditMemRatio + st.mutex.RUnlock() + return +} + +// SetCacheStatusEditMemRatio safely sets the Configuration value for state's 'Cache.StatusEditMemRatio' field +func (st *ConfigState) SetCacheStatusEditMemRatio(v float64) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.Cache.StatusEditMemRatio = v + st.reloadToViper() +} + +// CacheStatusEditMemRatioFlag returns the flag name for the 'Cache.StatusEditMemRatio' field +func CacheStatusEditMemRatioFlag() string { return "cache-status-edit-mem-ratio" } + +// GetCacheStatusEditMemRatio safely fetches the value for global configuration 'Cache.StatusEditMemRatio' field +func GetCacheStatusEditMemRatio() float64 { return global.GetCacheStatusEditMemRatio() } + +// SetCacheStatusEditMemRatio safely sets the value for global configuration 'Cache.StatusEditMemRatio' field +func SetCacheStatusEditMemRatio(v float64) { global.SetCacheStatusEditMemRatio(v) } + // GetCacheStatusFaveMemRatio safely fetches the Configuration value for state's 'Cache.StatusFaveMemRatio' field func (st *ConfigState) GetCacheStatusFaveMemRatio() (v float64) { st.mutex.RLock() diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index 70132fe58..cf612fd2e 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -81,6 +81,7 @@ type DBService struct { db.SinBinStatus db.Status db.StatusBookmark + db.StatusEdit db.StatusFave db.Tag db.Thread @@ -272,6 +273,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { db: db, state: state, }, + StatusEdit: &statusEditDB{ + db: db, + state: state, + }, StatusFave: &statusFaveDB{ db: db, state: state, diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 45e9864a3..c480e5957 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -181,13 +181,17 @@ func (s *statusDB) getStatus(ctx context.Context, lookup string, dbQuery func(*g func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) error { var ( err error - errs = gtserror.NewMultiError(9) + errs gtserror.MultiError ) + // For sub-models we only want + // barebones versions of them. + ctx = gtscontext.SetBarebones(ctx) + if status.Account == nil { // Status author is not set, fetch from database. status.Account, err = s.state.DB.GetAccountByID( - gtscontext.SetBarebones(ctx), + ctx, status.AccountID, ) if err != nil { @@ -199,7 +203,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if status.InReplyTo == nil { // Status parent is not set, fetch from database. status.InReplyTo, err = s.GetStatusByID( - gtscontext.SetBarebones(ctx), + ctx, status.InReplyToID, ) if err != nil && !errors.Is(err, db.ErrNoEntries) { @@ -210,7 +214,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if status.InReplyToAccount == nil { // Status parent author is not set, fetch from database. status.InReplyToAccount, err = s.state.DB.GetAccountByID( - gtscontext.SetBarebones(ctx), + ctx, status.InReplyToAccountID, ) if err != nil { @@ -223,7 +227,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if status.BoostOf == nil { // Status boost is not set, fetch from database. status.BoostOf, err = s.GetStatusByID( - gtscontext.SetBarebones(ctx), + ctx, status.BoostOfID, ) if err != nil { @@ -234,7 +238,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if status.BoostOfAccount == nil { // Status boost author is not set, fetch from database. status.BoostOfAccount, err = s.state.DB.GetAccountByID( - gtscontext.SetBarebones(ctx), + ctx, status.BoostOfAccountID, ) if err != nil { @@ -246,7 +250,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if status.PollID != "" && status.Poll == nil { // Status poll is not set, fetch from database. status.Poll, err = s.state.DB.GetPollByID( - gtscontext.SetBarebones(ctx), + ctx, status.PollID, ) if err != nil { @@ -257,7 +261,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.AttachmentsPopulated() { // Status attachments are out-of-date with IDs, repopulate. status.Attachments, err = s.state.DB.GetAttachmentsByIDs( - ctx, // these are already barebones + ctx, status.AttachmentIDs, ) if err != nil { @@ -279,7 +283,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.MentionsPopulated() { // Status mentions are out-of-date with IDs, repopulate. status.Mentions, err = s.state.DB.GetMentions( - ctx, // leave fully populated for now + ctx, status.MentionIDs, ) if err != nil { @@ -290,7 +294,7 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) if !status.EmojisPopulated() { // Status emojis are out-of-date with IDs, repopulate. status.Emojis, err = s.state.DB.GetEmojisByIDs( - ctx, // these are already barebones + ctx, status.EmojiIDs, ) if err != nil { @@ -298,10 +302,21 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status) } } + if !status.EditsPopulated() { + // Status edits are out-of-date with IDs, repopulate. + status.Edits, err = s.state.DB.GetStatusEditsByIDs( + ctx, + status.EditIDs, + ) + if err != nil { + errs.Appendf("error populating status edits: %w", err) + } + } + if status.CreatedWithApplicationID != "" && status.CreatedWithApplication == nil { // Populate the status' expected CreatedWithApplication (not always set). status.CreatedWithApplication, err = s.state.DB.GetApplicationByID( - ctx, // these are already barebones + ctx, status.CreatedWithApplicationID, ) if err != nil { diff --git a/internal/db/bundb/statusedit.go b/internal/db/bundb/statusedit.go new file mode 100644 index 000000000..d492683b5 --- /dev/null +++ b/internal/db/bundb/statusedit.go @@ -0,0 +1,147 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bundb + +import ( + "context" + "slices" + + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/log" + "github.com/superseriousbusiness/gotosocial/internal/state" + "github.com/superseriousbusiness/gotosocial/internal/util" + "github.com/uptrace/bun" +) + +type statusEditDB struct { + db *bun.DB + state *state.State +} + +func (s *statusEditDB) GetStatusEditByID(ctx context.Context, id string) (*gtsmodel.StatusEdit, error) { + // Fetch edit from database cache with loader callback. + edit, err := s.state.Caches.DB.StatusEdit.LoadOne("ID", + func() (*gtsmodel.StatusEdit, error) { + var edit gtsmodel.StatusEdit + + // Not cached, load edit + // from database by its ID. + if err := s.db.NewSelect(). + Model(&edit). + Where("? = ?", bun.Ident("id"), id). + Scan(ctx); err != nil { + return nil, err + } + + return &edit, nil + }, + ) + if err != nil { + return nil, err + } + + if gtscontext.Barebones(ctx) { + // no need to fully populate. + return edit, nil + } + + // Further populate the edit fields where applicable. + if err := s.PopulateStatusEdit(ctx, edit); err != nil { + return nil, err + } + + return edit, nil +} + +func (s *statusEditDB) GetStatusEditsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.StatusEdit, error) { + // Load status edits for IDs via cache loader callbacks. + edits, err := s.state.Caches.DB.StatusEdit.LoadIDs("ID", + ids, + func(uncached []string) ([]*gtsmodel.StatusEdit, error) { + // Preallocate expected length of uncached edits. + edits := make([]*gtsmodel.StatusEdit, 0, len(uncached)) + + // Perform database query scanning + // the remaining (uncached) edit IDs. + if err := s.db.NewSelect(). + Model(&edits). + Where("? IN (?)", bun.Ident("id"), bun.In(uncached)). + Scan(ctx); err != nil { + return nil, err + } + + return edits, nil + }, + ) + if err != nil { + return nil, err + } + + // Reorder the edits by their + // IDs to ensure in correct order. + getID := func(e *gtsmodel.StatusEdit) string { return e.ID } + util.OrderBy(edits, ids, getID) + + if gtscontext.Barebones(ctx) { + // no need to fully populate. + return edits, nil + } + + // Populate all loaded edits, removing those we fail to + // populate (removes needing so many nil checks everywhere). + edits = slices.DeleteFunc(edits, func(edit *gtsmodel.StatusEdit) bool { + if err := s.PopulateStatusEdit(ctx, edit); err != nil { + log.Errorf(ctx, "error populating edit %s: %v", edit.ID, err) + return true + } + return false + }) + + return edits, nil +} + +func (s *statusEditDB) PopulateStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error { + var err error + var errs gtserror.MultiError + + // For sub-models we only want + // barebones versions of them. + ctx = gtscontext.SetBarebones(ctx) + + if !edit.AttachmentsPopulated() { + // Fetch all attachments for status edit's IDs. + edit.Attachments, err = s.state.DB.GetAttachmentsByIDs( + ctx, + edit.AttachmentIDs, + ) + if err != nil { + errs.Appendf("error populating edit attachments: %w", err) + } + } + + return errs.Combine() +} + +func (s *statusEditDB) PutStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error { + return s.state.Caches.DB.StatusEdit.Store(edit, func() error { + _, err := s.db.NewInsert().Model(edit).Exec(ctx) + return err + }) +} diff --git a/internal/db/db.go b/internal/db/db.go index c42985912..11dd2e507 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -51,6 +51,7 @@ type DB interface { SinBinStatus Status StatusBookmark + StatusEdit StatusFave Tag Thread diff --git a/internal/db/statusedit.go b/internal/db/statusedit.go new file mode 100644 index 000000000..47a76e57a --- /dev/null +++ b/internal/db/statusedit.go @@ -0,0 +1,39 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package db + +import ( + "context" + + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +type StatusEdit interface { + + // GetStatusEditByID ... + GetStatusEditByID(ctx context.Context, id string) (*gtsmodel.StatusEdit, error) + + // GetStatusEditsByIDs ... + GetStatusEditsByIDs(ctx context.Context, ids []string) ([]*gtsmodel.StatusEdit, error) + + // PopulateStatusEdit ... + PopulateStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error + + // PutStatusEdit ... + PutStatusEdit(ctx context.Context, edit *gtsmodel.StatusEdit) error +} diff --git a/internal/federation/dereferencing/announce.go b/internal/federation/dereferencing/announce.go index a3eaf199d..eb949f159 100644 --- a/internal/federation/dereferencing/announce.go +++ b/internal/federation/dereferencing/announce.go @@ -87,7 +87,7 @@ func (d *Dereferencer) EnrichAnnounce( boost.Federated = target.Federated // Ensure this Announce is permitted by the Announcee. - permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost) + permit, err := d.isPermittedStatus(ctx, requestUser, nil, boost, true) if err != nil { return nil, gtserror.Newf("error checking permitted status %s: %w", boost.URI, err) } @@ -99,10 +99,7 @@ func (d *Dereferencer) EnrichAnnounce( } // Generate an ID for the boost wrapper status. - boost.ID, err = id.NewULIDFromTime(boost.CreatedAt) - if err != nil { - return nil, gtserror.Newf("error generating id: %w", err) - } + boost.ID = id.NewULIDFromTime(boost.CreatedAt) // Store the boost wrapper status in database. switch err = d.state.DB.PutStatus(ctx, boost); { diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go index c90730826..718916b35 100644 --- a/internal/federation/dereferencing/status.go +++ b/internal/federation/dereferencing/status.go @@ -302,6 +302,7 @@ func (d *Dereferencer) enrichStatusSafely( uri, status, statusable, + isNew, ) // Check for a returned HTTP code via error. @@ -374,6 +375,7 @@ func (d *Dereferencer) enrichStatus( uri *url.URL, status *gtsmodel.Status, statusable ap.Statusable, + isNew bool, ) ( *gtsmodel.Status, ap.Statusable, @@ -476,8 +478,7 @@ func (d *Dereferencer) enrichStatus( // Ensure the final parsed status URI or URL matches // the input URI we fetched (or received) it as. - matches, err := util.URIMatches( - uri, + matches, err := util.URIMatches(uri, append( ap.GetURL(statusable), // status URL(s) ap.GetJSONLDId(statusable), // status URI @@ -497,19 +498,10 @@ func (d *Dereferencer) enrichStatus( ) } - var isNew bool - - // Based on the original provided - // status model, determine whether - // this is a new insert / update. - if isNew = (status.ID == ""); isNew { + if isNew { // Generate new status ID from the provided creation date. - latestStatus.ID, err = id.NewULIDFromTime(latestStatus.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - latestStatus.ID = id.NewULID() // just use "now" - } + latestStatus.ID = id.NewULIDFromTime(latestStatus.CreatedAt) } else { // Reuse existing status ID. @@ -538,8 +530,9 @@ func (d *Dereferencer) enrichStatus( } // Check if this is a permitted status we should accept. - // Function also sets "PendingApproval" bool as necessary. - permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus) + // Function also sets "PendingApproval" bool as necessary, + // and handles removal of existing statuses no longer permitted. + permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus, isNew) if err != nil { return nil, nil, gtserror.Newf("error checking permissibility for status %s: %w", uri, err) } @@ -555,11 +548,6 @@ func (d *Dereferencer) enrichStatus( return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err) } - // Ensure the status' poll remains consistent, else reset the poll. - if err := d.fetchStatusPoll(ctx, status, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error populating poll for status %s: %w", uri, err) - } - // Now that we know who this status replies to (handled by ASStatusToStatus) // and who it mentions, we can add a ThreadID to it if necessary. if err := d.threadStatus(ctx, latestStatus); err != nil { @@ -582,27 +570,28 @@ func (d *Dereferencer) enrichStatus( } if isNew { - // This is new, put the status in the database. - err := d.state.DB.PutStatus(ctx, latestStatus) - if err != nil { - return nil, nil, gtserror.Newf("error putting in database: %w", err) + // This is a new status, insert it into the database. + if err := d.insertStatus(ctx, latestStatus); err != nil { + return nil, nil, gtserror.Newf("error inserting new status %s: %w", uri, err) } } else { // This is an existing status, update the model in the database. - if err := d.state.DB.UpdateStatus(ctx, latestStatus); err != nil { - return nil, nil, gtserror.Newf("error updating database: %w", err) + if err := d.updateStatus(ctx, status, latestStatus); err != nil { + return nil, nil, gtserror.Newf("error updating existing status %s: %w", uri, err) } } return latestStatus, statusable, nil } +// fetchStatusMentions ... func (d *Dereferencer) fetchStatusMentions( ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { + // Allocate new slice to take the yet-to-be created mention IDs. status.MentionIDs = make([]string, len(status.Mentions)) @@ -637,11 +626,7 @@ func (d *Dereferencer) fetchStatusMentions( // Generate new ID according to status creation. // TODO: update this to use "edited_at" when we add // support for edited status revision history. - mention.ID, err = id.NewULIDFromTime(status.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - mention.ID = id.NewULID() // just use "now" - } + mention.ID = id.NewULIDFromTime(status.CreatedAt) // Set known further mention details. mention.CreatedAt = status.CreatedAt @@ -681,7 +666,9 @@ func (d *Dereferencer) fetchStatusMentions( return nil } +// threadStatus ... func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status) error { + if status.InReplyTo != nil { if parentThreadID := status.InReplyTo.ThreadID; parentThreadID != "" { // Simplest case: parent status @@ -719,24 +706,27 @@ func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status // it to the status. threadID := id.NewULID() - if err := d.state.DB.PutThread( - ctx, - >smodel.Thread{ - ID: threadID, - }, + // ... + if err := d.state.DB.PutThread(ctx, + >smodel.Thread{ID: threadID}, ); err != nil { return gtserror.Newf("error inserting new thread in db: %w", err) } + // Set thread on latest status. status.ThreadID = threadID return nil } +// fetchStatusTags populates the tags on 'status', fetching existing +// from the database and creating new where needed. 'existing' is used +// to fetch tags that have not changed since previous stored status. func (d *Dereferencer) fetchStatusTags( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { + // Allocate new slice to take the yet-to-be determined tag IDs. status.TagIDs = make([]string, len(status.Tags)) @@ -791,103 +781,14 @@ func (d *Dereferencer) fetchStatusTags( return nil } -func (d *Dereferencer) fetchStatusPoll( - ctx context.Context, - existing *gtsmodel.Status, - status *gtsmodel.Status, -) error { - var ( - // insertStatusPoll generates ID and inserts the poll attached to status into the database. - insertStatusPoll = func(ctx context.Context, status *gtsmodel.Status) error { - var err error - - // Generate new ID for poll from the status CreatedAt. - // TODO: update this to use "edited_at" when we add - // support for edited status revision history. - status.Poll.ID, err = id.NewULIDFromTime(status.CreatedAt) - if err != nil { - log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) - status.Poll.ID = id.NewULID() // just use "now" - } - - // Update the status<->poll links. - status.PollID = status.Poll.ID - status.Poll.StatusID = status.ID - status.Poll.Status = status - - // Insert this latest poll into the database. - err = d.state.DB.PutPoll(ctx, status.Poll) - if err != nil { - return gtserror.Newf("error putting in database: %w", err) - } - - return nil - } - - // deleteStatusPoll deletes the poll with ID, and all attached votes, from the database. - deleteStatusPoll = func(ctx context.Context, pollID string) error { - if err := d.state.DB.DeletePollByID(ctx, pollID); err != nil { - return gtserror.Newf("error deleting existing poll from database: %w", err) - } - return nil - } - ) - - switch { - case existing.Poll == nil && status.Poll == nil: - // no poll before or after, nothing to do. - return nil - - case existing.Poll == nil && status.Poll != nil: - // no previous poll, insert new poll! - return insertStatusPoll(ctx, status) - - case status.Poll == nil: - // existing poll has been deleted, remove this. - return deleteStatusPoll(ctx, existing.PollID) - - case pollChanged(existing.Poll, status.Poll): - // poll has changed since original, delete and reinsert new. - if err := deleteStatusPoll(ctx, existing.PollID); err != nil { - return err - } - return insertStatusPoll(ctx, status) - - case pollUpdated(existing.Poll, status.Poll): - // Since we last saw it, the poll has updated! - // Whether that be stats, or close time. - poll := existing.Poll - poll.Closing = pollJustClosed(existing.Poll, status.Poll) - poll.ClosedAt = status.Poll.ClosedAt - poll.Voters = status.Poll.Voters - poll.Votes = status.Poll.Votes - - // Update poll model in the database (specifically only the possible changed columns). - if err := d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil { - return gtserror.Newf("error updating poll: %w", err) - } - - // Update poll on status. - status.PollID = poll.ID - status.Poll = poll - return nil - - default: - // latest and existing - // polls are up to date. - poll := existing.Poll - status.PollID = poll.ID - status.Poll = poll - return nil - } -} - +// fetchStatusAttachments ... func (d *Dereferencer) fetchStatusAttachments( ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { + // Allocate new slice to take the yet-to-be fetched attachment IDs. status.AttachmentIDs = make([]string, len(status.Attachments)) @@ -916,8 +817,7 @@ func (d *Dereferencer) fetchStatusAttachments( } // Load this new media attachment. - attachment, err := d.GetMedia( - ctx, + attachment, err := d.GetMedia(ctx, requestUser, status.AccountID, placeholder.RemoteURL, @@ -958,11 +858,13 @@ func (d *Dereferencer) fetchStatusAttachments( return nil } +// fetchStatusEmojis ... func (d *Dereferencer) fetchStatusEmojis( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { + // Fetch the updated emojis for our status. emojis, changed, err := d.fetchEmojis(ctx, existing.Emojis, @@ -991,6 +893,175 @@ func (d *Dereferencer) fetchStatusEmojis( return nil } +// insertStatus handles the insert of a new status into the database, inserting new poll if necessary. +func (d *Dereferencer) insertStatus(ctx context.Context, status *gtsmodel.Status) error { + + if status.Poll != nil { + // Insert this poll attached to status in the database. + if err := d.insertStatusPoll(ctx, status); err != nil { + return err + } + } + + // Insert new status into the database. + err := d.state.DB.PutStatus(ctx, status) + if err != nil { + return gtserror.Newf("error putting status in database: %w", err) + } + + return nil +} + +// updateStatus handles the updating of an existing status in the +// database, handling any required poll changes and / or staus edits. +func (d *Dereferencer) updateStatus( + ctx context.Context, + existing *gtsmodel.Status, + status *gtsmodel.Status, +) error { + + // Handle any changes in status poll from existing to new. + pollChanged, err := d.updateStatusPoll(ctx, existing, status) + if err != nil { + return err + } + + // If there was no change to poll, look for + // any other changes in status content itself. + if pollChanged || statusChanged(existing, status) { + + // Status has been editted since last + // we saw it, take snapshot of existing. + var edit gtsmodel.StatusEdit + edit.ID = id.NewULIDFromTime(status.UpdatedAt) + edit.Content = existing.Content + edit.ContentWarning = existing.ContentWarning + edit.Text = existing.Text + edit.Language = existing.Language + edit.Sensitive = existing.Sensitive + edit.AttachmentIDs = existing.AttachmentIDs + edit.Attachments = existing.Attachments + + // Edit creation is last update time. + edit.CreatedAt = existing.UpdatedAt + + if existing.Poll != nil { + // Poll only set if existing contained them. + edit.PollOptions = existing.Poll.Options + + if pollChanged { + // Votes are only set if the poll + // itself changed during this edit. + edit.PollVotes = existing.Poll.Votes + } + } + + // Insert this new edit of existing status into database. + if err := d.state.DB.PutStatusEdit(ctx, &edit); err != nil { + return gtserror.Newf("error putting edit in database: %w", err) + } + + // Add edit to list of edits on the status. + status.EditIDs = append(status.EditIDs, edit.ID) + status.Edits = append(status.Edits, &edit) + } + + // Update the existing status in database with new details. + if err := d.state.DB.UpdateStatus(ctx, status); err != nil { + return gtserror.Newf("error updating status in database: %w", err) + } + + return nil +} + +// updateStatusPoll handles updates of a status poll from an existing +// stored status to latest model. this handles the case of simple vote +// count updates (without being classified as a change of the poll +// itself), as well as full poll changes that delete existing instance. +// 'changed' indicates whether the entire poll was changed. +func (d *Dereferencer) updateStatusPoll( + ctx context.Context, + existing *gtsmodel.Status, + status *gtsmodel.Status, +) ( + changed bool, + err error, +) { + switch { + case existing.Poll == nil && status.Poll == nil: + // no poll before or after, nothing to do. + return false, nil + + case existing.Poll == nil && status.Poll != nil: + // no previous poll, insert new status poll! + return true, d.insertStatusPoll(ctx, status) + + case status.Poll == nil: + // existing status poll has been deleted, remove this from the database. + if err = d.state.DB.DeletePollByID(ctx, existing.Poll.ID); err != nil { + err = gtserror.Newf("error deleting poll from database: %w", err) + } + return true, err + + case pollChanged(existing.Poll, status.Poll): + // existing status poll has been changed, remove this from the database. + if err = d.state.DB.DeletePollByID(ctx, existing.Poll.ID); err != nil { + return true, gtserror.Newf("error deleting poll from database: %w", err) + } + + // insert latest poll version into database. + return true, d.insertStatusPoll(ctx, status) + + case pollUpdated(existing.Poll, status.Poll): + // Since we last saw it, the poll has updated! + // Whether that be stats, or close time. + poll := existing.Poll + poll.Closing = pollJustClosed(existing.Poll, status.Poll) + poll.ClosedAt = status.Poll.ClosedAt + poll.Voters = status.Poll.Voters + poll.Votes = status.Poll.Votes + + // Update poll model in the database (specifically only the possible changed columns). + if err = d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil { + return false, gtserror.Newf("error updating poll: %w", err) + } + + // Update poll on status. + status.PollID = poll.ID + status.Poll = poll + return false, nil + + default: + // latest and existing + // polls are up to date. + poll := existing.Poll + status.PollID = poll.ID + status.Poll = poll + return false, nil + } +} + +// insertStatusPoll ... +func (d *Dereferencer) insertStatusPoll(ctx context.Context, status *gtsmodel.Status) error { + var err error + + // Generate new ID for poll from latest updated time. + status.Poll.ID = id.NewULIDFromTime(status.UpdatedAt) + + // Update the status<->poll links. + status.PollID = status.Poll.ID + status.Poll.StatusID = status.ID + status.Poll.Status = status + + // Insert this latest poll into the database. + err = d.state.DB.PutPoll(ctx, status.Poll) + if err != nil { + return gtserror.Newf("error putting poll in database: %w", err) + } + + return nil +} + // getPopulatedMention tries to populate the given // mention with the correct TargetAccount and (if not // yet set) TargetAccountURI, returning the populated diff --git a/internal/federation/dereferencing/status_permitted.go b/internal/federation/dereferencing/status_permitted.go index 9ad425c2f..5d05c5de4 100644 --- a/internal/federation/dereferencing/status_permitted.go +++ b/internal/federation/dereferencing/status_permitted.go @@ -62,6 +62,7 @@ func (d *Dereferencer) isPermittedStatus( requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, + isNew bool, ) ( permitted bool, // is permitted? err error, @@ -98,7 +99,7 @@ func (d *Dereferencer) isPermittedStatus( permitted = true } - if !permitted && existing != nil { + if !permitted && !isNew { log.Infof(ctx, "deleting unpermitted: %s", existing.URI) // Delete existing status from database as it's no longer permitted. @@ -110,11 +111,13 @@ func (d *Dereferencer) isPermittedStatus( return } +// isPermittedReply ... func (d *Dereferencer) isPermittedReply( ctx context.Context, requestUser string, reply *gtsmodel.Status, ) (bool, error) { + var ( replyURI = reply.URI // Definitely set. inReplyToURI = reply.InReplyToURI // Definitely set. @@ -149,8 +152,7 @@ func (d *Dereferencer) isPermittedReply( // If this status's parent was rejected, // implicitly this reply should be too; // there's nothing more to check here. - return false, d.unpermittedByParent( - ctx, + return false, d.unpermittedByParent(ctx, reply, thisReq, parentReq, @@ -164,6 +166,7 @@ func (d *Dereferencer) isPermittedReply( // be approved, then we should just reject it // again, as nothing's changed since last time. if thisRejected && acceptIRI == "" { + // Nothing changed, // still rejected. return false, nil @@ -174,6 +177,7 @@ func (d *Dereferencer) isPermittedReply( // to be approved. Continue permission checks. if inReplyTo == nil { + // If we didn't have the replied-to status // in our database (yet), we can't check // right now if this reply is permitted. diff --git a/internal/federation/dereferencing/util.go b/internal/federation/dereferencing/util.go index 297e90adc..4c337c981 100644 --- a/internal/federation/dereferencing/util.go +++ b/internal/federation/dereferencing/util.go @@ -52,7 +52,7 @@ func emojiChanged(existing, latest *gtsmodel.Emoji) bool { // pollChanged returns whether a poll has changed in way that // indicates that this should be an entirely new poll. i.e. if -// the available options have changed, or the expiry has increased. +// the available options have changed, or the expiry has changed. func pollChanged(existing, latest *gtsmodel.Poll) bool { return !slices.Equal(existing.Options, latest.Options) || !existing.ExpiresAt.Equal(latest.ExpiresAt) @@ -70,3 +70,12 @@ func pollUpdated(existing, latest *gtsmodel.Poll) bool { func pollJustClosed(existing, latest *gtsmodel.Poll) bool { return existing.ClosedAt.IsZero() && latest.Closed() } + +// statusChanged returns whether a status has changed in a way that +// indicates that existing should be snapshotted for version history. +func statusChanged(existing, latest *gtsmodel.Status) bool { + return !existing.UpdatedAt.Equal(latest.UpdatedAt) || + existing.Content != latest.Content || + existing.ContentWarning != latest.ContentWarning || + !slices.Equal(existing.AttachmentIDs, latest.AttachmentIDs) +} diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index f8bd068ab..88c3826f6 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -55,6 +55,8 @@ type Status struct { BoostOf *Status `bun:"-"` // status that corresponds to boostOfID BoostOfAccount *Account `bun:"rel:belongs-to"` // account that corresponds to boostOfAccountID ThreadID string `bun:"type:CHAR(26),nullzero"` // id of the thread to which this status belongs; only set for remote statuses if a local account is involved at some point in the thread, otherwise null + EditIDs []string `bun:",array"` // + Edits []*StatusEdit `bun:"-"` // PollID string `bun:"type:CHAR(26),nullzero"` // Poll *Poll `bun:"-"` // ContentWarning string `bun:",nullzero"` // cw string for this status @@ -92,7 +94,8 @@ func (s *Status) GetBoostOfAccountID() string { return s.BoostOfAccountID } -// AttachmentsPopulated returns whether media attachments are populated according to current AttachmentIDs. +// AttachmentsPopulated returns whether media attachments +// are populated according to current AttachmentIDs. func (s *Status) AttachmentsPopulated() bool { if len(s.AttachmentIDs) != len(s.Attachments) { // this is the quickest indicator. @@ -106,7 +109,8 @@ func (s *Status) AttachmentsPopulated() bool { return true } -// TagsPopulated returns whether tags are populated according to current TagIDs. +// TagsPopulated returns whether tags are +// populated according to current TagIDs. func (s *Status) TagsPopulated() bool { if len(s.TagIDs) != len(s.Tags) { // this is the quickest indicator. @@ -120,7 +124,8 @@ func (s *Status) TagsPopulated() bool { return true } -// MentionsPopulated returns whether mentions are populated according to current MentionIDs. +// MentionsPopulated returns whether mentions are +// populated according to current MentionIDs. func (s *Status) MentionsPopulated() bool { if len(s.MentionIDs) != len(s.Mentions) { // this is the quickest indicator. @@ -134,7 +139,8 @@ func (s *Status) MentionsPopulated() bool { return true } -// EmojisPopulated returns whether emojis are populated according to current EmojiIDs. +// EmojisPopulated returns whether emojis are +// populated according to current EmojiIDs. func (s *Status) EmojisPopulated() bool { if len(s.EmojiIDs) != len(s.Emojis) { // this is the quickest indicator. @@ -148,6 +154,21 @@ func (s *Status) EmojisPopulated() bool { return true } +// EditsPopulated returns whether edits are +// populated according to current EditIDs. +func (s *Status) EditsPopulated() bool { + if len(s.EditIDs) != len(s.Edits) { + // this is quickest indicator. + return false + } + for i, id := range s.EditIDs { + if s.Edits[i].ID != id { + return false + } + } + return true +} + // EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date // according to emoji attachments of the passed status, by comparing their emoji URIs. We don't // use IDs as this is used to determine whether there are new emojis to fetch. diff --git a/internal/gtsmodel/statusedit.go b/internal/gtsmodel/statusedit.go new file mode 100644 index 000000000..a87ed736d --- /dev/null +++ b/internal/gtsmodel/statusedit.go @@ -0,0 +1,61 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package gtsmodel + +import "time" + +// StatusEdit represents a **historical** view of a Status +// after a received edit. The Status itself will always +// contain the latest up-to-date information. +// +// Note that stored status edits may not exactly match that +// of the origin server, they are a best-effort by receiver +// to store version history. There is no AP history endpoint. +type StatusEdit struct { + ID string `bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // ID of this item in the database. + Content string `bun:""` // Content of status at time of edit; likely html-formatted but not guaranteed. + ContentWarning string `bun:",nullzero"` // Content warning of status at time of edit. + Text string `bun:""` // Original status text, without formatting, at time of edit. + Language string `bun:",nullzero"` // Status language at time of edit. + Sensitive *bool `bun:",nullzero,notnull,default:false"` // Status sensitive flag at time of edit. + AttachmentIDs []string `bun:",array"` // Database IDs of media attachments associated with status at time of edit. + Attachments []*MediaAttachment `bun:"-"` // Media attachments relating to .AttachmentIDs field (not always populated). + PollOptions []string `bun:",array"` // Poll options of status at time of edit, only set if status contains a poll. + PollVotes []int `bun:",array"` // Poll vote count at time of status edit, only set if poll votes were reset. + StatusID string `bun:"type:CHAR(26),nullzero,notnull"` // The originating status ID this is a historical edit of. + CreatedAt time.Time `bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // The creation time of this version of the status content (according to receiving server). + + // We don't bother having a *gtsmodel.Status model here + // as the StatusEdit is always just attached to a Status, + // so it doesn't need a self-reference back to it. +} + +// AttachmentsPopulated returns whether media attachments +// are populated according to current AttachmentIDs. +func (e *StatusEdit) AttachmentsPopulated() bool { + if len(e.AttachmentIDs) != len(e.Attachments) { + // this is the quickest indicator. + return false + } + for i, id := range e.AttachmentIDs { + if e.Attachments[i].ID != id { + return false + } + } + return true +} diff --git a/internal/id/ulid.go b/internal/id/ulid.go index 8de4cc4cc..2b50e444a 100644 --- a/internal/id/ulid.go +++ b/internal/id/ulid.go @@ -45,13 +45,14 @@ func NewULID() string { return ulid.String() } -// NewULIDFromTime returns a new ULID string using the given time, or an error if something goes wrong. -func NewULIDFromTime(t time.Time) (string, error) { +// NewULIDFromTime returns a new ULID string using +// given time, or from current time on any error. +func NewULIDFromTime(t time.Time) string { newUlid, err := ulid.New(ulid.Timestamp(t), rand.Reader) if err != nil { - return "", err + return NewULID() } - return newUlid.String(), nil + return newUlid.String() } // NewRandomULID returns a new ULID string using a random time in an ~80 year range around the current datetime, or an error if something goes wrong. diff --git a/internal/processing/workers/util.go b/internal/processing/workers/util.go index 62ea6c95c..cdc6a995d 100644 --- a/internal/processing/workers/util.go +++ b/internal/processing/workers/util.go @@ -537,11 +537,7 @@ func (u *utils) requestFave( } // Create + store new interaction request. - req, err = typeutils.StatusFaveToInteractionRequest(ctx, fave) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusFaveToInteractionRequest(fave) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } @@ -584,11 +580,7 @@ func (u *utils) requestReply( } // Create + store interaction request. - req, err = typeutils.StatusToInteractionRequest(ctx, reply) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusToInteractionRequest(reply) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } @@ -631,11 +623,7 @@ func (u *utils) requestAnnounce( } // Create + store interaction request. - req, err = typeutils.StatusToInteractionRequest(ctx, boost) - if err != nil { - return gtserror.Newf("error creating interaction request: %w", err) - } - + req = typeutils.StatusToInteractionRequest(boost) if err := u.state.DB.PutInteractionRequest(ctx, req); err != nil { return gtserror.Newf("db error storing interaction request: %w", err) } diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index 08a48fab3..1a7098673 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -111,6 +111,13 @@ func (c *Converter) ASRepresentationToAccount( acct.UpdatedAt = pub } + // Extract updated time if possible, i.e. last edited. + if upd := ap.GetUpdated(accountable); !upd.IsZero() { + acct.UpdatedAt = upd + } else { + acct.UpdatedAt = acct.CreatedAt + } + // Extract a preferred name (display name), fallback to username. if displayName := ap.ExtractName(accountable); displayName != "" { acct.DisplayName = displayName