mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-12-05 21:58:07 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] status refetch support (#1690)

Browse source 
* revamp http client to not limit requests, instead use sender worker

Signed-off-by: kim <grufwub@gmail.com>

* remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping

Signed-off-by: kim <grufwub@gmail.com>

* improve batch senders to keep popping recipients until remote URL found

Signed-off-by: kim <grufwub@gmail.com>

* fix recipient looping issue

Signed-off-by: kim <grufwub@gmail.com>

* move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP

Signed-off-by: kim <grufwub@gmail.com>

* first draft of status refetching logic

Signed-off-by: kim <grufwub@gmail.com>

* fix testrig to use new federation alloc func signature

Signed-off-by: kim <grufwub@gmail.com>

* fix log format directive

Signed-off-by: kim <grufwub@gmail.com>

* add status fetched_at migration

Signed-off-by: kim <grufwub@gmail.com>

* remove unused / unchecked for error types

Signed-off-by: kim <grufwub@gmail.com>

* add back the used type...

Signed-off-by: kim <grufwub@gmail.com>

* add separate internal getStatus() function for derefThread() that doesn't recurse

Signed-off-by: kim <grufwub@gmail.com>

* improved mention and media attachment error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix log and error format directives

Signed-off-by: kim <grufwub@gmail.com>

* update account deref to match status deref changes

Signed-off-by: kim <grufwub@gmail.com>

* very small code formatting change to make things clearer

Signed-off-by: kim <grufwub@gmail.com>

* add more code comments

Signed-off-by: kim <grufwub@gmail.com>

* improved code commenting

Signed-off-by: kim <grufwub@gmail.com>

* only check for required further derefs if needed

Signed-off-by: kim <grufwub@gmail.com>

* improved cache invalidation

Signed-off-by: kim <grufwub@gmail.com>

* tweak cache restarting to use a (very small) backoff

Signed-off-by: kim <grufwub@gmail.com>

* small readability changes and fixes

Signed-off-by: kim <grufwub@gmail.com>

* fix account sync issues

Signed-off-by: kim <grufwub@gmail.com>

* fix merge conflicts + update account enrichment to accept already-passed accountable

Signed-off-by: kim <grufwub@gmail.com>

* remove secondary function declaration

Signed-off-by: kim <grufwub@gmail.com>

* normalise dereferencer get status / account behaviour, fix remaining tests

Signed-off-by: kim <grufwub@gmail.com>

* fix remaining rebase conflicts, finish commenting code

Signed-off-by: kim <grufwub@gmail.com>

* appease the linter

Signed-off-by: kim <grufwub@gmail.com>

* add source file header

Signed-off-by: kim <grufwub@gmail.com>

* update to use TIMESTAMPTZ column type instead of just TIMESTAMP

Signed-off-by: kim <grufwub@gmail.com>

* don't pass in 'updated_at' to UpdateEmoji()

Signed-off-by: kim <grufwub@gmail.com>

* use new ap.Resolve{Account,Status}able() functions

Signed-off-by: kim <grufwub@gmail.com>

* remove the somewhat confusing rescoping of the same variable names

Signed-off-by: kim <grufwub@gmail.com>

* update migration file name, improved database delete error returns

Signed-off-by: kim <grufwub@gmail.com>

* formatting

Signed-off-by: kim <grufwub@gmail.com>

* improved multi-delete database functions to minimise DB calls

Signed-off-by: kim <grufwub@gmail.com>

* remove unused type

Signed-off-by: kim <grufwub@gmail.com>

* fix delete statements

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2023-05-12 10:15:54 +01:00 • committed by GitHub
commit 6c9d8e78eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
55 changed files with 1552 additions and 1118 deletions
Download patch file Download diff file Expand all files Collapse all files

438
internal/federation/dereferencing/account.go
View file

@ -38,7 +38,52 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, error) {
// accountUpToDate returns whether the given account model is both updateable (i.e.
// non-instance remote account) and whether it needs an update based on `fetched_at`.
func accountUpToDate(account *gtsmodel.Account) bool {
if account.IsLocal() {
// Can't update local accounts.
return true
}
if !account.CreatedAt.IsZero() && account.IsInstance() {
// Existing instance account. No need for update.
return true
}
// If this account was updated recently (last interval), we return as-is.
if next := account.FetchedAt.Add(6 * time.Hour); time.Now().Before(next) {
return true
}
return false
}
// GetAccountByURI: implements Dereferencer{}.GetAccountByURI.
func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, ap.Accountable, error) {
// Fetch and dereference account if necessary.
account, apubAcc, err := d.getAccountByURI(ctx,
requestUser,
uri,
)
if err != nil {
return nil, nil, err
}
if apubAcc != nil {
// This account was updated, enqueue re-dereference featured posts.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
})
}
return account, apubAcc, nil
}
// getAccountByURI is a package internal form of .GetAccountByURI() that doesn't bother dereferencing featured posts on update.
func (d *deref) getAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, ap.Accountable, error) {
var (
account *gtsmodel.Account
uriStr = uri.String()
@ -46,23 +91,23 @@ func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *ur
)
// Search the database for existing account with ID URI.
account, err = d.db.GetAccountByURI(ctx, uriStr)
account, err = d.state.DB.GetAccountByURI(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err)
return nil, nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err)
}
if account == nil {
// Else, search the database for existing by ID URL.
account, err = d.db.GetAccountByURL(ctx, uriStr)
account, err = d.state.DB.GetAccountByURL(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err)
return nil, nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err)
}
}
if account == nil {
// Ensure that this is isn't a search for a local account.
if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() {
return nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
return nil, nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
}
// Create and pass-through a new bare-bones model for dereferencing.
@ -70,163 +115,193 @@ func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *ur
ID: id.NewULID(),
Domain: uri.Host,
URI: uriStr,
}, d.defaultFetchLatest, false)
}, nil)
}
// Try to update existing account model
enriched, err := d.enrichAccount(ctx, requestUser, uri, account, d.defaultFetchLatest, false)
// Check whether needs update.
if accountUpToDate(account) {
return account, nil, nil
}
// Try to update existing account model.
latest, apubAcc, err := d.enrichAccount(ctx,
requestUser,
uri,
account,
nil,
)
if err != nil {
log.Errorf(ctx, "error enriching remote account: %v", err)
return account, nil // fall back to returning existing
// Update fetch-at to slow re-attempts.
account.FetchedAt = time.Now()
_ = d.state.DB.UpdateAccount(ctx, account, "fetched_at")
// Fallback to existing.
return account, nil, nil
}
return enriched, nil
return latest, apubAcc, nil
}
func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string) (*gtsmodel.Account, error) {
// GetAccountByUsernameDomain: implements Dereferencer{}.GetAccountByUsernameDomain.
func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string) (*gtsmodel.Account, ap.Accountable, error) {
if domain == config.GetHost() || domain == config.GetAccountDomain() {
// We do local lookups using an empty domain,
// else it will fail the db search below.
domain = ""
}
// Search the database for existing account with USERNAME@DOMAIN
account, err := d.db.GetAccountByUsernameDomain(ctx, username, domain)
// Search the database for existing account with USERNAME@DOMAIN.
account, err := d.state.DB.GetAccountByUsernameDomain(ctx, username, domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err)
return nil, nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err)
}
if account == nil {
// Check for failed local lookup.
if domain == "" {
return nil, NewErrNotRetrievable(err) // wrapped err will be db.ErrNoEntries
// failed local lookup, will be db.ErrNoEntries.
return nil, nil, NewErrNotRetrievable(err)
}
// Create and pass-through a new bare-bones model for dereferencing.
account = &gtsmodel.Account{
account, apubAcc, err := d.enrichAccount(ctx, requestUser, nil, &gtsmodel.Account{
ID: id.NewULID(),
Username: username,
Domain: domain,
}, nil)
if err != nil {
return nil, nil, err
}
// There's no known account to fall back on,
// so return error if we can't enrich account.
return d.enrichAccount(ctx, requestUser, nil, account, d.defaultFetchLatest, false)
// This account was updated, enqueue dereference featured posts.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
})
return account, apubAcc, nil
}
// We knew about this account already;
// try to update existing account model.
enriched, err := d.enrichAccount(ctx, requestUser, nil, account, d.defaultFetchLatest, false)
if err != nil {
log.Errorf(ctx, "error enriching account from remote: %v", err)
return account, nil // fall back to returning unchanged existing account model
}
return enriched, nil
}
func (d *deref) RefreshAccount(ctx context.Context, requestUser string, accountable ap.Accountable, account *gtsmodel.Account) (*gtsmodel.Account, error) {
// To avoid unnecessarily refetching multiple times from remote,
// we can just pass in the Accountable object that we received,
// if it was defined. If not, fall back to default fetch func.
var f fetchLatest
if accountable != nil {
f = func(
_ context.Context,
_ transport.Transport,
_ *url.URL,
_ string,
) (ap.Accountable, *gtsmodel.Account, error) {
return accountable, account, nil
}
} else {
f = d.defaultFetchLatest
}
// Set 'force' to 'true' to always fetch latest media etc.
return d.enrichAccount(ctx, requestUser, nil, account, f, true)
}
// fetchLatest defines a function for using a transport and uri to fetch the fetchLatest
// version of an account (and its AP representation) from a remote instance.
type fetchLatest func(ctx context.Context, transport transport.Transport, uri *url.URL, accountDomain string) (ap.Accountable, *gtsmodel.Account, error)
// defaultFetchLatest deduplicates latest fetching code that is used in several
// different functions. It simply calls the remote uri using the given transport,
// parses a returned AP representation into an account, and then returns both.
func (d *deref) defaultFetchLatest(ctx context.Context, transport transport.Transport, uri *url.URL, accountDomain string) (ap.Accountable, *gtsmodel.Account, error) {
// Dereference this account to get the latest available.
apubAcc, err := d.dereferenceAccountable(ctx, transport, uri)
if err != nil {
return nil, nil, fmt.Errorf("error dereferencing account %s: %w", uri, err)
}
// Convert the dereferenced AP account object to our GTS model.
latestAcc, err := d.typeConverter.ASRepresentationToAccount(
ctx, apubAcc, accountDomain,
// Try to update existing account model.
latest, apubAcc, err := d.RefreshAccount(ctx,
requestUser,
account,
nil,
false,
)
if err != nil {
return nil, nil, fmt.Errorf("error converting accountable to gts model for account %s: %w", uri, err)
// Fallback to existing.
return account, nil, nil //nolint
}
return apubAcc, latestAcc, nil
return latest, apubAcc, nil
}
// enrichAccount will ensure the given account is the most up-to-date model of the account, re-webfingering and re-dereferencing if necessary.
func (d *deref) enrichAccount(
ctx context.Context,
requestUser string,
uri *url.URL,
account *gtsmodel.Account,
f fetchLatest,
force bool,
) (*gtsmodel.Account, error) {
if account.IsLocal() {
// Can't update local accounts.
return account, nil
// RefreshAccount: implements Dereferencer{}.RefreshAccount.
func (d *deref) RefreshAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, apubAcc ap.Accountable, force bool) (*gtsmodel.Account, ap.Accountable, error) {
// Check whether needs update (and not forced).
if accountUpToDate(account) && !force {
return account, nil, nil
}
if !account.CreatedAt.IsZero() && account.IsInstance() {
// Existing instance account. No need for update.
return account, nil
}
if !force {
const interval = time.Hour * 48
// If this account was updated recently (last interval), we return as-is.
if next := account.FetchedAt.Add(interval); time.Now().Before(next) {
return account, nil
}
}
// Pre-fetch a transport for requesting username, used by later deref procedures.
transport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
// Parse the URI from account.
uri, err := url.Parse(account.URI)
if err != nil {
return nil, fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
return nil, nil, fmt.Errorf("RefreshAccount: invalid account uri %q: %w", account.URI, err)
}
// Try to update + deref existing account model.
latest, apubAcc, err := d.enrichAccount(ctx,
requestUser,
uri,
account,
apubAcc,
)
if err != nil {
log.Errorf(ctx, "error enriching remote account: %v", err)
// Update fetch-at to slow re-attempts.
account.FetchedAt = time.Now()
_ = d.state.DB.UpdateAccount(ctx, account, "fetched_at")
return nil, nil, err
}
// This account was updated, enqueue re-dereference featured posts.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
if err := d.dereferenceAccountFeatured(ctx, requestUser, account); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
})
return latest, apubAcc, nil
}
// RefreshAccountAsync: implements Dereferencer{}.RefreshAccountAsync.
func (d *deref) RefreshAccountAsync(ctx context.Context, requestUser string, account *gtsmodel.Account, apubAcc ap.Accountable, force bool) {
// Check whether needs update (and not forced).
if accountUpToDate(account) && !force {
return
}
// Parse the URI from account.
uri, err := url.Parse(account.URI)
if err != nil {
log.Errorf(ctx, "RefreshAccountAsync: invalid account uri %q: %v", account.URI, err)
return
}
// Enqueue a worker function to enrich this account async.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
latest, _, err := d.enrichAccount(ctx, requestUser, uri, account, apubAcc)
if err != nil {
log.Errorf(ctx, "error enriching remote account: %v", err)
return
}
// This account was updated, re-dereference account featured posts.
if err := d.dereferenceAccountFeatured(ctx, requestUser, latest); err != nil {
log.Errorf(ctx, "error fetching account featured collection: %v", err)
}
})
}
// enrichAccount will enrich the given account, whether a new barebones model, or existing model from the database. It handles necessary dereferencing, webfingering etc.
func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.URL, account *gtsmodel.Account, apubAcc ap.Accountable) (*gtsmodel.Account, ap.Accountable, error) {
// Pre-fetch a transport for requesting username, used by later deref procedures.
tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
if err != nil {
return nil, nil, fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
}
if account.Username != "" {
// A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info.
accDomain, accURI, err := d.fingerRemoteAccount(ctx, transport, account.Username, account.Domain)
accDomain, accURI, err := d.fingerRemoteAccount(ctx, tsport, account.Username, account.Domain)
if err != nil {
if account.URI == "" {
// this is a new account (to us) with username@domain but failed webfinger, nothing more we can do.
return nil, nil, &ErrNotRetrievable{fmt.Errorf("enrichAccount: error webfingering account: %w", err)}
}
switch {
case err != nil && account.URI == "":
// this is a new account (to us) with username@domain but failed webfinger, nothing more we can do.
return nil, fmt.Errorf("enrichAccount: error webfingering account: %w", err)
case err != nil:
// Simply log this error and move on, we already have an account URI.
log.Errorf(ctx, "error webfingering[1] remote account %s@%s: %v", account.Username, account.Domain, err)
}
case err == nil:
if err == nil {
if account.Domain != accDomain {
// Domain has changed, assume the activitypub
// account data provided may not be the latest.
apubAcc = nil
// After webfinger, we now have correct account domain from which we can do a final DB check.
alreadyAccount, err := d.db.GetAccountByUsernameDomain(ctx, account.Username, accDomain)
alreadyAccount, err := d.state.DB.GetAccountByUsernameDomain(ctx, account.Username, accDomain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("enrichAccount: db err looking for account again after webfinger: %w", err)
return nil, nil, fmt.Errorf("enrichAccount: db err looking for account again after webfinger: %w", err)
}
if err == nil {
if alreadyAccount != nil {
// Enrich existing account.
account = alreadyAccount
}
@ -240,30 +315,49 @@ func (d *deref) enrichAccount(
}
if uri == nil {
var err error
// No URI provided / found, must parse from account.
uri, err = url.Parse(account.URI)
if err != nil {
return nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err)
return nil, nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err)
}
}
// Check whether this account URI is a blocked domain / subdomain.
if blocked, err := d.db.IsDomainBlocked(ctx, uri.Host); err != nil {
return nil, newErrDB(fmt.Errorf("enrichAccount: error checking blocked domain: %w", err))
if blocked, err := d.state.DB.IsDomainBlocked(ctx, uri.Host); err != nil {
return nil, nil, fmt.Errorf("enrichAccount: error checking blocked domain: %w", err)
} else if blocked {
return nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host)
return nil, nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host)
}
// Mark deref+update handshake start.
d.startHandshake(requestUser, uri)
defer d.stopHandshake(requestUser, uri)
// Fetch latest version of the account, dereferencing if necessary.
apubAcc, latestAcc, err := f(ctx, transport, uri, account.Domain)
if err != nil {
return nil, fmt.Errorf("enrichAccount: error calling fetchLatest function: %w", err)
// By default we assume that apubAcc has been passed,
// indicating that the given account is already latest.
latestAcc := account
if apubAcc == nil {
// Dereference latest version of the account.
b, err := tsport.Dereference(ctx, uri)
if err != nil {
return nil, nil, &ErrNotRetrievable{fmt.Errorf("enrichAccount: error deferencing %s: %w", uri, err)}
}
// Attempt to resolve ActivityPub account from data.
apubAcc, err = ap.ResolveAccountable(ctx, b)
if err != nil {
return nil, nil, fmt.Errorf("enrichAccount: error resolving accountable from data for account %s: %w", uri, err)
}
// Convert the dereferenced AP account object to our GTS model.
latestAcc, err = d.typeConverter.ASRepresentationToAccount(ctx,
apubAcc,
account.Domain,
)
if err != nil {
return nil, nil, fmt.Errorf("enrichAccount: error converting accountable to gts model for account %s: %w", uri, err)
}
}
if account.Username == "" {
@ -281,11 +375,17 @@ func (d *deref) enrichAccount(
// Assume the host from the returned ActivityPub representation.
idProp := apubAcc.GetJSONLDId()
if idProp == nil || !idProp.IsIRI() {
return nil, errors.New("enrichAccount: no id property found on person, or id was not an iri")
return nil, nil, errors.New("enrichAccount: no id property found on person, or id was not an iri")
}
// Get IRI host value.
accHost := idProp.GetIRI().Host
accDomain, _, err := d.fingerRemoteAccount(ctx, transport, latestAcc.Username, accHost)
latestAcc.Domain, _, err = d.fingerRemoteAccount(ctx,
tsport,
latestAcc.Username,
accHost,
)
if err != nil {
// We still couldn't webfinger the account, so we're not certain
// what the accountDomain actually is. Still, we can make a solid
@ -293,9 +393,6 @@ func (d *deref) enrichAccount(
// If we're wrong, we can just try again in a couple days.
log.Errorf(ctx, "error webfingering[2] remote account %s@%s: %v", latestAcc.Username, accHost, err)
latestAcc.Domain = accHost
} else {
// Update account with latest info.
latestAcc.Domain = accDomain
}
}
@ -307,14 +404,15 @@ func (d *deref) enrichAccount(
latestAcc.AvatarMediaAttachmentID = account.AvatarMediaAttachmentID
latestAcc.HeaderMediaAttachmentID = account.HeaderMediaAttachmentID
if force || (latestAcc.AvatarRemoteURL != account.AvatarRemoteURL) {
if (latestAcc.AvatarMediaAttachmentID == "") ||
(latestAcc.AvatarRemoteURL != account.AvatarRemoteURL) {
// Reset the avatar media ID (handles removed).
latestAcc.AvatarMediaAttachmentID = ""
if latestAcc.AvatarRemoteURL != "" {
// Avatar has changed to a new one, fetch up-to-date copy and use new ID.
latestAcc.AvatarMediaAttachmentID, err = d.fetchRemoteAccountAvatar(ctx,
transport,
tsport,
latestAcc.AvatarRemoteURL,
latestAcc.ID,
)
@ -328,14 +426,15 @@ func (d *deref) enrichAccount(
}
}
if force || (latestAcc.HeaderRemoteURL != account.HeaderRemoteURL) {
if (latestAcc.HeaderMediaAttachmentID == "") ||
(latestAcc.HeaderRemoteURL != account.HeaderRemoteURL) {
// Reset the header media ID (handles removed).
latestAcc.HeaderMediaAttachmentID = ""
if latestAcc.HeaderRemoteURL != "" {
// Header has changed to a new one, fetch up-to-date copy and use new ID.
latestAcc.HeaderMediaAttachmentID, err = d.fetchRemoteAccountHeader(ctx,
transport,
tsport,
latestAcc.HeaderRemoteURL,
latestAcc.ID,
)
@ -363,15 +462,16 @@ func (d *deref) enrichAccount(
latestAcc.UpdatedAt = latestAcc.FetchedAt
// This is new, put it in the database.
err := d.db.PutAccount(ctx, latestAcc)
err := d.state.DB.PutAccount(ctx, latestAcc)
if errors.Is(err, db.ErrAlreadyExists) {
// TODO: replace this quick fix with per-URI deref locks.
latestAcc, err = d.db.GetAccountByURI(ctx, latestAcc.URI)
latestAcc, err = d.state.DB.GetAccountByURI(ctx, latestAcc.URI)
return latestAcc, nil, err
}
if err != nil {
return nil, fmt.Errorf("enrichAccount: error putting in database: %w", err)
return nil, nil, fmt.Errorf("enrichAccount: error putting in database: %w", err)
}
} else {
// Set time of update from the last-fetched date.
@ -382,35 +482,12 @@ func (d *deref) enrichAccount(
latestAcc.Language = account.Language
// This is an existing account, update the model in the database.
if err := d.db.UpdateAccount(ctx, latestAcc); err != nil {
return nil, fmt.Errorf("enrichAccount: error updating database: %w", err)
if err := d.state.DB.UpdateAccount(ctx, latestAcc); err != nil {
return nil, nil, fmt.Errorf("enrichAccount: error updating database: %w", err)
}
}
if latestAcc.FeaturedCollectionURI != "" {
// Fetch this account's pinned statuses, now that the account is in the database.
//
// The order is important here: if we tried to fetch the pinned statuses before
// storing the account, the process might end up calling enrichAccount again,
// causing us to get stuck in a loop. By calling it now, we make sure this doesn't
// happen!
if err := d.fetchRemoteAccountFeatured(ctx, requestUser, latestAcc.FeaturedCollectionURI, latestAcc.ID); err != nil {
log.Errorf(ctx, "error fetching featured collection for account %s: %v", uri, err)
}
}
return latestAcc, nil
}
// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
// it finds as something that an account model can be constructed out of.
func (d *deref) dereferenceAccountable(ctx context.Context, transport transport.Transport, remoteAccountID *url.URL) (ap.Accountable, error) {
b, err := transport.Dereference(ctx, remoteAccountID)
if err != nil {
return nil, fmt.Errorf("dereferenceAccountable: error deferencing %s: %w", remoteAccountID.String(), err)
}
return ap.ResolveAccountable(ctx, b)
return latestAcc, apubAcc, nil
}
func (d *deref) fetchRemoteAccountAvatar(ctx context.Context, tsport transport.Transport, avatarURL string, accountID string) (string, error) {
@ -531,7 +608,7 @@ func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gts
if len(maybeEmojiIDs) > len(maybeEmojis) {
maybeEmojis = make([]*gtsmodel.Emoji, 0, len(maybeEmojiIDs))
for _, emojiID := range maybeEmojiIDs {
maybeEmoji, err := d.db.GetEmojiByID(ctx, emojiID)
maybeEmoji, err := d.state.DB.GetEmojiByID(ctx, emojiID)
if err != nil {
return false, err
}
@ -631,18 +708,18 @@ func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gts
return changed, nil
}
// fetchRemoteAccountFeatured dereferences an account's featuredCollectionURI (if not empty).
// For each discovered status, this status will be dereferenced (if necessary) and marked as
// pinned (if necessary). Then, old pins will be removed if they're not included in new pins.
func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUsername string, featuredCollectionURI string, accountID string) error {
uri, err := url.Parse(featuredCollectionURI)
// dereferenceAccountFeatured dereferences an account's featuredCollectionURI (if not empty). For each discovered status, this status will
// be dereferenced (if necessary) and marked as pinned (if necessary). Then, old pins will be removed if they're not included in new pins.
func (d *deref) dereferenceAccountFeatured(ctx context.Context, requestUser string, account *gtsmodel.Account) error {
uri, err := url.Parse(account.FeaturedCollectionURI)
if err != nil {
return err
}
tsport, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
// Pre-fetch a transport for requesting username, used by later deref procedures.
tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
if err != nil {
return err
return fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
}
b, err := tsport.Dereference(ctx, uri)
@ -661,7 +738,7 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
}
if t.GetTypeName() != ap.ObjectOrderedCollection {
return fmt.Errorf("%s was not an OrderedCollection", featuredCollectionURI)
return fmt.Errorf("%s was not an OrderedCollection", uri)
}
collection, ok := t.(vocab.ActivityStreamsOrderedCollection)
@ -675,7 +752,7 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
}
// Get previous pinned statuses (we'll need these later).
wasPinned, err := d.db.GetAccountPinnedStatuses(ctx, accountID)
wasPinned, err := d.state.DB.GetAccountPinnedStatuses(ctx, account.ID)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return fmt.Errorf("error getting account pinned statuses: %w", err)
}
@ -720,11 +797,10 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
// we still know it was *meant* to be pinned.
statusURIs = append(statusURIs, statusURI)
status, _, err := d.GetStatus(ctx, requestingUsername, statusURI, false, false)
status, _, err := d.getStatusByURI(ctx, requestUser, statusURI)
if err != nil {
// We couldn't get the status, bummer.
// Just log + move on, we can try later.
log.Errorf(ctx, "error getting status from featured collection %s: %s", featuredCollectionURI, err)
// We couldn't get the status, bummer. Just log + move on, we can try later.
log.Errorf(ctx, "error getting status from featured collection %s: %v", statusURI, err)
continue
}
@ -733,7 +809,7 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
continue
}
if status.AccountID != accountID {
if status.AccountID != account.ID {
// Someone's pinned a status that doesn't
// belong to them, this doesn't work for us.
continue
@ -748,8 +824,9 @@ func (d *deref) fetchRemoteAccountFeatured(ctx context.Context, requestingUserna
// All conditions are met for this status to
// be pinned, so we can finally update it.
status.PinnedAt = time.Now()
if err := d.db.UpdateStatus(ctx, status, "pinned_at"); err != nil {
log.Errorf(ctx, "error updating status in featured collection %s: %s", featuredCollectionURI, err)
if err := d.state.DB.UpdateStatus(ctx, status, "pinned_at"); err != nil {
log.Errorf(ctx, "error updating status in featured collection %s: %v", status.URI, err)
continue
}
}
@ -768,8 +845,9 @@ outerLoop:
// Status was pinned before, but is not included
// in most recent pinned uris, so unpin it now.
status.PinnedAt = time.Time{}
if err := d.db.UpdateStatus(ctx, status, "pinned_at"); err != nil {
return fmt.Errorf("error unpinning status: %w", err)
if err := d.state.DB.UpdateStatus(ctx, status, "pinned_at"); err != nil {
log.Errorf(ctx, "error unpinning status %s: %v", status.URI, err)
continue
}
}

20
internal/federation/dereferencing/account_test.go
View file

@ -37,7 +37,7 @@ func (suite *AccountTestSuite) TestDereferenceGroup() {
fetchingAccount := suite.testAccounts["local_account_1"]
groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group")
group, err := suite.dereferencer.GetAccountByURI(
group, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
groupURL,
@ -61,7 +61,7 @@ func (suite *AccountTestSuite) TestDereferenceService() {
fetchingAccount := suite.testAccounts["local_account_1"]
serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh")
service, err := suite.dereferencer.GetAccountByURI(
service, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
serviceURL,
@ -93,7 +93,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURL() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
@ -112,7 +112,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURLNoSharedInb
suite.FailNow(err.Error())
}
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
@ -126,7 +126,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsername() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
@ -140,7 +140,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomain() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
@ -154,7 +154,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
fetchedAccount, _, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
targetAccount.Username,
@ -168,7 +168,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
fetchedAccount, _, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
"thisaccountdoesnotexist",
@ -183,7 +183,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername()
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDomain() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
fetchedAccount, _, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
"thisaccountdoesnotexist",
@ -198,7 +198,7 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDom
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
fetchedAccount, _, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"),

9
internal/federation/dereferencing/announce.go
View file

@ -40,7 +40,7 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
}
// Check whether the originating status is from a blocked host
if blocked, err := d.db.IsDomainBlocked(ctx, boostedURI.Host); blocked || err != nil {
if blocked, err := d.state.DB.IsDomainBlocked(ctx, boostedURI.Host); blocked || err != nil {
return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedURI.Host)
}
@ -48,7 +48,7 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
if boostedURI.Host == config.GetHost() {
// This is a local status, fetch from the database
status, err := d.db.GetStatusByURI(ctx, boostedURI.String())
status, err := d.state.DB.GetStatusByURI(ctx, boostedURI.String())
if err != nil {
return fmt.Errorf("DereferenceAnnounce: error fetching local status %q: %v", announce.BoostOf.URI, err)
}
@ -57,14 +57,11 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
boostedStatus = status
} else {
// This is a boost of a remote status, we need to dereference it.
status, statusable, err := d.GetStatus(ctx, requestingUsername, boostedURI, true, true)
status, _, err := d.GetStatusByURI(ctx, requestingUsername, boostedURI)
if err != nil {
return fmt.Errorf("DereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.BoostOf.URI, err)
}
// Dereference all statuses in the thread of the boosted status
d.DereferenceThread(ctx, requestingUsername, boostedURI, status, statusable)
// Set boosted status
boostedStatus = status
}

6
internal/federation/dereferencing/collectionpage.go
View file

@ -29,9 +29,9 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/ap"
)
// DereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong.
func (d *deref) DereferenceCollectionPage(ctx context.Context, username string, pageIRI *url.URL) (ap.CollectionPageable, error) {
if blocked, err := d.db.IsDomainBlocked(ctx, pageIRI.Host); blocked || err != nil {
// dereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong.
func (d *deref) dereferenceCollectionPage(ctx context.Context, username string, pageIRI *url.URL) (ap.CollectionPageable, error) {
if blocked, err := d.state.DB.IsDomainBlocked(ctx, pageIRI.Host); blocked || err != nil {
return nil, fmt.Errorf("DereferenceCollectionPage: domain %s is blocked", pageIRI.Host)
}

51
internal/federation/dereferencing/dereferencer.go
View file

@ -24,42 +24,61 @@ import (
"codeberg.org/gruf/go-mutexes"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
type Dereferencer interface {
// GetAccountByURI will attempt to fetch an account by its URI, first checking the database and in the case of a remote account will either check the
// last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before.
GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, error)
// GetAccountByURI will attempt to fetch an accounts by its URI, first checking the database. In the case of a newly-met remote model, or a remote model
// whose last_fetched date is beyond a certain interval, the account will be dereferenced. In the case of dereferencing, some low-priority account information
// may be enqueued for asynchronous fetching, e.g. featured account statuses (pins). An ActivityPub object indicates the account was dereferenced.
GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Account, ap.Accountable, error)
// GetAccountByUsernameDomain will attempt to fetch an account by username@domain, first checking the database and in the case of a remote account will either
// check the last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before.
GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string) (*gtsmodel.Account, error)
// GetAccountByUsernameDomain will attempt to fetch an accounts by its username@domain, first checking the database. In the case of a newly-met remote model,
// or a remote model whose last_fetched date is beyond a certain interval, the account will be dereferenced. In the case of dereferencing, some low-priority
// account information may be enqueued for asynchronous fetching, e.g. featured account statuses (pins). An ActivityPub object indicates the account was dereferenced.
GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string) (*gtsmodel.Account, ap.Accountable, error)
// RefreshAccount forces a refresh of the given account by fetching the current/latest state of the account from the remote instance.
// An updated account model is returned, but not yet inserted/updated in the database; this is the caller's responsibility.
RefreshAccount(ctx context.Context, requestUser string, accountable ap.Accountable, account *gtsmodel.Account) (*gtsmodel.Account, error)
// RefreshAccount updates the given account if remote and last_fetched is beyond fetch interval, or if force is set. An updated account model is returned,
// but in the case of dereferencing, some low-priority account information may be enqueued for asynchronous fetching, e.g. featured account statuses (pins).
// An ActivityPub object indicates the account was dereferenced (i.e. updated).
RefreshAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, apubAcc ap.Accountable, force bool) (*gtsmodel.Account, ap.Accountable, error)
GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
// RefreshAccountAsync enqueues the given account for an asychronous update fetching, if last_fetched is beyond fetch interval, or if forcc is set.
// This is a more optimized form of manually enqueueing .UpdateAccount() to the federation worker, since it only enqueues update if necessary.
RefreshAccountAsync(ctx context.Context, requestUser string, account *gtsmodel.Account, apubAcc ap.Accountable, force bool)
// GetStatusByURI will attempt to fetch a status by its URI, first checking the database. In the case of a newly-met remote model, or a remote model
// whose last_fetched date is beyond a certain interval, the status will be dereferenced. In the case of dereferencing, some low-priority status information
// may be enqueued for asynchronous fetching, e.g. dereferencing the remainder of the status thread. An ActivityPub object indicates the status was dereferenced.
GetStatusByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Status, ap.Statusable, error)
// RefreshStatus updates the given status if remote and last_fetched is beyond fetch interval, or if force is set. An updated status model is returned,
// but in the case of dereferencing, some low-priority status information may be enqueued for asynchronous fetching, e.g. dereferencing the remainder of the
// status thread. An ActivityPub object indicates the status was dereferenced (i.e. updated).
RefreshStatus(ctx context.Context, requestUser string, status *gtsmodel.Status, apubStatus ap.Statusable, force bool) (*gtsmodel.Status, ap.Statusable, error)
// RefreshStatusAsync enqueues the given status for an asychronous update fetching, if last_fetched is beyond fetch interval, or if force is set.
// This is a more optimized form of manually enqueueing .UpdateStatus() to the federation worker, since it only enqueues update if necessary.
RefreshStatusAsync(ctx context.Context, requestUser string, status *gtsmodel.Status, apubStatus ap.Statusable, force bool)
EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)
GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
DereferenceThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable)
GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error)
GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, domain string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error)
Handshaking(username string, remoteAccountID *url.URL) bool
}
type deref struct {
db db.DB
state *state.State
typeConverter typeutils.TypeConverter
transportController transport.Controller
mediaManager media.Manager
@ -74,9 +93,9 @@ type deref struct {
}
// NewDereferencer returns a Dereferencer initialized with the given parameters.
func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer {
func NewDereferencer(state *state.State, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer {
return &deref{
db: db,
state: state,
typeConverter: typeConverter,
transportController: transportController,
mediaManager: mediaManager,

2
internal/federation/dereferencing/dereferencer_test.go
View file

@ -65,7 +65,7 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
suite.state.DB = suite.db
suite.state.Storage = suite.storage
media := testrig.NewTestMediaManager(&suite.state)
suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media")), media)
suite.dereferencer = dereferencing.NewDereferencer(&suite.state, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(&suite.state, testrig.NewMockHTTPClient(nil, "../../../testrig/media")), media)
testrig.StandardDBSetup(suite.db, nil)
}

2
internal/federation/dereferencing/emoji.go
View file

@ -109,7 +109,7 @@ func (d *deref) populateEmojis(ctx context.Context, rawEmojis []*gtsmodel.Emoji,
// it should be fleshed out already and we won't
// have to get it from the database again
gotEmoji = e
} else if gotEmoji, err = d.db.GetEmojiByShortcodeDomain(ctx, e.Shortcode, e.Domain); err != nil && err != db.ErrNoEntries {
} else if gotEmoji, err = d.state.DB.GetEmojiByShortcodeDomain(ctx, e.Shortcode, e.Domain); err != nil && err != db.ErrNoEntries {
log.Errorf(ctx, "error checking database for emoji %s: %s", shortcodeDomain, err)
continue
}

66
internal/federation/dereferencing/error.go
View file

@ -19,25 +19,8 @@ package dereferencing
import (
"fmt"
"net/http"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
// ErrDB denotes that a proper error has occurred when doing
// a database call, as opposed to a simple db.ErrNoEntries.
type ErrDB struct {
wrapped error
}
func (err *ErrDB) Error() string {
return fmt.Sprintf("database error during dereferencing: %v", err.wrapped)
}
func newErrDB(err error) error {
return &ErrDB{wrapped: err}
}
// ErrNotRetrievable denotes that an item could not be dereferenced
// with the given parameters.
type ErrNotRetrievable struct {
@ -51,52 +34,3 @@ func (err *ErrNotRetrievable) Error() string {
func NewErrNotRetrievable(err error) error {
return &ErrNotRetrievable{wrapped: err}
}
// ErrTransportError indicates that something unforeseen went wrong creating
// a transport, or while making an http call to a remote resource with a transport.
type ErrTransportError struct {
wrapped error
}
func (err *ErrTransportError) Error() string {
return fmt.Sprintf("transport error: %v", err.wrapped)
}
func newErrTransportError(err error) error {
return &ErrTransportError{wrapped: err}
}
// ErrOther denotes some other kind of weird error, perhaps from a malformed json
// or some other weird crapola.
type ErrOther struct {
wrapped error
}
func (err *ErrOther) Error() string {
return fmt.Sprintf("unexpected error: %v", err.wrapped)
}
func newErrOther(err error) error {
return &ErrOther{wrapped: err}
}
func wrapDerefError(derefErr error, fluff string) error {
// Wrap with fluff.
err := derefErr
if fluff != "" {
err = fmt.Errorf("%s: %w", fluff, derefErr)
}
// Check for unretrievable HTTP status code errors.
if code := gtserror.StatusCode(derefErr); // nocollapse
code == http.StatusGone || code == http.StatusNotFound {
return NewErrNotRetrievable(err)
}
// Check for other untrievable errors.
if gtserror.NotFound(derefErr) {
return NewErrNotRetrievable(err)
}
return err
}

3
internal/federation/dereferencing/finger.go
View file

@ -69,6 +69,5 @@ func (d *deref) fingerRemoteAccount(ctx context.Context, transport transport.Tra
}
}
err = errors.New("fingerRemoteAccount: no match found in webfinger response")
return
return "", nil, errors.New("fingerRemoteAccount: no match found in webfinger response")
}

2
internal/federation/dereferencing/instance.go
View file

@ -26,7 +26,7 @@ import (
)
func (d *deref) GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
if blocked, err := d.db.IsDomainBlocked(ctx, remoteInstanceURI.Host); blocked || err != nil {
if blocked, err := d.state.DB.IsDomainBlocked(ctx, remoteInstanceURI.Host); blocked || err != nil {
return nil, fmt.Errorf("GetRemoteInstance: domain %s is blocked", remoteInstanceURI.Host)
}

649
internal/federation/dereferencing/status.go
View file

@ -21,8 +21,9 @@ import (
"context"
"errors"
"fmt"
"io"
"net/url"
"strings"
"time"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/config"
@ -34,374 +35,430 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
// EnrichRemoteStatus takes a remote status that's already been inserted into the database in a minimal form,
// and populates it with additional fields, media, etc.
//
// EnrichRemoteStatus is mostly useful for calling after a status has been initially created by
// the federatingDB's Create function, but additional dereferencing is needed on it.
func (d *deref) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) {
if err := d.populateStatusFields(ctx, status, username, includeParent); err != nil {
return nil, err
// statusUpToDate returns whether the given status model is both updateable
// (i.e. remote status) and whether it needs an update based on `fetched_at`.
func statusUpToDate(status *gtsmodel.Status) bool {
if *status.Local {
// Can't update local statuses.
return true
}
if err := d.db.UpdateStatus(ctx, status); err != nil {
return nil, err
// If this status was updated recently (last interval), we return as-is.
if next := status.FetchedAt.Add(2 * time.Hour); time.Now().Before(next) {
return true
}
return status, nil
return false
}
// GetStatus completely dereferences a status, converts it to a GtS model status,
// puts it in the database, and returns it to a caller.
//
// If refetch is true, then regardless of whether we have the original status in the database or not,
// the ap.Statusable representation of the status will be dereferenced and returned.
//
// If refetch is false, the ap.Statusable will only be returned if this is a new status, so callers
// should check whether or not this is nil.
//
// GetAccount will guard against trying to do http calls to fetch a status that belongs to this instance.
// Instead of making calls, it will just return the status early if it finds it, or return an error.
func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) {
uriString := statusURI.String()
// GetStatus: implements Dereferencer{}.GetStatus().
func (d *deref) GetStatusByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Status, ap.Statusable, error) {
// Fetch and dereference status if necessary.
status, apubStatus, err := d.getStatusByURI(ctx,
requestUser,
uri,
)
if err != nil {
return nil, nil, err
}
// try to get by URI first
status, dbErr := d.db.GetStatusByURI(ctx, uriString)
if dbErr != nil {
if !errors.Is(dbErr, db.ErrNoEntries) {
// real error
return nil, nil, newErrDB(fmt.Errorf("GetRemoteStatus: error during GetStatusByURI for %s: %w", uriString, dbErr))
if apubStatus != nil {
// This status was updated, enqueue re-dereferencing the whole thread.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
d.dereferenceThread(ctx, requestUser, uri, status, apubStatus)
})
}
return status, apubStatus, nil
}
// getStatusByURI is a package internal form of .GetStatusByURI() that doesn't bother dereferencing the whole thread on update.
func (d *deref) getStatusByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Status, ap.Statusable, error) {
var (
status *gtsmodel.Status
uriStr = uri.String()
err error
)
// Search the database for existing status with ID URI.
status, err = d.state.DB.GetStatusByURI(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, nil, fmt.Errorf("GetStatusByURI: error checking database for status %s by uri: %w", uriStr, err)
}
if status == nil {
// Else, search the database for existing by ID URL.
status, err = d.state.DB.GetStatusByURL(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, nil, fmt.Errorf("GetStatusByURI: error checking database for status %s by url: %w", uriStr, err)
}
// no problem, just press on
} else if !refetch {
// we already had the status and we aren't being asked to refetch the AP representation
}
if status == nil {
// Ensure that this is isn't a search for a local status.
if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() {
return nil, nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
}
// Create and pass-through a new bare-bones model for deref.
return d.enrichStatus(ctx, requestUser, uri, &gtsmodel.Status{
Local: func() *bool { var false bool; return &false }(),
URI: uriStr,
}, nil)
}
// Try to update + deref existing status model.
latest, apubStatus, err := d.enrichStatus(ctx,
requestUser,
uri,
status,
nil,
)
if err != nil {
log.Errorf(ctx, "error enriching remote status: %v", err)
// Update fetch-at to slow re-attempts.
status.FetchedAt = time.Now()
_ = d.state.DB.UpdateStatus(ctx, status, "fetched_at")
// Fallback to existing.
return status, nil, nil
}
// try to get by URL if we couldn't get by URI now
if status == nil {
status, dbErr = d.db.GetStatusByURL(ctx, uriString)
if dbErr != nil {
if !errors.Is(dbErr, db.ErrNoEntries) {
// real error
return nil, nil, newErrDB(fmt.Errorf("GetRemoteStatus: error during GetStatusByURI for %s: %w", uriString, dbErr))
}
// no problem, just press on
} else if !refetch {
// we already had the status and we aren't being asked to refetch the AP representation
return status, nil, nil
}
}
// guard against having our own statuses passed in
if host := statusURI.Host; host == config.GetHost() || host == config.GetAccountDomain() {
// this is our status, definitely don't search for it
if status != nil {
return status, nil, nil
}
return nil, nil, NewErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString))
}
// if we got here, either we didn't have the status
// in the db, or we had it but need to refetch it
tsport, err := d.transportController.NewTransportForUsername(ctx, username)
if err != nil {
return nil, nil, newErrTransportError(fmt.Errorf("GetRemoteStatus: error creating transport for %s: %w", username, err))
}
statusable, derefErr := d.dereferenceStatusable(ctx, tsport, statusURI)
if derefErr != nil {
return nil, nil, wrapDerefError(derefErr, "GetRemoteStatus: error dereferencing statusable")
}
if status != nil && refetch {
// we already had the status in the db, and we've also
// now fetched the AP representation as requested
return status, statusable, nil
}
// from here on out we can consider this to be a 'new' status because we didn't have the status in the db already
accountURI, err := ap.ExtractAttributedTo(statusable)
if err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %w", err))
}
// we need to get the author of the status else we can't serialize it properly
if _, err = d.GetAccountByURI(ctx, username, accountURI); err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err))
}
status, err = d.typeConverter.ASStatusToStatus(ctx, statusable)
if err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: error converting statusable to status: %s", err))
}
ulid, err := id.NewULIDFromTime(status.CreatedAt)
if err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: error generating new id for status: %s", err))
}
status.ID = ulid
if err := d.populateStatusFields(ctx, status, username, includeParent); err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err))
}
if err := d.db.PutStatus(ctx, status); err != nil && !errors.Is(err, db.ErrAlreadyExists) {
return nil, nil, newErrDB(fmt.Errorf("GetRemoteStatus: error putting new status: %s", err))
}
return status, statusable, nil
return latest, apubStatus, nil
}
func (d *deref) dereferenceStatusable(ctx context.Context, tsport transport.Transport, remoteStatusID *url.URL) (ap.Statusable, error) {
if blocked, err := d.db.IsDomainBlocked(ctx, remoteStatusID.Host); blocked || err != nil {
return nil, fmt.Errorf("DereferenceStatusable: domain %s is blocked", remoteStatusID.Host)
// RefreshStatus: implements Dereferencer{}.RefreshStatus().
func (d *deref) RefreshStatus(ctx context.Context, requestUser string, status *gtsmodel.Status, apubStatus ap.Statusable, force bool) (*gtsmodel.Status, ap.Statusable, error) {
// Check whether needs update.
if statusUpToDate(status) {
return status, nil, nil
}
b, err := tsport.Dereference(ctx, remoteStatusID)
// Parse the URI from status.
uri, err := url.Parse(status.URI)
if err != nil {
return nil, fmt.Errorf("dereferenceStatusable: error deferencing %s: %w", remoteStatusID.String(), err)
return nil, nil, fmt.Errorf("RefreshStatus: invalid status uri %q: %w", status.URI, err)
}
return ap.ResolveStatusable(ctx, b)
// Try to update + deref existing status model.
latest, apubStatus, err := d.enrichStatus(ctx,
requestUser,
uri,
status,
apubStatus,
)
if err != nil {
return nil, nil, err
}
// This status was updated, enqueue re-dereferencing the whole thread.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
d.dereferenceThread(ctx, requestUser, uri, latest, apubStatus)
})
return latest, apubStatus, nil
}
// populateStatusFields fetches all the information we temporarily pinned to an incoming
// federated status, back in the federating db's Create function.
//
// When a status comes in from the federation API, there are certain fields that
// haven't been dereferenced yet, because we needed to provide a snappy synchronous
// response to the caller. By the time it reaches this function though, it's being
// processed asynchronously, so we have all the time in the world to fetch the various
// bits and bobs that are attached to the status, and properly flesh it out, before we
// send the status to any timelines and notify people.
//
// Things to dereference and fetch here:
//
// 1. Media attachments.
// 2. Hashtags.
// 3. Emojis.
// 4. Mentions.
// 5. Replied-to-status.
//
// SIDE EFFECTS:
// This function will deference all of the above, insert them in the database as necessary,
// and attach them to the status. The status itself will not be added to the database yet,
// that's up the caller to do.
func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Status, requestingUsername string, includeParent bool) error {
statusIRI, err := url.Parse(status.URI)
if err != nil {
return fmt.Errorf("populateStatusFields: couldn't parse status URI %s: %s", status.URI, err)
// RefreshStatusAsync: implements Dereferencer{}.RefreshStatusAsync().
func (d *deref) RefreshStatusAsync(ctx context.Context, requestUser string, status *gtsmodel.Status, apubStatus ap.Statusable, force bool) {
// Check whether needs update.
if statusUpToDate(status) {
return
}
blocked, err := d.db.IsURIBlocked(ctx, statusIRI)
// Parse the URI from status.
uri, err := url.Parse(status.URI)
if err != nil {
return fmt.Errorf("populateStatusFields: error checking blocked status of %s: %s", statusIRI, err)
}
if blocked {
return fmt.Errorf("populateStatusFields: domain %s is blocked", statusIRI)
log.Errorf(ctx, "RefreshStatusAsync: invalid status uri %q: %v", status.URI, err)
return
}
// in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one
if status.ID == "" {
newID, err := id.NewULIDFromTime(status.CreatedAt)
// Enqueue a worker function to re-fetch this status async.
d.state.Workers.Federator.MustEnqueueCtx(ctx, func(ctx context.Context) {
latest, apubStatus, err := d.enrichStatus(ctx, requestUser, uri, status, apubStatus)
if err != nil {
return fmt.Errorf("populateStatusFields: error creating ulid for status: %s", err)
log.Errorf(ctx, "error enriching remote status: %v", err)
return
}
status.ID = newID
// This status was updated, re-dereference the whole thread.
d.dereferenceThread(ctx, requestUser, uri, latest, apubStatus)
})
}
// enrichStatus will enrich the given status, whether a new barebones model, or existing model from the database. It handles necessary dereferencing etc.
func (d *deref) enrichStatus(ctx context.Context, requestUser string, uri *url.URL, status *gtsmodel.Status, apubStatus ap.Statusable) (*gtsmodel.Status, ap.Statusable, error) {
// Pre-fetch a transport for requesting username, used by later dereferencing.
tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: couldn't create transport: %w", err)
}
// 1. Media attachments.
if err := d.populateStatusAttachments(ctx, status, requestingUsername); err != nil {
return fmt.Errorf("populateStatusFields: error populating status attachments: %s", err)
// Check whether this account URI is a blocked domain / subdomain.
if blocked, err := d.state.DB.IsDomainBlocked(ctx, uri.Host); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error checking blocked domain: %w", err)
} else if blocked {
return nil, nil, fmt.Errorf("enrichStatus: %s is blocked", uri.Host)
}
// 2. Hashtags
// TODO
var derefd bool
// 3. Emojis
if err := d.populateStatusEmojis(ctx, status, requestingUsername); err != nil {
return fmt.Errorf("populateStatusFields: error populating status emojis: %s", err)
if apubStatus == nil {
// Dereference latest version of the status.
b, err := tsport.Dereference(ctx, uri)
if err != nil {
return nil, nil, &ErrNotRetrievable{fmt.Errorf("enrichStatus: error deferencing %s: %w", uri, err)}
}
// Attempt to resolve ActivityPub status from data.
apubStatus, err = ap.ResolveStatusable(ctx, b)
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error resolving statusable from data for account %s: %w", uri, err)
}
// Mark as deref'd.
derefd = true
}
// 4. Mentions
// TODO: do we need to handle removing empty mention objects and just using mention IDs slice?
if err := d.populateStatusMentions(ctx, status, requestingUsername); err != nil {
return fmt.Errorf("populateStatusFields: error populating status mentions: %s", err)
// Get the attributed-to status in order to fetch profile.
attributedTo, err := ap.ExtractAttributedTo(apubStatus)
if err != nil {
return nil, nil, errors.New("enrichStatus: attributedTo was empty")
}
// 5. Replied-to-status (only if requested)
if includeParent {
if err := d.populateStatusRepliedTo(ctx, status, requestingUsername); err != nil {
return fmt.Errorf("populateStatusFields: error populating status repliedTo: %s", err)
// Ensure we have the author account of the status dereferenced (+ up-to-date).
if author, _, err := d.getAccountByURI(ctx, requestUser, attributedTo); err != nil {
if status.AccountID == "" {
// Provided status account is nil, i.e. this is a new status / author, so a deref fail is unrecoverable.
return nil, nil, fmt.Errorf("enrichStatus: failed to dereference status author %s: %w", uri, err)
}
} else if status.AccountID != "" && status.AccountID != author.ID {
// There already existed an account for this status author, but account ID changed. This shouldn't happen!
log.Warnf(ctx, "status author account ID changed: old=%s new=%s", status.AccountID, author.ID)
}
// By default we assume that apubStatus has been passed,
// indicating that the given status is already latest.
latestStatus := status
if derefd {
// ActivityPub model was recently dereferenced, so assume that passed status
// may contain out-of-date information, convert AP model to our GTS model.
latestStatus, err = d.typeConverter.ASStatusToStatus(ctx, apubStatus)
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error converting statusable to gts model for status %s: %w", uri, err)
}
}
// Use existing status ID.
latestStatus.ID = status.ID
if latestStatus.ID == "" {
// Generate new status ID from the provided creation date.
latestStatus.ID, err = id.NewULIDFromTime(latestStatus.CreatedAt)
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: invalid created at date: %w", err)
}
}
// Carry-over values and set fetch time.
latestStatus.FetchedAt = time.Now()
latestStatus.Local = status.Local
// Ensure the status' mentions are populated, and pass in existing to check for changes.
if err := d.fetchStatusMentions(ctx, requestUser, status, latestStatus); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error populating mentions for status %s: %w", uri, err)
}
// TODO: populateStatusTags()
// Ensure the status' media attachments are populated, passing in existing to check for changes.
if err := d.fetchStatusAttachments(ctx, tsport, status, latestStatus); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error populating attachments for status %s: %w", uri, err)
}
// Ensure the status' emoji attachments are populated, passing in existing to check for changes.
if err := d.fetchStatusEmojis(ctx, requestUser, status, latestStatus); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error populating emojis for status %s: %w", uri, err)
}
if status.CreatedAt.IsZero() {
// CreatedAt will be zero if no local copy was
// found in one of the GetStatusBy___() functions.
//
// This is new, put the status in the database.
err := d.state.DB.PutStatus(ctx, latestStatus)
if errors.Is(err, db.ErrAlreadyExists) {
// TODO: replace this quick fix with per-URI deref locks.
latestStatus, err = d.state.DB.GetStatusByURI(ctx, latestStatus.URI)
return latestStatus, nil, err
}
if err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error putting in database: %w", err)
}
} else {
// This is an existing status, update the model in the database.
if err := d.state.DB.UpdateStatus(ctx, latestStatus); err != nil {
return nil, nil, fmt.Errorf("enrichStatus: error updating database: %w", err)
}
}
return latestStatus, apubStatus, nil
}
func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status) error {
// Allocate new slice to take the yet-to-be created mention IDs.
status.MentionIDs = make([]string, len(status.Mentions))
for i := range status.Mentions {
mention := status.Mentions[i]
// Look for existing mention with target account URI first.
existing, ok := existing.GetMentionByTargetURI(mention.TargetAccountURI)
if ok && existing.ID != "" {
status.Mentions[i] = existing
status.MentionIDs[i] = existing.ID
continue
}
// Ensure that mention account URI is parseable.
accountURI, err := url.Parse(mention.TargetAccountURI)
if err != nil {
log.Errorf(ctx, "invalid account uri %q: %v", mention.TargetAccountURI, err)
continue
}
// Ensure we have the account of the mention target dereferenced.
mention.TargetAccount, _, err = d.getAccountByURI(ctx, requestUser, accountURI)
if err != nil {
log.Errorf(ctx, "failed to dereference account %s: %v", accountURI, err)
continue
}
// Generate new ID according to status creation.
mention.ID, err = id.NewULIDFromTime(status.CreatedAt)
if err != nil {
log.Errorf(ctx, "invalid created at date: %v", err)
mention.ID = id.NewULID() // just use "now"
}
// Set known further mention details.
mention.CreatedAt = status.CreatedAt
mention.UpdatedAt = status.UpdatedAt
mention.OriginAccount = status.Account
mention.OriginAccountID = status.AccountID
mention.OriginAccountURI = status.AccountURI
mention.TargetAccountID = mention.TargetAccount.ID
mention.TargetAccountURI = mention.TargetAccount.URI
mention.TargetAccountURL = mention.TargetAccount.URL
mention.StatusID = status.ID
mention.Status = status
// Place the new mention into the database.
if err := d.state.DB.PutMention(ctx, mention); err != nil {
return fmt.Errorf("error putting mention in database: %w", err)
}
// Set the *new* mention and ID.
status.Mentions[i] = mention
status.MentionIDs[i] = mention.ID
}
for i := 0; i < len(status.MentionIDs); i++ {
if status.MentionIDs[i] == "" {
// This is a failed mention population, likely due
// to invalid incoming data / now-deleted accounts.
copy(status.Mentions[i:], status.Mentions[i+1:])
copy(status.MentionIDs[i:], status.MentionIDs[i+1:])
status.Mentions = status.Mentions[:len(status.Mentions)-1]
status.MentionIDs = status.MentionIDs[:len(status.MentionIDs)-1]
}
}
return nil
}
func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
// At this point, mentions should have the namestring and mentionedAccountURI set on them.
// We can use these to find the accounts.
func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Transport, existing *gtsmodel.Status, status *gtsmodel.Status) error {
// Allocate new slice to take the yet-to-be fetched attachment IDs.
status.AttachmentIDs = make([]string, len(status.Attachments))
mentionIDs := []string{}
newMentions := []*gtsmodel.Mention{}
for _, m := range status.Mentions {
if m.ID != "" {
// we've already populated this mention, since it has an ID
log.Debug(ctx, "mention already populated")
mentionIDs = append(mentionIDs, m.ID)
newMentions = append(newMentions, m)
for i := range status.Attachments {
placeholder := status.Attachments[i]
// Look for existing media attachment with remoet URL first.
existing, ok := existing.GetAttachmentByRemoteURL(placeholder.RemoteURL)
if ok && existing.ID != "" {
status.Attachments[i] = existing
status.AttachmentIDs[i] = existing.ID
continue
}
if m.TargetAccountURI == "" {
log.Debug(ctx, "target URI not set on mention")
continue
}
targetAccountURI, err := url.Parse(m.TargetAccountURI)
// Ensure a valid media attachment remote URL.
remoteURL, err := url.Parse(placeholder.RemoteURL)
if err != nil {
log.Debugf(ctx, "error parsing mentioned account uri %s: %s", m.TargetAccountURI, err)
log.Errorf(ctx, "invalid remote media url %q: %v", placeholder.RemoteURL, err)
continue
}
var targetAccount *gtsmodel.Account
errs := []string{}
// check if account is in the db already
if a, err := d.db.GetAccountByURI(ctx, targetAccountURI.String()); err != nil {
errs = append(errs, err.Error())
} else {
log.Debugf(ctx, "got target account %s with id %s through GetAccountByURI", targetAccountURI, a.ID)
targetAccount = a
}
if targetAccount == nil {
// we didn't find the account in our database already
// check if we can get the account remotely (dereference it)
if a, err := d.GetAccountByURI(ctx, requestingUsername, targetAccountURI); err != nil {
errs = append(errs, err.Error())
} else {
log.Debugf(ctx, "got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID)
targetAccount = a
}
}
if targetAccount == nil {
log.Debugf(ctx, "couldn't get target account %s: %s", m.TargetAccountURI, strings.Join(errs, " : "))
continue
}
mID, err := id.NewRandomULID()
if err != nil {
return fmt.Errorf("populateStatusMentions: error generating ulid: %s", err)
}
newMention := &gtsmodel.Mention{
ID: mID,
StatusID: status.ID,
Status: m.Status,
CreatedAt: status.CreatedAt,
UpdatedAt: status.UpdatedAt,
OriginAccountID: status.AccountID,
OriginAccountURI: status.AccountURI,
OriginAccount: status.Account,
TargetAccountID: targetAccount.ID,
TargetAccount: targetAccount,
NameString: m.NameString,
TargetAccountURI: targetAccount.URI,
TargetAccountURL: targetAccount.URL,
}
if err := d.db.PutMention(ctx, newMention); err != nil {
return fmt.Errorf("populateStatusMentions: error creating mention: %s", err)
}
mentionIDs = append(mentionIDs, newMention.ID)
newMentions = append(newMentions, newMention)
}
status.MentionIDs = mentionIDs
status.Mentions = newMentions
return nil
}
func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
// At this point we should know:
// * the media type of the file we're looking for (a.File.ContentType)
// * the file type (a.Type)
// * the remote URL (a.RemoteURL)
// This should be enough to dereference the piece of media.
attachmentIDs := []string{}
attachments := []*gtsmodel.MediaAttachment{}
for _, a := range status.Attachments {
a.AccountID = status.AccountID
a.StatusID = status.ID
processingMedia, err := d.GetRemoteMedia(ctx, requestingUsername, a.AccountID, a.RemoteURL, &media.AdditionalMediaInfo{
CreatedAt: &a.CreatedAt,
StatusID: &a.StatusID,
RemoteURL: &a.RemoteURL,
Description: &a.Description,
Blurhash: &a.Blurhash,
// Start pre-processing remote media at remote URL.
processing, err := d.mediaManager.PreProcessMedia(ctx, func(ctx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(ctx, remoteURL)
}, nil, status.AccountID, &media.AdditionalMediaInfo{
StatusID: &status.ID,
RemoteURL: &placeholder.RemoteURL,
Description: &placeholder.Description,
Blurhash: &placeholder.Blurhash,
})
if err != nil {
log.Errorf(ctx, "couldn't get remote media %s: %s", a.RemoteURL, err)
log.Errorf(ctx, "error processing attachment: %v", err)
continue
}
attachment, err := processingMedia.LoadAttachment(ctx)
// Force attachment loading *right now*.
media, err := processing.LoadAttachment(ctx)
if err != nil {
log.Errorf(ctx, "couldn't load remote attachment %s: %s", a.RemoteURL, err)
log.Errorf(ctx, "error loading attachment: %v", err)
continue
}
attachmentIDs = append(attachmentIDs, attachment.ID)
attachments = append(attachments, attachment)
// Set the *new* attachment and ID.
status.Attachments[i] = media
status.AttachmentIDs[i] = media.ID
}
status.AttachmentIDs = attachmentIDs
status.Attachments = attachments
for i := 0; i < len(status.AttachmentIDs); i++ {
if status.AttachmentIDs[i] == "" {
// This is a failed attachment population, this may
// be due to us not currently supporting a media type.
copy(status.Attachments[i:], status.Attachments[i+1:])
copy(status.AttachmentIDs[i:], status.AttachmentIDs[i+1:])
status.Attachments = status.Attachments[:len(status.Attachments)-1]
status.AttachmentIDs = status.AttachmentIDs[:len(status.AttachmentIDs)-1]
}
}
return nil
}
func (d *deref) populateStatusEmojis(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
emojis, err := d.populateEmojis(ctx, status.Emojis, requestingUsername)
func (d *deref) fetchStatusEmojis(ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status) error {
// Fetch the full-fleshed-out emoji objects for our status.
emojis, err := d.populateEmojis(ctx, status.Emojis, requestUser)
if err != nil {
return err
return fmt.Errorf("failed to populate emojis: %w", err)
}
// Iterate over and get their IDs.
emojiIDs := make([]string, 0, len(emojis))
for _, e := range emojis {
emojiIDs = append(emojiIDs, e.ID)
}
// Set known emoji details.
status.Emojis = emojis
status.EmojiIDs = emojiIDs
return nil
}
func (d *deref) populateStatusRepliedTo(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error {
if status.InReplyToURI != "" && status.InReplyToID == "" {
statusURI, err := url.Parse(status.InReplyToURI)
if err != nil {
return err
}
replyToStatus, _, err := d.GetStatus(ctx, requestingUsername, statusURI, false, false)
if err != nil {
return fmt.Errorf("populateStatusRepliedTo: couldn't get reply to status with uri %s: %s", status.InReplyToURI, err)
}
// we have the status
status.InReplyToID = replyToStatus.ID
status.InReplyTo = replyToStatus
status.InReplyToAccountID = replyToStatus.AccountID
status.InReplyToAccount = replyToStatus.Account
}
return nil
}

6
internal/federation/dereferencing/status_test.go
View file

@ -36,7 +36,7 @@ func (suite *StatusTestSuite) TestDereferenceSimpleStatus() {
fetchingAccount := suite.testAccounts["local_account_1"]
statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839")
status, _, err := suite.dereferencer.GetStatus(context.Background(), fetchingAccount.Username, statusURL, false, false)
status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL)
suite.NoError(err)
suite.NotNil(status)
@ -76,7 +76,7 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
fetchingAccount := suite.testAccounts["local_account_1"]
statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE5Y30E3W4P7TRE0R98KAYQV")
status, _, err := suite.dereferencer.GetStatus(context.Background(), fetchingAccount.Username, statusURL, false, false)
status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL)
suite.NoError(err)
suite.NotNil(status)
@ -127,7 +127,7 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithImageAndNoContent() {
fetchingAccount := suite.testAccounts["local_account_1"]
statusURL := testrig.URLMustParse("https://turnip.farm/users/turniplover6969/statuses/70c53e54-3146-42d5-a630-83c8b6c7c042")
status, _, err := suite.dereferencer.GetStatus(context.Background(), fetchingAccount.Username, statusURL, false, false)
status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL)
suite.NoError(err)
suite.NotNil(status)

49
internal/federation/dereferencing/thread.go
View file

@ -35,34 +35,16 @@ import (
// ancesters we are willing to follow before returning error.
const maxIter = 1000
// DereferenceThread takes a statusable (something that has withReplies and withInReplyTo),
// and dereferences statusables in the conversation.
//
// This process involves working up and down the chain of replies, and parsing through the collections of IDs
// presented by remote instances as part of their replies collections, and will likely involve making several calls to
// multiple different hosts.
//
// This does not return error, as for robustness we do not want to error-out on a status because another further up / down has issues.
func (d *deref) DereferenceThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) {
l := log.WithContext(ctx).
WithFields(kv.Fields{
{"username", username},
{"statusIRI", status.URI},
}...)
// Log function start
l.Trace("beginning")
// dereferenceThread will dereference statuses both above and below the given status in a thread, it returns no error and is intended to be called asychronously.
func (d *deref) dereferenceThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) {
// Ensure that ancestors have been fully dereferenced
if err := d.dereferenceStatusAncestors(ctx, username, status); err != nil {
l.Errorf("error dereferencing status ancestors: %v", err)
// we don't return error, we have deref'd as much as we can
log.Errorf(ctx, "error dereferencing status ancestors: %v", err)
}
// Ensure that descendants have been fully dereferenced
if err := d.dereferenceStatusDescendants(ctx, username, statusIRI, statusable); err != nil {
l.Errorf("error dereferencing status descendants: %v", err)
// we don't return error, we have deref'd as much as we can
log.Errorf(ctx, "error dereferencing status descendants: %v", err)
}
}
@ -103,7 +85,7 @@ func (d *deref) dereferenceStatusAncestors(ctx context.Context, username string,
}
// Fetch this status from the database
localStatus, err := d.db.GetStatusByID(ctx, id)
localStatus, err := d.state.DB.GetStatusByID(ctx, id)
if err != nil {
return fmt.Errorf("error fetching local status %q: %w", id, err)
}
@ -115,7 +97,10 @@ func (d *deref) dereferenceStatusAncestors(ctx context.Context, username string,
l.Tracef("following remote status ancestors: %s", status.InReplyToURI)
// Fetch the remote status found at this IRI
remoteStatus, _, err := d.GetStatus(ctx, username, replyIRI, false, false)
remoteStatus, _, err := d.getStatusByURI(ctx,
username,
replyIRI,
)
if err != nil {
return fmt.Errorf("error fetching remote status %q: %w", status.InReplyToURI, err)
}
@ -277,10 +262,15 @@ stackLoop:
continue itemLoop
}
// Dereference the remote status and store in the database
_, statusable, err := d.GetStatus(ctx, username, itemIRI, true, false)
// Dereference the remote status and store in the database.
_, statusable, err := d.getStatusByURI(ctx, username, itemIRI)
if err != nil {
l.Errorf("error dereferencing remote status %q: %s", itemIRI.String(), err)
l.Errorf("error dereferencing remote status %s: %v", itemIRI, err)
continue itemLoop
}
if statusable == nil {
// Already up-to-date.
continue itemLoop
}
@ -307,7 +297,10 @@ stackLoop:
}
// Dereference this next collection page by its IRI
collectionPage, err := d.DereferenceCollectionPage(ctx, username, pageNextIRI)
collectionPage, err := d.dereferenceCollectionPage(ctx,
username,
pageNextIRI,
)
if err != nil {
l.Errorf("error dereferencing remote collection page %q: %s", pageNextIRI.String(), err)
continue stackLoop