mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-22 06:57:29 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[chore] reformat GetAccount() functionality, support updating accounts based on last_fetch (#1411)

Browse source 
* reformat GetAccount() functionality, and add UpdateAccount() function.

* use fetched_at instead of last_webfingered_at

* catch local "not found" errors. small formatting / error string changes

* remove now unused error type

* return nil when wrapping nil error

* update expected error messages

* return correct url for foss satan webfinger

* add AP model for Some_User

* normalize local domain

* return notretrievable where appropriate

* expose NewErrNotRetrievable

* ensure webfinger for new accounts searched by uri

* update local account short circuit

* allow enrich to fail for already-known accounts

* remove unused LastWebfingeredAt

* expose test maps on mock http client

* update Update test

* reformat GetAccount() functionality, and add UpdateAccount() function.

* use fetched_at instead of last_webfingered_at

* catch local "not found" errors. small formatting / error string changes

* remove nil error checks (we shouldn't be passing nil errors to newError() initializers)

* remove mutex unlock on transport init fail (it hasn't yet been locked!)

* woops add back the error wrapping to use ErrNotRetrievable

* caches were never being started... 🙈

---------

Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
This commit is contained in:
kim 2023-02-03 20:03:05 +00:00 • committed by GitHub
commit 33aee1b1e9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 657 additions and 1159 deletions
Download patch file Download diff file Expand all files Collapse all files

52
internal/federation/dereference.go
View file

@ -1,52 +0,0 @@
/*
GoToSocial
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package federation
import (
"context"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (f *federator) GetAccount(ctx context.Context, params dereferencing.GetAccountParams) (*gtsmodel.Account, error) {
return f.dereferencer.GetAccount(ctx, params)
}
func (f *federator) GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) {
return f.dereferencer.GetStatus(ctx, username, remoteStatusID, refetch, includeParent)
}
func (f *federator) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) {
return f.dereferencer.EnrichRemoteStatus(ctx, username, status, includeParent)
}
func (f *federator) DereferenceRemoteThread(ctx context.Context, username string, statusIRI *url.URL, status *gtsmodel.Status, statusable ap.Statusable) {
f.dereferencer.DereferenceThread(ctx, username, statusIRI, status, statusable)
}
func (f *federator) GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
return f.dereferencer.GetRemoteInstance(ctx, username, remoteInstanceURI)
}
func (f *federator) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error {
return f.dereferencer.DereferenceAnnounce(ctx, announce, requestingUsername)
}

687
internal/federation/dereferencing/account.go
View file

@ -25,10 +25,8 @@ import (
"fmt"
"io"
"net/url"
"strings"
"time"
"github.com/miekg/dns"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/ap"
@ -38,375 +36,222 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
var webfingerInterval = -48 * time.Hour // 2 days in the past
func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) {
var (
account *gtsmodel.Account
uriStr = uri.String()
err error
)
func instanceAccount(account *gtsmodel.Account) bool {
return strings.EqualFold(account.Username, account.Domain) ||
account.FollowersURI == "" ||
account.FollowingURI == "" ||
(account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor"))
}
// GetAccountParams wraps parameters for an account lookup.
type GetAccountParams struct {
// The username of the user doing the lookup request (optional).
// If not set, then the GtS instance account will be used to do the lookup.
RequestingUsername string
// The ActivityPub URI of the account (optional).
// If not set (nil), the ActivityPub URI of the account will be discovered
// via webfinger, so you must set RemoteAccountUsername and RemoteAccountHost
// if this parameter is not set.
RemoteAccountID *url.URL
// The username of the account (optional).
// If RemoteAccountID is not set, then this value must be set.
RemoteAccountUsername string
// The host of the account (optional).
// If RemoteAccountID is not set, then this value must be set.
RemoteAccountHost string
// Whether to do a blocking call to the remote instance. If true,
// then the account's media and other fields will be fully dereferenced before it is returned.
// If false, then the account's media and other fields will be dereferenced in the background,
// so only a minimal account representation will be returned by GetRemoteAccount.
Blocking bool
// Whether to skip making calls to remote instances. This is useful when you want to
// quickly fetch a remote account from the database or fail, and don't want to cause
// http requests to go flying around.
SkipResolve bool
// PartialAccount can be used if the GetRemoteAccount call results from a federated/ap
// account update. In this case, we will already have a partial representation of the account,
// derived from converting the AP representation to a gtsmodel representation. If this field
// is provided, then GetRemoteAccount will use this as a basis for building the full account.
PartialAccount *gtsmodel.Account
}
type lookupType int
const (
lookupPartialLocal lookupType = iota
lookupPartial
lookupURILocal
lookupURI
lookupMentionLocal
lookupMention
lookupBad
)
func getLookupType(params GetAccountParams) lookupType {
switch {
case params.PartialAccount != nil:
if params.PartialAccount.Domain == "" || params.PartialAccount.Domain == config.GetHost() || params.PartialAccount.Domain == config.GetAccountDomain() {
return lookupPartialLocal
}
return lookupPartial
case params.RemoteAccountID != nil:
if host := params.RemoteAccountID.Host; host == config.GetHost() || host == config.GetAccountDomain() {
return lookupURILocal
}
return lookupURI
case params.RemoteAccountUsername != "":
if params.RemoteAccountHost == "" || params.RemoteAccountHost == config.GetHost() || params.RemoteAccountHost == config.GetAccountDomain() {
return lookupMentionLocal
}
return lookupMention
default:
return lookupBad
}
}
// GetAccount completely dereferences an account, converts it to a GtS model account,
// puts or updates it in the database (if necessary), and returns it to a caller.
//
// GetAccount will guard against trying to do http calls to fetch an account that belongs to this instance.
// Instead of making calls, it will just return the account early if it finds it, or return an error.
//
// Even if a fastfail context is used, and something goes wrong, an account might still be returned instead
// of an error, if we already had the account in our database (in other words, if we just needed to try
// fingering/refreshing the account again). The rationale for this is that it's more useful to be able
// to provide *something* to the caller, even if that something is not necessarily 100% up to date.
func (d *deref) GetAccount(ctx context.Context, params GetAccountParams) (foundAccount *gtsmodel.Account, err error) {
/*
In this function we want to retrieve a gtsmodel representation of a remote account, with its proper
accountDomain set, while making as few calls to remote instances as possible to save time and bandwidth.
There are a few different paths through this function, and the path taken depends on how much
initial information we are provided with via parameters, how much information we already have stored,
and what we're allowed to do according to the parameters we've been passed.
Scenario 1: We're not allowed to resolve remotely, but we've got either the account URI or the
account username + host, so we can check in our database and return if possible.
Scenario 2: We are allowed to resolve remotely, and we have an account URI but no username or host.
In this case, we can use the URI to resolve the remote account and find the username,
and then we can webfinger the account to discover the accountDomain if necessary.
Scenario 3: We are allowed to resolve remotely, and we have the username and host but no URI.
In this case, we can webfinger the account to discover the URI, and then dereference
from that.
*/
// this first step checks if we have the
// account in the database somewhere already,
// or if we've been provided it as a partial
switch getLookupType(params) {
case lookupPartialLocal:
params.SkipResolve = true
fallthrough
case lookupPartial:
foundAccount = params.PartialAccount
case lookupURILocal:
params.SkipResolve = true
fallthrough
case lookupURI:
// see if we have this in the db already with this uri/url
uri := params.RemoteAccountID.String()
if a, dbErr := d.db.GetAccountByURI(ctx, uri); dbErr == nil {
// got it, break here to leave early
foundAccount = a
break
} else if !errors.Is(dbErr, db.ErrNoEntries) {
// a real error
err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account with uri %s: %w", uri, dbErr))
break
}
// dbErr was just db.ErrNoEntries so search by url instead
if a, dbErr := d.db.GetAccountByURL(ctx, uri); dbErr == nil {
// got it
foundAccount = a
break
} else if !errors.Is(dbErr, db.ErrNoEntries) {
// a real error
err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account with url %s: %w", uri, dbErr))
break
}
case lookupMentionLocal:
params.SkipResolve = true
params.RemoteAccountHost = ""
fallthrough
case lookupMention:
// see if we have this in the db already with this username/host
if a, dbErr := d.db.GetAccountByUsernameDomain(ctx, params.RemoteAccountUsername, params.RemoteAccountHost); dbErr == nil {
foundAccount = a
} else if !errors.Is(dbErr, db.ErrNoEntries) {
// a real error
err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account %s: %w", params.RemoteAccountUsername, dbErr))
}
default:
err = newErrBadRequest(errors.New("GetRemoteAccount: no identifying parameters were set so we cannot get account"))
// Search the database for existing account with ID URI.
account, err = d.db.GetAccountByURI(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err)
}
// bail if we've set a real error, and not just no entries in the db
if account == nil {
// Else, search the database for existing by ID URL.
account, err = d.db.GetAccountByURL(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err)
}
}
if account == nil {
// Ensure that this is isn't a search for a local account.
if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() {
return nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
}
// Create and pass-through a new bare-bones model for dereferencing.
return d.enrichAccount(ctx, requestUser, uri, &gtsmodel.Account{
ID: id.NewULID(),
Domain: uri.Host,
URI: uriStr,
}, false, true)
}
// Try to update existing account model
enriched, err := d.enrichAccount(ctx, requestUser, uri, account, false, block)
if err != nil {
return
log.Errorf("error enriching remote account: %v", err)
return account, nil // fall back to returning existing
}
if params.SkipResolve {
// if we can't resolve, return already since there's nothing more we can do
if foundAccount == nil {
err = newErrNotRetrievable(errors.New("GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it"))
return enriched, nil
}
func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error) {
if domain == config.GetHost() || domain == config.GetAccountDomain() {
// We do local lookups using an empty domain,
// else it will fail the db search below.
domain = ""
}
// Search the database for existing account with USERNAME@DOMAIN
account, err := d.db.GetAccountByUsernameDomain(ctx, username, domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err)
}
if account == nil {
// Check for failed local lookup.
if domain == "" {
return nil, NewErrNotRetrievable(err) // will be db.ErrNoEntries
}
return
// Create and pass-through a new bare-bones model for dereferencing.
return d.enrichAccount(ctx, requestUser, nil, &gtsmodel.Account{
ID: id.NewULID(),
Username: username,
Domain: domain,
}, false, true)
}
// if we reach this point, we have some remote calls to make
// Try to update existing account model
enriched, err := d.enrichAccount(ctx, requestUser, nil, account, false, block)
if err != nil {
log.Errorf("GetAccountByUsernameDomain: error enriching account from remote: %v", err)
return account, nil // fall back to returning unchanged existing account model
}
var accountable ap.Accountable
if params.RemoteAccountUsername == "" && params.RemoteAccountHost == "" {
// if we're still missing some params, try to populate them now
params.RemoteAccountHost = params.RemoteAccountID.Host
if foundAccount != nil {
// username is easy if we found something already
params.RemoteAccountUsername = foundAccount.Username
} else {
// if we didn't already have it, we have to dereference it from remote
var derefErr error
accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
if derefErr != nil {
err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable")
return
}
return enriched, nil
}
var apError error
params.RemoteAccountUsername, apError = ap.ExtractPreferredUsername(accountable)
if apError != nil {
err = newErrOther(fmt.Errorf("GetRemoteAccount: error extracting Accountable username: %w", apError))
return
}
func (d *deref) UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error) {
return d.enrichAccount(ctx, requestUser, nil, account, force, false)
}
// enrichAccount will ensure the given account is the most up-to-date model of the account, re-webfingering and re-dereferencing if necessary.
func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.URL, account *gtsmodel.Account, force, block bool) (*gtsmodel.Account, error) {
if account.IsLocal() {
// Can't update local accounts.
return account, nil
}
if !account.CreatedAt.IsZero() && account.IsInstance() {
// Existing instance account. No need for update.
return account, nil
}
if !force {
const interval = time.Hour * 48
// If this account was updated recently (last interval), we return as-is.
if next := account.FetchedAt.Add(interval); time.Now().Before(next) {
return account, nil
}
}
// if we reach this point, params.RemoteAccountHost and params.RemoteAccountUsername must be set
// params.RemoteAccountID may or may not be set, but we have enough information to fetch it if we need it
if account.Username != "" {
// A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info.
accDomain, accURI, err := d.fingerRemoteAccount(ctx, requestUser, account.Username, account.Domain)
// we finger to fetch the account domain but just in case we're not fingering, make a best guess
// already about what the account domain might be; this var will be overwritten later if necessary
var accountDomain string
switch {
case foundAccount != nil:
accountDomain = foundAccount.Domain
case params.RemoteAccountID != nil:
accountDomain = params.RemoteAccountID.Host
default:
accountDomain = params.RemoteAccountHost
}
// to save on remote calls, only webfinger if:
// - we don't know the remote account ActivityPub ID yet OR
// - we haven't found the account yet in some other way OR
// - we were passed a partial account in params OR
// - we haven't webfingered the account for two days AND the account isn't an instance account
var fingered time.Time
var refreshFinger bool
if foundAccount != nil {
refreshFinger = foundAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(foundAccount)
}
if params.RemoteAccountID == nil || foundAccount == nil || params.PartialAccount != nil || refreshFinger {
if ad, accountURI, fingerError := d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost); fingerError != nil {
if !refreshFinger {
// only return with an error if this wasn't just a refresh finger;
// that is, if we actually *needed* to finger in order to get the account,
// otherwise we can just continue and we'll try again in 2 days
err = newErrNotRetrievable(fmt.Errorf("GetRemoteAccount: error while fingering: %w", fingerError))
return
}
log.Infof("error doing non-vital webfinger refresh call to %s: %s", params.RemoteAccountHost, err)
} else {
accountDomain = ad
params.RemoteAccountID = accountURI
if err != nil && account.URI == "" {
// this is a new account (to us) with username@domain but failed
// webfinger, there is nothing more we can do in this situation.
return nil, fmt.Errorf("enrichAccount: error webfingering account: %w", err)
}
fingered = time.Now()
}
if !fingered.IsZero() && foundAccount == nil {
// if we just fingered and now have a discovered account domain but still no account,
// we should do a final lookup in the database with the discovered username + accountDomain
// to make absolutely sure we don't already have this account
if a, dbErr := d.db.GetAccountByUsernameDomain(ctx, params.RemoteAccountUsername, accountDomain); dbErr == nil {
foundAccount = a
} else if !errors.Is(dbErr, db.ErrNoEntries) {
// a real error
err = newErrDB(fmt.Errorf("GetRemoteAccount: unexpected error while looking for account %s: %w", params.RemoteAccountUsername, dbErr))
return
if err == nil {
// Update account with latest info.
account.URI = accURI.String()
account.Domain = accDomain
uri = accURI
}
}
// we may have some extra information already, like the account we had in the db, or the
// accountable representation that we dereferenced from remote
if foundAccount == nil {
// if we still don't have a remoteAccountID here we're boned
if params.RemoteAccountID == nil {
err = newErrNotRetrievable(errors.New("GetRemoteAccount: could not populate find an account nor populate params.RemoteAccountID"))
return
}
if uri == nil {
var err error
// deference accountable if we didn't earlier
if accountable == nil {
var derefErr error
accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
if derefErr != nil {
err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable")
return
}
}
// then convert
foundAccount, err = d.typeConverter.ASRepresentationToAccount(ctx, accountable, accountDomain, false)
// No URI provided / found, must parse from account.
uri, err = url.Parse(account.URI)
if err != nil {
err = newErrOther(fmt.Errorf("GetRemoteAccount: error converting Accountable to account: %w", err))
return
}
// this is a new account so we need to generate a new ID for it
var ulid string
ulid, err = id.NewRandomULID()
if err != nil {
err = newErrOther(fmt.Errorf("GetRemoteAccount: error generating new id for account: %w", err))
return
}
foundAccount.ID = ulid
if _, populateErr := d.populateAccountFields(ctx, foundAccount, params.RequestingUsername, params.Blocking); populateErr != nil {
// it's not the end of the world if we can't populate account fields, but we do want to log it
log.Errorf("GetRemoteAccount: error populating further account fields: %s", populateErr)
}
foundAccount.LastWebfingeredAt = fingered
foundAccount.UpdatedAt = time.Now()
if dbErr := d.db.PutAccount(ctx, foundAccount); dbErr != nil {
err = newErrDB(fmt.Errorf("GetRemoteAccount: error putting new account: %w", dbErr))
return
}
return // the new account
}
// we had the account already, but now we know the account domain, so update it if it's different
var accountDomainChanged bool
if !strings.EqualFold(foundAccount.Domain, accountDomain) {
accountDomainChanged = true
foundAccount.Domain = accountDomain
}
// if SharedInboxURI is nil, that means we don't know yet if this account has
// a shared inbox available for it, so we need to check this here
var sharedInboxChanged bool
if foundAccount.SharedInboxURI == nil {
// we need the accountable for this, so get it if we don't have it yet
if accountable == nil {
var derefErr error
accountable, derefErr = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
if derefErr != nil {
err = wrapDerefError(derefErr, "GetRemoteAccount: error dereferencing Accountable")
return
}
}
// This can be:
// - an empty string (we know it doesn't have a shared inbox) OR
// - a string URL (we know it does a shared inbox).
// Set it either way!
var sharedInbox string
if sharedInboxURI := ap.ExtractSharedInbox(accountable); sharedInboxURI != nil {
// only trust shared inbox if it has at least two domains,
// from the right, in common with the domain of the account
if dns.CompareDomainName(foundAccount.Domain, sharedInboxURI.Host) >= 2 {
sharedInbox = sharedInboxURI.String()
}
}
sharedInboxChanged = true
foundAccount.SharedInboxURI = &sharedInbox
}
// make sure the account fields are populated before returning:
// the caller might want to block until everything is loaded
fieldsChanged, populateErr := d.populateAccountFields(ctx, foundAccount, params.RequestingUsername, params.Blocking)
if populateErr != nil {
// it's not the end of the world if we can't populate account fields, but we do want to log it
log.Errorf("GetRemoteAccount: error populating further account fields: %s", populateErr)
}
var fingeredChanged bool
if !fingered.IsZero() {
fingeredChanged = true
foundAccount.LastWebfingeredAt = fingered
}
if accountDomainChanged || sharedInboxChanged || fieldsChanged || fingeredChanged {
if dbErr := d.db.UpdateAccount(ctx, foundAccount); dbErr != nil {
err = newErrDB(fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %w", dbErr))
return
return nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err)
}
}
return // the account we already had + possibly updated
// Check whether this account URI is a blocked domain / subdomain
if blocked, err := d.db.IsDomainBlocked(ctx, uri.Host); err != nil {
return nil, newErrDB(fmt.Errorf("enrichAccount: error checking blocked domain: %w", err))
} else if blocked {
return nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host)
}
// Mark deref+update handshake start
d.startHandshake(requestUser, uri)
defer d.stopHandshake(requestUser, uri)
// Dereference this account to get the latest available.
apubAcc, err := d.dereferenceAccountable(ctx, requestUser, uri)
if err != nil {
return nil, fmt.Errorf("enrichAccount: error dereferencing account %s: %w", uri, err)
}
// Convert the dereferenced AP account object to our GTS model.
latestAcc, err := d.typeConverter.ASRepresentationToAccount(
ctx, apubAcc, account.Domain,
)
if err != nil {
return nil, fmt.Errorf("enrichAccount: error converting accountable to gts model for account %s: %w", uri, err)
}
if account.Username == "" {
// No username was provided, so no webfinger was attempted earlier.
//
// Now we have a username we can attempt it now, this ensures up-to-date accountdomain info.
accDomain, _, err := d.fingerRemoteAccount(ctx, requestUser, latestAcc.Username, uri.Host)
if err == nil {
// Update account with latest info.
latestAcc.Domain = accDomain
}
}
// Ensure ID is set and update fetch time.
latestAcc.ID = account.ID
latestAcc.FetchedAt = time.Now()
// Fetch latest account media (TODO: check for changed URI to previous).
if err = d.fetchRemoteAccountMedia(ctx, latestAcc, requestUser, block); err != nil {
log.Errorf("error fetching remote media for account %s: %v", uri, err)
}
// Fetch the latest remote account emoji IDs used in account display name/bio.
_, err = d.fetchRemoteAccountEmojis(ctx, latestAcc, requestUser)
if err != nil {
log.Errorf("error fetching remote emojis for account %s: %v", uri, err)
}
if account.CreatedAt.IsZero() {
// CreatedAt will be zero if no local copy was
// found in one of the GetAccountBy___() functions.
//
// Set time of creation from the last-fetched date.
latestAcc.CreatedAt = latestAcc.FetchedAt
latestAcc.UpdatedAt = latestAcc.FetchedAt
// This is a new account, we need to place it in the database.
if err := d.db.PutAccount(ctx, latestAcc); err != nil {
return nil, fmt.Errorf("enrichAccount: error putting in database: %w", err)
}
} else {
// Set time of update from the last-fetched date.
latestAcc.UpdatedAt = latestAcc.FetchedAt
// Use existing account values.
latestAcc.CreatedAt = account.CreatedAt
latestAcc.Language = account.Language
// This is an existing account, update the model in the database.
if err := d.db.UpdateAccount(ctx, latestAcc); err != nil {
return nil, fmt.Errorf("enrichAccount: error updating database: %w", err)
}
}
return latestAcc, nil
}
// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
@ -414,13 +259,6 @@ func (d *deref) GetAccount(ctx context.Context, params GetAccountParams) (foundA
//
// Will work for Person, Application, or Service models.
func (d *deref) dereferenceAccountable(ctx context.Context, username string, remoteAccountID *url.URL) (ap.Accountable, error) {
d.startHandshake(username, remoteAccountID)
defer d.stopHandshake(username, remoteAccountID)
if blocked, err := d.db.IsDomainBlocked(ctx, remoteAccountID.Host); blocked || err != nil {
return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host)
}
transport, err := d.transportController.NewTransportForUsername(ctx, username)
if err != nil {
return nil, fmt.Errorf("DereferenceAccountable: transport err: %w", err)
@ -441,83 +279,23 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %w", err)
}
//nolint shutup linter
switch t.GetTypeName() {
case ap.ActorApplication:
p, ok := t.(vocab.ActivityStreamsApplication)
if !ok {
return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application")
}
return p, nil
return t.(vocab.ActivityStreamsApplication), nil
case ap.ActorGroup:
p, ok := t.(vocab.ActivityStreamsGroup)
if !ok {
return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams group")
}
return p, nil
return t.(vocab.ActivityStreamsGroup), nil
case ap.ActorOrganization:
p, ok := t.(vocab.ActivityStreamsOrganization)
if !ok {
return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams organization")
}
return p, nil
return t.(vocab.ActivityStreamsOrganization), nil
case ap.ActorPerson:
p, ok := t.(vocab.ActivityStreamsPerson)
if !ok {
return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person")
}
return p, nil
return t.(vocab.ActivityStreamsPerson), nil
case ap.ActorService:
p, ok := t.(vocab.ActivityStreamsService)
if !ok {
return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service")
}
return p, nil
return t.(vocab.ActivityStreamsService), nil
}
return nil, newErrWrongType(fmt.Errorf("DereferenceAccountable: type name %s not supported as Accountable", t.GetTypeName()))
}
// populateAccountFields makes a best effort to populate fields on an account such as emojis, avatar, header.
// Will return true if one of these things changed on the passed-in account.
func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) {
// if we're dealing with an instance account, just bail, we don't need to do anything
if instanceAccount(account) {
return false, nil
}
accountURI, err := url.Parse(account.URI)
if err != nil {
return false, fmt.Errorf("populateAccountFields: couldn't parse account URI %s: %w", account.URI, err)
}
blocked, dbErr := d.db.IsDomainBlocked(ctx, accountURI.Host)
if dbErr != nil {
return false, fmt.Errorf("populateAccountFields: eror checking for block of domain %s: %w", accountURI.Host, err)
}
if blocked {
return false, fmt.Errorf("populateAccountFields: domain %s is blocked", accountURI.Host)
}
var changed bool
// fetch the header and avatar
if mediaChanged, err := d.fetchRemoteAccountMedia(ctx, account, requestingUsername, blocking); err != nil {
return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %w", err)
} else if mediaChanged {
changed = mediaChanged
}
// fetch any emojis used in note, fields, display name, etc
if emojisChanged, err := d.fetchRemoteAccountEmojis(ctx, account, requestingUsername); err != nil {
return false, fmt.Errorf("populateAccountFields: error fetching emojis for account: %w", err)
} else if emojisChanged {
changed = emojisChanged
}
return changed, nil
}
// fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account,
// using a transport on behalf of requestingUsername.
//
@ -530,39 +308,30 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc
//
// If blocking is true, then the calls to the media manager made by this function will be blocking:
// in other words, the function won't return until the header and the avatar have been fully processed.
func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) {
var (
changed bool
t transport.Transport
)
func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) error {
// Fetch a transport beforehand for either(or both) avatar / header dereferencing.
tsport, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
if err != nil {
return fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
}
if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") {
if targetAccount.AvatarRemoteURL != "" {
var processingMedia *media.ProcessingMedia
// Parse the target account's avatar URL into URL object.
avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL)
if err != nil {
return err
}
d.dereferencingAvatarsLock.Lock() // LOCK HERE
// first check if we're already processing this media
if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok {
// we're already on it, no worries
processingMedia = alreadyProcessing
} else {
// we're not already processing it so start now
avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL)
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return changed, err
}
if t == nil {
var err error
t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername)
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
}
}
data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
return t.DereferenceMedia(innerCtx, avatarIRI)
return tsport.DereferenceMedia(innerCtx, avatarIRI)
}
avatar := true
@ -572,7 +341,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
})
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return changed, err
return err
}
// store it in our map to indicate it's in process
@ -595,7 +364,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
// block until loaded if required...
if blocking {
if err := loadAndCleanup(ctx, load, cleanup); err != nil {
return changed, err
return err
}
} else {
// ...otherwise do it async
@ -609,36 +378,25 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
}
targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID()
changed = true
}
if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "") {
if targetAccount.HeaderRemoteURL != "" {
var processingMedia *media.ProcessingMedia
// Parse the target account's header URL into URL object.
headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL)
if err != nil {
return err
}
d.dereferencingHeadersLock.Lock() // LOCK HERE
// first check if we're already processing this media
if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok {
// we're already on it, no worries
processingMedia = alreadyProcessing
} else {
// we're not already processing it so start now
headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL)
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return changed, err
}
if t == nil {
var err error
t, err = d.transportController.NewTransportForUsername(ctx, requestingUsername)
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return false, fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
}
}
data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
return t.DereferenceMedia(innerCtx, headerIRI)
return tsport.DereferenceMedia(innerCtx, headerIRI)
}
header := true
@ -648,7 +406,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
})
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return changed, err
return err
}
// store it in our map to indicate it's in process
@ -671,7 +429,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
// block until loaded if required...
if blocking {
if err := loadAndCleanup(ctx, load, cleanup); err != nil {
return changed, err
return err
}
} else {
// ...otherwise do it async
@ -685,10 +443,9 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
}
targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID()
changed = true
}
return changed, nil
return nil
}
func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) {

323
internal/federation/dereferencing/account_test.go
View file

@ -27,7 +27,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -39,17 +38,19 @@ func (suite *AccountTestSuite) TestDereferenceGroup() {
fetchingAccount := suite.testAccounts["local_account_1"]
groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group")
group, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: groupURL,
})
group, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
groupURL,
false,
)
suite.NoError(err)
suite.NotNil(group)
// group values should be set
suite.Equal("https://unknown-instance.com/groups/some_group", group.URI)
suite.Equal("https://unknown-instance.com/@some_group", group.URL)
suite.WithinDuration(time.Now(), group.LastWebfingeredAt, 5*time.Second)
suite.WithinDuration(time.Now(), group.FetchedAt, 5*time.Second)
// group should be in the database
dbGroup, err := suite.db.GetAccountByURI(context.Background(), group.URI)
@ -62,17 +63,19 @@ func (suite *AccountTestSuite) TestDereferenceService() {
fetchingAccount := suite.testAccounts["local_account_1"]
serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh")
service, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: serviceURL,
})
service, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
serviceURL,
false,
)
suite.NoError(err)
suite.NotNil(service)
// service values should be set
suite.Equal("https://owncast.example.org/federation/user/rgh", service.URI)
suite.Equal("https://owncast.example.org/federation/user/rgh", service.URL)
suite.WithinDuration(time.Now(), service.LastWebfingeredAt, 5*time.Second)
suite.WithinDuration(time.Now(), service.FetchedAt, 5*time.Second)
// service should be in the database
dbService, err := suite.db.GetAccountByURI(context.Background(), service.URI)
@ -93,10 +96,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURL() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: testrig.URLMustParse(targetAccount.URI),
})
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
false,
)
suite.NoError(err)
suite.NotNil(fetchedAccount)
suite.Empty(fetchedAccount.Domain)
@ -111,10 +116,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsRemoteURLNoSharedInb
suite.FailNow(err.Error())
}
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: testrig.URLMustParse(targetAccount.URI),
})
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
false,
)
suite.NoError(err)
suite.NotNil(fetchedAccount)
suite.Empty(fetchedAccount.Domain)
@ -124,10 +131,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsername() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountUsername: targetAccount.Username,
})
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
false,
)
suite.NoError(err)
suite.NotNil(fetchedAccount)
suite.Empty(fetchedAccount.Domain)
@ -137,11 +146,12 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomain() {
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountUsername: targetAccount.Username,
RemoteAccountHost: config.GetHost(),
})
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse(targetAccount.URI),
false,
)
suite.NoError(err)
suite.NotNil(fetchedAccount)
suite.Empty(fetchedAccount.Domain)
@ -151,12 +161,13 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
fetchingAccount := suite.testAccounts["local_account_1"]
targetAccount := suite.testAccounts["local_account_2"]
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: testrig.URLMustParse(targetAccount.URI),
RemoteAccountUsername: targetAccount.Username,
RemoteAccountHost: config.GetHost(),
})
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
targetAccount.Username,
config.GetHost(),
false,
)
suite.NoError(err)
suite.NotNil(fetchedAccount)
suite.Empty(fetchedAccount.Domain)
@ -165,248 +176,50 @@ func (suite *AccountTestSuite) TestDereferenceLocalAccountAsUsernameDomainAndURL
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsername() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountUsername: "thisaccountdoesnotexist",
})
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
"thisaccountdoesnotexist",
config.GetHost(),
false,
)
var errNotRetrievable *dereferencing.ErrNotRetrievable
suite.ErrorAs(err, &errNotRetrievable)
suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
suite.EqualError(err, "item could not be retrieved: no entries")
suite.Nil(fetchedAccount)
}
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUsernameDomain() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountUsername: "thisaccountdoesnotexist",
RemoteAccountHost: "localhost:8080",
})
fetchedAccount, err := suite.dereferencer.GetAccountByUsernameDomain(
context.Background(),
fetchingAccount.Username,
"thisaccountdoesnotexist",
"localhost:8080",
false,
)
var errNotRetrievable *dereferencing.ErrNotRetrievable
suite.ErrorAs(err, &errNotRetrievable)
suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
suite.EqualError(err, "item could not be retrieved: no entries")
suite.Nil(fetchedAccount)
}
func (suite *AccountTestSuite) TestDereferenceLocalAccountWithUnknownUserURI() {
fetchingAccount := suite.testAccounts["local_account_1"]
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"),
})
fetchedAccount, err := suite.dereferencer.GetAccountByURI(
context.Background(),
fetchingAccount.Username,
testrig.URLMustParse("http://localhost:8080/users/thisaccountdoesnotexist"),
false,
)
var errNotRetrievable *dereferencing.ErrNotRetrievable
suite.ErrorAs(err, &errNotRetrievable)
suite.EqualError(err, "item could not be retrieved: GetRemoteAccount: couldn't retrieve account locally and not allowed to resolve it")
suite.EqualError(err, "item could not be retrieved: no entries")
suite.Nil(fetchedAccount)
}
func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial() {
fetchingAccount := suite.testAccounts["local_account_1"]
remoteAccount := suite.testAccounts["remote_account_1"]
remoteAccountPartial := &gtsmodel.Account{
ID: remoteAccount.ID,
ActorType: remoteAccount.ActorType,
Language: remoteAccount.Language,
CreatedAt: remoteAccount.CreatedAt,
UpdatedAt: remoteAccount.UpdatedAt,
Username: remoteAccount.Username,
Domain: remoteAccount.Domain,
DisplayName: remoteAccount.DisplayName,
URI: remoteAccount.URI,
InboxURI: remoteAccount.URI,
SharedInboxURI: remoteAccount.SharedInboxURI,
PublicKeyURI: remoteAccount.PublicKeyURI,
URL: remoteAccount.URL,
FollowingURI: remoteAccount.FollowingURI,
FollowersURI: remoteAccount.FollowersURI,
OutboxURI: remoteAccount.OutboxURI,
FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
Emojis: []*gtsmodel.Emoji{
// dereference an emoji we don't have stored yet
{
URI: "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",
Shortcode: "kip_van_den_bos",
UpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
ImageUpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
ImageRemoteURL: "http://fossbros-anonymous.io/emoji/kip.gif",
Disabled: testrig.FalseBool(),
VisibleInPicker: testrig.FalseBool(),
Domain: "fossbros-anonymous.io",
},
},
}
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: testrig.URLMustParse(remoteAccount.URI),
RemoteAccountHost: remoteAccount.Domain,
RemoteAccountUsername: remoteAccount.Username,
PartialAccount: remoteAccountPartial,
Blocking: true,
})
suite.NoError(err)
suite.NotNil(fetchedAccount)
suite.NotNil(fetchedAccount.EmojiIDs)
suite.NotNil(fetchedAccount.Emojis)
}
func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial2() {
fetchingAccount := suite.testAccounts["local_account_1"]
knownEmoji := suite.testEmojis["yell"]
remoteAccount := suite.testAccounts["remote_account_1"]
remoteAccountPartial := &gtsmodel.Account{
ID: remoteAccount.ID,
ActorType: remoteAccount.ActorType,
Language: remoteAccount.Language,
CreatedAt: remoteAccount.CreatedAt,
UpdatedAt: remoteAccount.UpdatedAt,
Username: remoteAccount.Username,
Domain: remoteAccount.Domain,
DisplayName: remoteAccount.DisplayName,
URI: remoteAccount.URI,
InboxURI: remoteAccount.URI,
SharedInboxURI: remoteAccount.SharedInboxURI,
PublicKeyURI: remoteAccount.PublicKeyURI,
URL: remoteAccount.URL,
FollowingURI: remoteAccount.FollowingURI,
FollowersURI: remoteAccount.FollowersURI,
OutboxURI: remoteAccount.OutboxURI,
FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
Emojis: []*gtsmodel.Emoji{
// an emoji we already have
{
URI: knownEmoji.URI,
Shortcode: knownEmoji.Shortcode,
UpdatedAt: knownEmoji.UpdatedAt,
ImageUpdatedAt: knownEmoji.ImageUpdatedAt,
ImageRemoteURL: knownEmoji.ImageRemoteURL,
Disabled: knownEmoji.Disabled,
VisibleInPicker: knownEmoji.VisibleInPicker,
Domain: knownEmoji.Domain,
},
},
}
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: testrig.URLMustParse(remoteAccount.URI),
RemoteAccountHost: remoteAccount.Domain,
RemoteAccountUsername: remoteAccount.Username,
PartialAccount: remoteAccountPartial,
Blocking: true,
})
suite.NoError(err)
suite.NotNil(fetchedAccount)
suite.NotNil(fetchedAccount.EmojiIDs)
suite.NotNil(fetchedAccount.Emojis)
}
func (suite *AccountTestSuite) TestDereferenceRemoteAccountWithPartial3() {
fetchingAccount := suite.testAccounts["local_account_1"]
knownEmoji := suite.testEmojis["yell"]
remoteAccount := suite.testAccounts["remote_account_1"]
remoteAccountPartial := &gtsmodel.Account{
ID: remoteAccount.ID,
ActorType: remoteAccount.ActorType,
Language: remoteAccount.Language,
CreatedAt: remoteAccount.CreatedAt,
UpdatedAt: remoteAccount.UpdatedAt,
Username: remoteAccount.Username,
Domain: remoteAccount.Domain,
DisplayName: remoteAccount.DisplayName,
URI: remoteAccount.URI,
InboxURI: remoteAccount.URI,
SharedInboxURI: remoteAccount.SharedInboxURI,
PublicKeyURI: remoteAccount.PublicKeyURI,
URL: remoteAccount.URL,
FollowingURI: remoteAccount.FollowingURI,
FollowersURI: remoteAccount.FollowersURI,
OutboxURI: remoteAccount.OutboxURI,
FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
Emojis: []*gtsmodel.Emoji{
// an emoji we already have
{
URI: knownEmoji.URI,
Shortcode: knownEmoji.Shortcode,
UpdatedAt: knownEmoji.UpdatedAt,
ImageUpdatedAt: knownEmoji.ImageUpdatedAt,
ImageRemoteURL: knownEmoji.ImageRemoteURL,
Disabled: knownEmoji.Disabled,
VisibleInPicker: knownEmoji.VisibleInPicker,
Domain: knownEmoji.Domain,
},
},
}
fetchedAccount, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: testrig.URLMustParse(remoteAccount.URI),
RemoteAccountHost: remoteAccount.Domain,
RemoteAccountUsername: remoteAccount.Username,
PartialAccount: remoteAccountPartial,
Blocking: true,
})
suite.NoError(err)
suite.NotNil(fetchedAccount)
suite.NotNil(fetchedAccount.EmojiIDs)
suite.NotNil(fetchedAccount.Emojis)
suite.Equal(knownEmoji.URI, fetchedAccount.Emojis[0].URI)
remoteAccountPartial2 := &gtsmodel.Account{
ID: remoteAccount.ID,
ActorType: remoteAccount.ActorType,
Language: remoteAccount.Language,
CreatedAt: remoteAccount.CreatedAt,
UpdatedAt: remoteAccount.UpdatedAt,
Username: remoteAccount.Username,
Domain: remoteAccount.Domain,
DisplayName: remoteAccount.DisplayName,
URI: remoteAccount.URI,
InboxURI: remoteAccount.URI,
SharedInboxURI: remoteAccount.SharedInboxURI,
PublicKeyURI: remoteAccount.PublicKeyURI,
URL: remoteAccount.URL,
FollowingURI: remoteAccount.FollowingURI,
FollowersURI: remoteAccount.FollowersURI,
OutboxURI: remoteAccount.OutboxURI,
FeaturedCollectionURI: remoteAccount.FeaturedCollectionURI,
Emojis: []*gtsmodel.Emoji{
// dereference an emoji we don't have stored yet
{
URI: "http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1",
Shortcode: "kip_van_den_bos",
UpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
ImageUpdatedAt: testrig.TimeMustParse("2022-09-13T12:13:12+02:00"),
ImageRemoteURL: "http://fossbros-anonymous.io/emoji/kip.gif",
Disabled: testrig.FalseBool(),
VisibleInPicker: testrig.FalseBool(),
Domain: "fossbros-anonymous.io",
},
},
}
fetchedAccount2, err := suite.dereferencer.GetAccount(context.Background(), dereferencing.GetAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: testrig.URLMustParse(remoteAccount.URI),
RemoteAccountHost: remoteAccount.Domain,
RemoteAccountUsername: remoteAccount.Username,
PartialAccount: remoteAccountPartial2,
Blocking: true,
})
suite.NoError(err)
suite.NotNil(fetchedAccount2)
suite.NotNil(fetchedAccount2.EmojiIDs)
suite.NotNil(fetchedAccount2.Emojis)
suite.Equal("http://fossbros-anonymous.io/emoji/01GD5HCC2YECT012TK8PAGX4D1", fetchedAccount2.Emojis[0].URI)
}
func TestAccountTestSuite(t *testing.T) {
suite.Run(t, new(AccountTestSuite))
}

41
internal/federation/dereferencing/dereferencer.go
View file

@ -33,7 +33,17 @@ import (
// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
type Dereferencer interface {
GetAccount(ctx context.Context, params GetAccountParams) (*gtsmodel.Account, error)
// GetAccountByURI will attempt to fetch an account by its URI, first checking the database and in the case of a remote account will either check the
// last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before.
GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error)
// GetAccountByUsernameDomain will attempt to fetch an account by username@domain, first checking the database and in the case of a remote account will either
// check the last_fetched (and updating if beyond fetch interval) or dereferencing for the first-time if this remote account has never been encountered before.
GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error)
// UpdateAccount updates the given account if last_fetched is beyond fetch interval (or if force is set). An updated account model is returned, any media fetching is done async.
UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error)
GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)
@ -44,7 +54,7 @@ type Dereferencer interface {
GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error)
GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, domain string, id string, emojiURI string, ai *media.AdditionalEmojiInfo, refresh bool) (*media.ProcessingEmoji, error)
Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool
Handshaking(username string, remoteAccountID *url.URL) bool
}
type deref struct {
@ -53,28 +63,25 @@ type deref struct {
transportController transport.Controller
mediaManager media.Manager
dereferencingAvatars map[string]*media.ProcessingMedia
dereferencingAvatarsLock *sync.Mutex
dereferencingAvatarsLock sync.Mutex
dereferencingHeaders map[string]*media.ProcessingMedia
dereferencingHeadersLock *sync.Mutex
dereferencingHeadersLock sync.Mutex
dereferencingEmojis map[string]*media.ProcessingEmoji
dereferencingEmojisLock *sync.Mutex
dereferencingEmojisLock sync.Mutex
handshakes map[string][]*url.URL
handshakeSync *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
handshakeSync sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
}
// NewDereferencer returns a Dereferencer initialized with the given parameters.
func NewDereferencer(db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaManager media.Manager) Dereferencer {
return &deref{
db: db,
typeConverter: typeConverter,
transportController: transportController,
mediaManager: mediaManager,
dereferencingAvatars: make(map[string]*media.ProcessingMedia),
dereferencingAvatarsLock: &sync.Mutex{},
dereferencingHeaders: make(map[string]*media.ProcessingMedia),
dereferencingHeadersLock: &sync.Mutex{},
dereferencingEmojis: make(map[string]*media.ProcessingEmoji),
dereferencingEmojisLock: &sync.Mutex{},
handshakeSync: &sync.Mutex{},
db: db,
typeConverter: typeConverter,
transportController: transportController,
mediaManager: mediaManager,
dereferencingAvatars: make(map[string]*media.ProcessingMedia),
dereferencingHeaders: make(map[string]*media.ProcessingMedia),
dereferencingEmojis: make(map[string]*media.ProcessingEmoji),
handshakes: make(map[string][]*url.URL),
}
}

18
internal/federation/dereferencing/error.go
View file

@ -49,24 +49,10 @@ func (err *ErrNotRetrievable) Error() string {
return fmt.Sprintf("item could not be retrieved: %v", err.wrapped)
}
func newErrNotRetrievable(err error) error {
func NewErrNotRetrievable(err error) error {
return &ErrNotRetrievable{wrapped: err}
}
// ErrBadRequest denotes that insufficient or improperly formed parameters
// were passed into one of the dereference functions.
type ErrBadRequest struct {
wrapped error
}
func (err *ErrBadRequest) Error() string {
return fmt.Sprintf("bad request: %v", err.wrapped)
}
func newErrBadRequest(err error) error {
return &ErrBadRequest{wrapped: err}
}
// ErrTransportError indicates that something unforeseen went wrong creating
// a transport, or while making an http call to a remote resource with a transport.
type ErrTransportError struct {
@ -121,7 +107,7 @@ func wrapDerefError(derefErr error, fluff string) error {
switch {
case errors.Is(derefErr, transport.ErrGone):
err = newErrNotRetrievable(err)
err = NewErrNotRetrievable(err)
case errors.As(derefErr, &errWrongType):
err = newErrWrongType(err)
default:

13
internal/federation/dereferencing/handshake.go
View file

@ -19,11 +19,10 @@
package dereferencing
import (
"context"
"net/url"
)
func (d *deref) Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool {
func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool {
d.handshakeSync.Lock()
defer d.handshakeSync.Unlock()
@ -53,11 +52,6 @@ func (d *deref) startHandshake(username string, remoteAccountID *url.URL) {
d.handshakeSync.Lock()
defer d.handshakeSync.Unlock()
// lazily initialize handshakes
if d.handshakes == nil {
d.handshakes = make(map[string][]*url.URL)
}
remoteIDs, ok := d.handshakes[username]
if !ok {
// there was nothing in there yet, so just add this entry and return
@ -74,13 +68,8 @@ func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) {
d.handshakeSync.Lock()
defer d.handshakeSync.Unlock()
if d.handshakes == nil {
return
}
remoteIDs, ok := d.handshakes[username]
if !ok {
// there was nothing in there yet anyway so just bail
return
}

13
internal/federation/dereferencing/status.go
View file

@ -100,7 +100,7 @@ func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.U
if status != nil {
return status, nil, nil
}
return nil, nil, newErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString))
return nil, nil, NewErrNotRetrievable(fmt.Errorf("GetRemoteStatus: uri %s is apparently ours, but we have nothing in the db for it, will not proceed to dereference our own status", uriString))
}
// if we got here, either we didn't have the status
@ -123,11 +123,7 @@ func (d *deref) GetStatus(ctx context.Context, username string, statusURI *url.U
}
// we need to get the author of the status else we can't serialize it properly
if _, err = d.GetAccount(ctx, GetAccountParams{
RequestingUsername: username,
RemoteAccountID: accountURI,
Blocking: true,
}); err != nil {
if _, err = d.GetAccountByURI(ctx, username, accountURI, true); err != nil {
return nil, nil, newErrOther(fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err))
}
@ -353,10 +349,7 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta
if targetAccount == nil {
// we didn't find the account in our database already
// check if we can get the account remotely (dereference it)
if a, err := d.GetAccount(ctx, GetAccountParams{
RequestingUsername: requestingUsername,
RemoteAccountID: targetAccountURI,
}); err != nil {
if a, err := d.GetAccountByURI(ctx, requestingUsername, targetAccountURI, false); err != nil {
errs = append(errs, err.Error())
} else {
log.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID)

24
internal/federation/federatingdb/create.go
View file

@ -100,11 +100,7 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec
return fmt.Errorf("activityBlock: could not convert Block to gts model block")
}
newID, err := id.NewULID()
if err != nil {
return err
}
block.ID = newID
block.ID = id.NewULID()
if err := f.db.PutBlock(ctx, block); err != nil {
return fmt.Errorf("activityBlock: database error inserting block: %s", err)
@ -263,11 +259,7 @@ func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, re
return fmt.Errorf("activityFollow: could not convert Follow to follow request: %s", err)
}
newID, err := id.NewULID()
if err != nil {
return err
}
followRequest.ID = newID
followRequest.ID = id.NewULID()
if err := f.db.Put(ctx, followRequest); err != nil {
return fmt.Errorf("activityFollow: database error inserting follow request: %s", err)
@ -298,11 +290,7 @@ func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, rece
return fmt.Errorf("activityLike: could not convert Like to fave: %s", err)
}
newID, err := id.NewULID()
if err != nil {
return err
}
fave.ID = newID
fave.ID = id.NewULID()
if err := f.db.Put(ctx, fave); err != nil {
return fmt.Errorf("activityLike: database error inserting fave: %s", err)
@ -333,11 +321,7 @@ func (f *federatingDB) activityFlag(ctx context.Context, asType vocab.Type, rece
return fmt.Errorf("activityFlag: could not convert Flag to report: %w", err)
}
newID, err := id.NewULID()
if err != nil {
return err
}
report.ID = newID
report.ID = id.NewULID()
if err := f.db.PutReport(ctx, report); err != nil {
return fmt.Errorf("activityFlag: database error inserting report: %w", err)

2
internal/federation/federatingdb/update.go
View file

@ -116,7 +116,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
accountable = i
}
updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "", true)
updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "")
if err != nil {
return fmt.Errorf("UPDATE: error converting to account: %s", err)
}

8
internal/federation/federatingprotocol.go
View file

@ -31,7 +31,6 @@ import (
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/transport"
@ -206,10 +205,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
}
}
requestingAccount, err := f.GetAccount(transport.WithFastfail(ctx), dereferencing.GetAccountParams{
RequestingUsername: username,
RemoteAccountID: publicKeyOwnerURI,
})
requestingAccount, err := f.GetAccountByURI(
transport.WithFastfail(ctx), username, publicKeyOwnerURI, false,
)
if err != nil {
return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err)
}

19
internal/federation/federator.go
View file

@ -23,12 +23,10 @@ import (
"net/url"
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@ -53,20 +51,9 @@ type Federator interface {
// If something goes wrong during authentication, nil, false, and an error will be returned.
AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, gtserror.WithCode)
/*
dereferencing functions
*/
DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL, status *gtsmodel.Status, statusable ap.Statusable)
DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
GetAccount(ctx context.Context, params dereferencing.GetAccountParams) (*gtsmodel.Account, error)
GetStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)
GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
// Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID.
Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool
pub.CommonBehavior
pub.FederatingProtocol
dereferencing.Dereferencer
}
type federator struct {
@ -75,9 +62,9 @@ type federator struct {
clock pub.Clock
typeConverter typeutils.TypeConverter
transportController transport.Controller
dereferencer dereferencing.Dereferencer
mediaManager media.Manager
actor pub.FederatingActor
dereferencing.Dereferencer
}
// NewFederator returns a new federator
@ -91,8 +78,8 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr
clock: &Clock{},
typeConverter: typeConverter,
transportController: transportController,
dereferencer: dereferencer,
mediaManager: mediaManager,
Dereferencer: dereferencer,
}
actor := newFederatingActor(f, f, federatingDB, clock)
f.actor = actor

12
internal/federation/gone.go
View file

@ -2,12 +2,10 @@ package federation
import (
"context"
"fmt"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
// CheckGone checks if a tombstone exists in the database for AP Actor or Object with the given uri.
@ -17,18 +15,10 @@ func (f *federator) CheckGone(ctx context.Context, uri *url.URL) (bool, error) {
// HandleGone puts a tombstone in the database, which marks an AP Actor or Object with the given uri as gone.
func (f *federator) HandleGone(ctx context.Context, uri *url.URL) error {
tombstoneID, err := id.NewULID()
if err != nil {
err = fmt.Errorf("HandleGone: error generating id for new tombstone %s: %s", uri, err)
log.Error(err)
return err
}
tombstone := &gtsmodel.Tombstone{
ID: tombstoneID,
ID: id.NewULID(),
Domain: uri.Host,
URI: uri.String(),
}
return f.db.PutTombstone(ctx, tombstone)
}

9
internal/federation/handshake.go
View file

@ -17,12 +17,3 @@
*/
package federation
import (
"context"
"net/url"
)
func (f *federator) Handshaking(ctx context.Context, username string, remoteAccountID *url.URL) bool {
return f.dereferencer.Handshaking(ctx, username, remoteAccountID)
}