[chore] move client/federator workerpools to Workers{} (#1575)

* replace concurrency worker pools with base models in State.Workers, update code and tests accordingly

* improve code comment

* change back testrig default log level

* un-comment-out TestAnnounceTwice() and fix

---------

Signed-off-by: kim <grufwub@gmail.com>
Reviewed-by: tobi
This commit is contained in:
kim 2023-03-01 18:26:53 +00:00 committed by GitHub
commit baf933cb9f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
130 changed files with 1037 additions and 1083 deletions

View file

@ -31,7 +31,7 @@ import (
)
func (p *Processor) AccountAction(ctx context.Context, account *gtsmodel.Account, form *apimodel.AdminAccountActionRequest) gtserror.WithCode {
targetAccount, err := p.db.GetAccountByID(ctx, form.TargetAccountID)
targetAccount, err := p.state.DB.GetAccountByID(ctx, form.TargetAccountID)
if err != nil {
return gtserror.NewErrorInternalError(err)
}
@ -47,7 +47,7 @@ func (p *Processor) AccountAction(ctx context.Context, account *gtsmodel.Account
case string(gtsmodel.AdminActionSuspend):
adminAction.Type = gtsmodel.AdminActionSuspend
// pass the account delete through the client api channel for processing
p.clientWorker.Queue(messages.FromClientAPI{
p.state.Workers.EnqueueClientAPI(ctx, messages.FromClientAPI{
APObjectType: ap.ActorPerson,
APActivityType: ap.ActivityDelete,
OriginAccount: account,
@ -57,7 +57,7 @@ func (p *Processor) AccountAction(ctx context.Context, account *gtsmodel.Account
return gtserror.NewErrorBadRequest(fmt.Errorf("admin action type %s is not supported for this endpoint", form.Type))
}
if err := p.db.Put(ctx, adminAction); err != nil {
if err := p.state.DB.Put(ctx, adminAction); err != nil {
return gtserror.NewErrorInternalError(err)
}

View file

@ -19,32 +19,25 @@
package admin
import (
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
type Processor struct {
state *state.State
tc typeutils.TypeConverter
mediaManager media.Manager
transportController transport.Controller
storage *storage.Driver
clientWorker *concurrency.WorkerPool[messages.FromClientAPI]
db db.DB
}
// New returns a new admin processor.
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *storage.Driver, clientWorker *concurrency.WorkerPool[messages.FromClientAPI]) Processor {
func New(state *state.State, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller) Processor {
return Processor{
state: state,
tc: tc,
mediaManager: mediaManager,
transportController: transportController,
storage: storage,
clientWorker: clientWorker,
db: db,
}
}

View file

@ -28,7 +28,7 @@ func (p *Processor) DomainBlockCreate(ctx context.Context, account *gtsmodel.Acc
domain = strings.ToLower(domain)
// first check if we already have a block -- if err == nil we already had a block so we can skip a whole lot of work
block, err := p.db.GetDomainBlock(ctx, domain)
block, err := p.state.DB.GetDomainBlock(ctx, domain)
if err != nil {
if !errors.Is(err, db.ErrNoEntries) {
// something went wrong in the DB
@ -47,7 +47,7 @@ func (p *Processor) DomainBlockCreate(ctx context.Context, account *gtsmodel.Acc
}
// Insert the new block into the database
if err := p.db.CreateDomainBlock(ctx, newBlock); err != nil {
if err := p.state.DB.CreateDomainBlock(ctx, newBlock); err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("db error putting new domain block %s: %s", domain, err))
}
@ -80,7 +80,7 @@ func (p *Processor) initiateDomainBlockSideEffects(ctx context.Context, account
// if we have an instance entry for this domain, update it with the new block ID and clear all fields
instance := &gtsmodel.Instance{}
if err := p.db.GetWhere(ctx, []db.Where{{Key: "domain", Value: block.Domain}}, instance); err == nil {
if err := p.state.DB.GetWhere(ctx, []db.Where{{Key: "domain", Value: block.Domain}}, instance); err == nil {
updatingColumns := []string{
"title",
"updated_at",
@ -105,15 +105,15 @@ func (p *Processor) initiateDomainBlockSideEffects(ctx context.Context, account
instance.ContactAccountUsername = ""
instance.ContactAccountID = ""
instance.Version = ""
if err := p.db.UpdateByID(ctx, instance, instance.ID, updatingColumns...); err != nil {
if err := p.state.DB.UpdateByID(ctx, instance, instance.ID, updatingColumns...); err != nil {
l.Errorf("domainBlockProcessSideEffects: db error updating instance: %s", err)
}
l.Debug("domainBlockProcessSideEffects: instance entry updated")
}
// if we have an instance account for this instance, delete it
if instanceAccount, err := p.db.GetAccountByUsernameDomain(ctx, block.Domain, block.Domain); err == nil {
if err := p.db.DeleteAccount(ctx, instanceAccount.ID); err != nil {
if instanceAccount, err := p.state.DB.GetAccountByUsernameDomain(ctx, block.Domain, block.Domain); err == nil {
if err := p.state.DB.DeleteAccount(ctx, instanceAccount.ID); err != nil {
l.Errorf("domainBlockProcessSideEffects: db error deleting instance account: %s", err)
}
}
@ -125,7 +125,7 @@ func (p *Processor) initiateDomainBlockSideEffects(ctx context.Context, account
selectAccountsLoop:
for {
accounts, err := p.db.GetInstanceAccounts(ctx, block.Domain, maxID, limit)
accounts, err := p.state.DB.GetInstanceAccounts(ctx, block.Domain, maxID, limit)
if err != nil {
if err == db.ErrNoEntries {
// no accounts left for this instance so we're done
@ -141,7 +141,7 @@ selectAccountsLoop:
l.Debugf("putting delete for account %s in the clientAPI channel", a.Username)
// pass the account delete through the client api channel for processing
p.clientWorker.Queue(messages.FromClientAPI{
p.state.Workers.EnqueueClientAPI(ctx, messages.FromClientAPI{
APObjectType: ap.ActorPerson,
APActivityType: ap.ActivityDelete,
GTSModel: block,
@ -195,7 +195,7 @@ func (p *Processor) DomainBlocksImport(ctx context.Context, account *gtsmodel.Ac
func (p *Processor) DomainBlocksGet(ctx context.Context, account *gtsmodel.Account, export bool) ([]*apimodel.DomainBlock, gtserror.WithCode) {
domainBlocks := []*gtsmodel.DomainBlock{}
if err := p.db.GetAll(ctx, &domainBlocks); err != nil {
if err := p.state.DB.GetAll(ctx, &domainBlocks); err != nil {
if !errors.Is(err, db.ErrNoEntries) {
// something has gone really wrong
return nil, gtserror.NewErrorInternalError(err)
@ -219,7 +219,7 @@ func (p *Processor) DomainBlocksGet(ctx context.Context, account *gtsmodel.Accou
func (p *Processor) DomainBlockGet(ctx context.Context, account *gtsmodel.Account, id string, export bool) (*apimodel.DomainBlock, gtserror.WithCode) {
domainBlock := &gtsmodel.DomainBlock{}
if err := p.db.GetByID(ctx, id, domainBlock); err != nil {
if err := p.state.DB.GetByID(ctx, id, domainBlock); err != nil {
if !errors.Is(err, db.ErrNoEntries) {
// something has gone really wrong
return nil, gtserror.NewErrorInternalError(err)
@ -240,7 +240,7 @@ func (p *Processor) DomainBlockGet(ctx context.Context, account *gtsmodel.Accoun
func (p *Processor) DomainBlockDelete(ctx context.Context, account *gtsmodel.Account, id string) (*apimodel.DomainBlock, gtserror.WithCode) {
domainBlock := &gtsmodel.DomainBlock{}
if err := p.db.GetByID(ctx, id, domainBlock); err != nil {
if err := p.state.DB.GetByID(ctx, id, domainBlock); err != nil {
if !errors.Is(err, db.ErrNoEntries) {
// something has gone really wrong
return nil, gtserror.NewErrorInternalError(err)
@ -256,13 +256,13 @@ func (p *Processor) DomainBlockDelete(ctx context.Context, account *gtsmodel.Acc
}
// Delete the domain block
if err := p.db.DeleteDomainBlock(ctx, domainBlock.Domain); err != nil {
if err := p.state.DB.DeleteDomainBlock(ctx, domainBlock.Domain); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// remove the domain block reference from the instance, if we have an entry for it
i := &gtsmodel.Instance{}
if err := p.db.GetWhere(ctx, []db.Where{
if err := p.state.DB.GetWhere(ctx, []db.Where{
{Key: "domain", Value: domainBlock.Domain},
{Key: "domain_block_id", Value: id},
}, i); err == nil {
@ -270,21 +270,21 @@ func (p *Processor) DomainBlockDelete(ctx context.Context, account *gtsmodel.Acc
i.SuspendedAt = time.Time{}
i.DomainBlockID = ""
i.UpdatedAt = time.Now()
if err := p.db.UpdateByID(ctx, i, i.ID, updatingColumns...); err != nil {
if err := p.state.DB.UpdateByID(ctx, i, i.ID, updatingColumns...); err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("couldn't update database entry for instance %s: %s", domainBlock.Domain, err))
}
}
// unsuspend all accounts whose suspension origin was this domain block
// 1. remove the 'suspended_at' entry from their accounts
if err := p.db.UpdateWhere(ctx, []db.Where{
if err := p.state.DB.UpdateWhere(ctx, []db.Where{
{Key: "suspension_origin", Value: domainBlock.ID},
}, "suspended_at", nil, &[]*gtsmodel.Account{}); err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("database error removing suspended_at from accounts: %s", err))
}
// 2. remove the 'suspension_origin' entry from their accounts
if err := p.db.UpdateWhere(ctx, []db.Where{
if err := p.state.DB.UpdateWhere(ctx, []db.Where{
{Key: "suspension_origin", Value: domainBlock.ID},
}, "suspension_origin", nil, &[]*gtsmodel.Account{}); err != nil {
return nil, gtserror.NewErrorInternalError(fmt.Errorf("database error removing suspension_origin from accounts: %s", err))

View file

@ -42,7 +42,7 @@ func (p *Processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account,
return nil, gtserror.NewErrorUnauthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin")
}
maybeExisting, err := p.db.GetEmojiByShortcodeDomain(ctx, form.Shortcode, "")
maybeExisting, err := p.state.DB.GetEmojiByShortcodeDomain(ctx, form.Shortcode, "")
if maybeExisting != nil {
return nil, gtserror.NewErrorConflict(fmt.Errorf("emoji with shortcode %s already exists", form.Shortcode), fmt.Sprintf("emoji with shortcode %s already exists", form.Shortcode))
}
@ -110,7 +110,7 @@ func (p *Processor) EmojisGet(
return nil, gtserror.NewErrorUnauthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin")
}
emojis, err := p.db.GetEmojis(ctx, domain, includeDisabled, includeEnabled, shortcode, maxShortcodeDomain, minShortcodeDomain, limit)
emojis, err := p.state.DB.GetEmojis(ctx, domain, includeDisabled, includeEnabled, shortcode, maxShortcodeDomain, minShortcodeDomain, limit)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err := fmt.Errorf("EmojisGet: db error: %s", err)
return nil, gtserror.NewErrorInternalError(err)
@ -176,7 +176,7 @@ func (p *Processor) EmojiGet(ctx context.Context, account *gtsmodel.Account, use
return nil, gtserror.NewErrorUnauthorized(fmt.Errorf("user %s not an admin", user.ID), "user is not an admin")
}
emoji, err := p.db.GetEmojiByID(ctx, id)
emoji, err := p.state.DB.GetEmojiByID(ctx, id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
err = fmt.Errorf("EmojiGet: no emoji with id %s found in the db", id)
@ -197,7 +197,7 @@ func (p *Processor) EmojiGet(ctx context.Context, account *gtsmodel.Account, use
// EmojiDelete deletes one emoji from the database, with the given id.
func (p *Processor) EmojiDelete(ctx context.Context, id string) (*apimodel.AdminEmoji, gtserror.WithCode) {
emoji, err := p.db.GetEmojiByID(ctx, id)
emoji, err := p.state.DB.GetEmojiByID(ctx, id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
err = fmt.Errorf("EmojiDelete: no emoji with id %s found in the db", id)
@ -218,7 +218,7 @@ func (p *Processor) EmojiDelete(ctx context.Context, id string) (*apimodel.Admin
return nil, gtserror.NewErrorInternalError(err)
}
if err := p.db.DeleteEmojiByID(ctx, id); err != nil {
if err := p.state.DB.DeleteEmojiByID(ctx, id); err != nil {
err := fmt.Errorf("EmojiDelete: db error: %s", err)
return nil, gtserror.NewErrorInternalError(err)
}
@ -228,7 +228,7 @@ func (p *Processor) EmojiDelete(ctx context.Context, id string) (*apimodel.Admin
// EmojiUpdate updates one emoji with the given id, using the provided form parameters.
func (p *Processor) EmojiUpdate(ctx context.Context, id string, form *apimodel.EmojiUpdateRequest) (*apimodel.AdminEmoji, gtserror.WithCode) {
emoji, err := p.db.GetEmojiByID(ctx, id)
emoji, err := p.state.DB.GetEmojiByID(ctx, id)
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
err = fmt.Errorf("EmojiUpdate: no emoji with id %s found in the db", id)
@ -253,7 +253,7 @@ func (p *Processor) EmojiUpdate(ctx context.Context, id string, form *apimodel.E
// EmojiCategoriesGet returns all custom emoji categories that exist on this instance.
func (p *Processor) EmojiCategoriesGet(ctx context.Context) ([]*apimodel.EmojiCategory, gtserror.WithCode) {
categories, err := p.db.GetEmojiCategories(ctx)
categories, err := p.state.DB.GetEmojiCategories(ctx)
if err != nil {
err := fmt.Errorf("EmojiCategoriesGet: db error: %s", err)
return nil, gtserror.NewErrorInternalError(err)
@ -277,7 +277,7 @@ func (p *Processor) EmojiCategoriesGet(ctx context.Context) ([]*apimodel.EmojiCa
*/
func (p *Processor) getOrCreateEmojiCategory(ctx context.Context, name string) (*gtsmodel.EmojiCategory, error) {
category, err := p.db.GetEmojiCategoryByName(ctx, name)
category, err := p.state.DB.GetEmojiCategoryByName(ctx, name)
if err == nil {
return category, nil
}
@ -299,7 +299,7 @@ func (p *Processor) getOrCreateEmojiCategory(ctx context.Context, name string) (
Name: name,
}
if err := p.db.PutEmojiCategory(ctx, category); err != nil {
if err := p.state.DB.PutEmojiCategory(ctx, category); err != nil {
err = fmt.Errorf("GetOrCreateEmojiCategory: error putting new emoji category in the database: %s", err)
return nil, err
}
@ -319,7 +319,7 @@ func (p *Processor) emojiUpdateCopy(ctx context.Context, emoji *gtsmodel.Emoji,
return nil, gtserror.NewErrorBadRequest(err, err.Error())
}
maybeExisting, err := p.db.GetEmojiByShortcodeDomain(ctx, *shortcode, "")
maybeExisting, err := p.state.DB.GetEmojiByShortcodeDomain(ctx, *shortcode, "")
if maybeExisting != nil {
err := fmt.Errorf("emojiUpdateCopy: emoji %s could not be copied, emoji with shortcode %s already exists on this instance", emoji.ID, *shortcode)
return nil, gtserror.NewErrorConflict(err, err.Error())
@ -339,7 +339,7 @@ func (p *Processor) emojiUpdateCopy(ctx context.Context, emoji *gtsmodel.Emoji,
newEmojiURI := uris.GenerateURIForEmoji(newEmojiID)
data := func(ctx context.Context) (reader io.ReadCloser, fileSize int64, err error) {
rc, err := p.storage.GetStream(ctx, emoji.ImagePath)
rc, err := p.state.Storage.GetStream(ctx, emoji.ImagePath)
return rc, int64(emoji.ImageFileSize), err
}
@ -386,7 +386,7 @@ func (p *Processor) emojiUpdateDisable(ctx context.Context, emoji *gtsmodel.Emoj
emojiDisabled := true
emoji.Disabled = &emojiDisabled
updatedEmoji, err := p.db.UpdateEmoji(ctx, emoji, "updated_at", "disabled")
updatedEmoji, err := p.state.DB.UpdateEmoji(ctx, emoji, "updated_at", "disabled")
if err != nil {
err = fmt.Errorf("emojiUpdateDisable: error updating emoji %s: %s", emoji.ID, err)
return nil, gtserror.NewErrorInternalError(err)
@ -443,7 +443,7 @@ func (p *Processor) emojiUpdateModify(ctx context.Context, emoji *gtsmodel.Emoji
}
var err error
updatedEmoji, err = p.db.UpdateEmoji(ctx, emoji, columns...)
updatedEmoji, err = p.state.DB.UpdateEmoji(ctx, emoji, columns...)
if err != nil {
err = fmt.Errorf("emojiUpdateModify: error updating emoji %s: %s", emoji.ID, err)
return nil, gtserror.NewErrorInternalError(err)

View file

@ -43,7 +43,7 @@ func (p *Processor) ReportsGet(
minID string,
limit int,
) (*apimodel.PageableResponse, gtserror.WithCode) {
reports, err := p.db.GetReports(ctx, resolved, accountID, targetAccountID, maxID, sinceID, minID, limit)
reports, err := p.state.DB.GetReports(ctx, resolved, accountID, targetAccountID, maxID, sinceID, minID, limit)
if err != nil {
if err == db.ErrNoEntries {
return util.EmptyPageableResponse(), nil
@ -95,7 +95,7 @@ func (p *Processor) ReportsGet(
// ReportGet returns one report, with the given ID.
func (p *Processor) ReportGet(ctx context.Context, account *gtsmodel.Account, id string) (*apimodel.AdminReport, gtserror.WithCode) {
report, err := p.db.GetReportByID(ctx, id)
report, err := p.state.DB.GetReportByID(ctx, id)
if err != nil {
if err == db.ErrNoEntries {
return nil, gtserror.NewErrorNotFound(err)
@ -113,7 +113,7 @@ func (p *Processor) ReportGet(ctx context.Context, account *gtsmodel.Account, id
// ReportResolve marks a report with the given id as resolved, and stores the provided actionTakenComment (if not null).
func (p *Processor) ReportResolve(ctx context.Context, account *gtsmodel.Account, id string, actionTakenComment *string) (*apimodel.AdminReport, gtserror.WithCode) {
report, err := p.db.GetReportByID(ctx, id)
report, err := p.state.DB.GetReportByID(ctx, id)
if err != nil {
if err == db.ErrNoEntries {
return nil, gtserror.NewErrorNotFound(err)
@ -134,7 +134,7 @@ func (p *Processor) ReportResolve(ctx context.Context, account *gtsmodel.Account
columns = append(columns, "action_taken")
}
updatedReport, err := p.db.UpdateReport(ctx, report, columns...)
updatedReport, err := p.state.DB.UpdateReport(ctx, report, columns...)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}