mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-18 15:39:20 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

replace async client API / federator msg processing with worker pools (#497)

Browse source 
* replace async client API / federator msg processing with worker pools
* appease our lord-and-saviour, the linter
This commit is contained in:
kim 2022-04-28 13:23:11 +01:00 • committed by GitHub
commit 420e2fb22b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
64 changed files with 573 additions and 336 deletions
Download patch file Download diff file Expand all files Collapse all files

39
internal/processing/account/account.go
View file

@ -33,6 +33,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/visibility"
"github.com/superseriousbusiness/gotosocial/internal/worker"
"github.com/superseriousbusiness/oauth2/v4"
)
@ -81,28 +82,28 @@ type Processor interface {
}
type processor struct {
tc typeutils.TypeConverter
mediaManager media.Manager
fromClientAPI chan messages.FromClientAPI
oauthServer oauth.Server
filter visibility.Filter
formatter text.Formatter
db db.DB
federator federation.Federator
parseMention gtsmodel.ParseMentionFunc
tc typeutils.TypeConverter
mediaManager media.Manager
clientWorker *worker.Worker[messages.FromClientAPI]
oauthServer oauth.Server
filter visibility.Filter
formatter text.Formatter
db db.DB
federator federation.Federator
parseMention gtsmodel.ParseMentionFunc
}
// New returns a new account processor.
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor {
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, clientWorker *worker.Worker[messages.FromClientAPI], federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor {
return &processor{
tc: tc,
mediaManager: mediaManager,
fromClientAPI: fromClientAPI,
oauthServer: oauthServer,
filter: visibility.NewFilter(db),
formatter: text.NewFormatter(db),
db: db,
federator: federator,
parseMention: parseMention,
tc: tc,
mediaManager: mediaManager,
clientWorker: clientWorker,
oauthServer: oauthServer,
filter: visibility.NewFilter(db),
formatter: text.NewFormatter(db),
db: db,
federator: federator,
parseMention: parseMention,
}
}

19
internal/processing/account/account_test.go
View file

@ -19,6 +19,8 @@
package account_test
import (
"context"
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/pub"
@ -33,6 +35,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/processing/account"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/worker"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -78,6 +81,16 @@ func (suite *AccountStandardTestSuite) SetupTest() {
testrig.InitTestLog()
testrig.InitTestConfig()
fedWorker := worker.New[messages.FromFederator](-1, -1)
clientWorker := worker.New[messages.FromClientAPI](-1, -1)
clientWorker.SetProcessor(func(_ context.Context, msg messages.FromClientAPI) error {
suite.fromClientAPIChan <- msg
return nil
})
_ = fedWorker.Start()
_ = clientWorker.Start()
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
@ -85,11 +98,11 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
suite.httpClient = testrig.NewMockHTTPClient(nil)
suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db)
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager)
suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)
suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator))
suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, clientWorker, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator))
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
}

4
internal/processing/account/create.go
View file

@ -85,12 +85,12 @@ func (p *processor) Create(ctx context.Context, applicationToken oauth2.TokenInf
// there are side effects for creating a new account (sending confirmation emails etc)
// so pass a message to the processor so that it can do it asynchronously
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ObjectProfile,
APActivityType: ap.ActivityCreate,
GTSModel: user.Account,
OriginAccount: user.Account,
}
})
return &apimodel.Token{
AccessToken: accessToken.GetAccess(),

12
internal/processing/account/createblock.go
View file

@ -113,7 +113,7 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel
// follow request status changed so send the UNDO activity to the channel for async processing
if frChanged {
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityFollow,
APActivityType: ap.ActivityUndo,
GTSModel: &gtsmodel.Follow{
@ -123,12 +123,12 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel
},
OriginAccount: requestingAccount,
TargetAccount: targetAccount,
}
})
}
// follow status changed so send the UNDO activity to the channel for async processing
if fChanged {
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityFollow,
APActivityType: ap.ActivityUndo,
GTSModel: &gtsmodel.Follow{
@ -138,17 +138,17 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel
},
OriginAccount: requestingAccount,
TargetAccount: targetAccount,
}
})
}
// handle the rest of the block process asynchronously
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityBlock,
APActivityType: ap.ActivityCreate,
GTSModel: block,
OriginAccount: requestingAccount,
TargetAccount: targetAccount,
}
})
return p.RelationshipGet(ctx, requestingAccount, targetAccountID)
}

4
internal/processing/account/createfollow.go
View file

@ -101,13 +101,13 @@ func (p *processor) FollowCreate(ctx context.Context, requestingAccount *gtsmode
}
// otherwise we leave the follow request as it is and we handle the rest of the process asynchronously
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityFollow,
APActivityType: ap.ActivityCreate,
GTSModel: fr,
OriginAccount: requestingAccount,
TargetAccount: targetAcct,
}
})
// return whatever relationship results from this
return p.RelationshipGet(ctx, requestingAccount, form.ID)

10
internal/processing/account/delete.go
View file

@ -159,13 +159,13 @@ selectStatusesLoop:
// pass the status delete through the client api channel for processing
s.Account = account
l.Debug("putting status in the client api channel")
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ObjectNote,
APActivityType: ap.ActivityDelete,
GTSModel: s,
OriginAccount: account,
TargetAccount: account,
}
})
if err := p.db.DeleteByID(ctx, s.ID, s); err != nil {
if err != db.ErrNoEntries {
@ -195,13 +195,13 @@ selectStatusesLoop:
}
l.Debug("putting boost undo in the client api channel")
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityAnnounce,
APActivityType: ap.ActivityUndo,
GTSModel: s,
OriginAccount: b.Account,
TargetAccount: account,
}
})
if err := p.db.DeleteByID(ctx, b.ID, b); err != nil {
if err != db.ErrNoEntries {
@ -331,7 +331,7 @@ func (p *processor) DeleteLocal(ctx context.Context, account *gtsmodel.Account,
}
// put the delete in the processor queue to handle the rest of it asynchronously
p.fromClientAPI <- fromClientAPIMessage
p.clientWorker.Queue(fromClientAPIMessage)
return nil
}

4
internal/processing/account/removeblock.go
View file

@ -54,13 +54,13 @@ func (p *processor) BlockRemove(ctx context.Context, requestingAccount *gtsmodel
// block status changed so send the UNDO activity to the channel for async processing
if blockChanged {
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityBlock,
APActivityType: ap.ActivityUndo,
GTSModel: block,
OriginAccount: requestingAccount,
TargetAccount: targetAccount,
}
})
}
// return whatever relationship results from all this

8
internal/processing/account/removefollow.go
View file

@ -80,7 +80,7 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode
// follow request status changed so send the UNDO activity to the channel for async processing
if frChanged {
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityFollow,
APActivityType: ap.ActivityUndo,
GTSModel: &gtsmodel.Follow{
@ -90,12 +90,12 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode
},
OriginAccount: requestingAccount,
TargetAccount: targetAcct,
}
})
}
// follow status changed so send the UNDO activity to the channel for async processing
if fChanged {
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ActivityFollow,
APActivityType: ap.ActivityUndo,
GTSModel: &gtsmodel.Follow{
@ -105,7 +105,7 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode
},
OriginAccount: requestingAccount,
TargetAccount: targetAcct,
}
})
}
// return whatever relationship results from all this

4
internal/processing/account/update.go
View file

@ -117,12 +117,12 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form
return nil, fmt.Errorf("could not update account %s: %s", account.ID, err)
}
p.fromClientAPI <- messages.FromClientAPI{
p.clientWorker.Queue(messages.FromClientAPI{
APObjectType: ap.ObjectProfile,
APActivityType: ap.ActivityUpdate,
GTSModel: updatedAccount,
OriginAccount: updatedAccount,
}
})
acctSensitive, err := p.tc.AccountToAPIAccountSensitive(ctx, updatedAccount)
if err != nil {