mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 13:22:25 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[performance] improved request batching (removes need for queueing) (#1687)

Browse source 
* revamp http client to not limit requests, instead use sender worker

Signed-off-by: kim <grufwub@gmail.com>

* remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping

Signed-off-by: kim <grufwub@gmail.com>

* improve batch senders to keep popping recipients until remote URL found

Signed-off-by: kim <grufwub@gmail.com>

* fix recipient looping issue

Signed-off-by: kim <grufwub@gmail.com>

* fix missing mutex unlock

Signed-off-by: kim <grufwub@gmail.com>

* move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP

Signed-off-by: kim <grufwub@gmail.com>

* slight code reformatting

Signed-off-by: kim <grufwub@gmail.com>

* a whitespace

Signed-off-by: kim <grufwub@gmail.com>

* remove unused code

Signed-off-by: kim <grufwub@gmail.com>

* add missing license headers

Signed-off-by: kim <grufwub@gmail.com>

* fix request backoff calculation

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2023-04-28 16:45:21 +01:00 • committed by GitHub
commit 6a29c5ffd4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 431 additions and 493 deletions
Download patch file Download diff file Expand all files Collapse all files

24
internal/transport/controller.go
View file

@ -24,7 +24,7 @@ import (
"encoding/json"
"fmt"
"net/url"
"time"
"runtime"
"codeberg.org/gruf/go-byteutil"
"codeberg.org/gruf/go-cache/v3"
@ -32,7 +32,7 @@ import (
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/httpclient"
"github.com/superseriousbusiness/gotosocial/internal/state"
)
@ -49,14 +49,14 @@ type controller struct {
state *state.State
fedDB federatingdb.DB
clock pub.Clock
client pub.HttpClient
client httpclient.SigningClient
trspCache cache.Cache[string, *transport]
badHosts cache.Cache[string, struct{}]
userAgent string
senders int // no. concurrent batch delivery routines.
}
// NewController returns an implementation of the Controller interface for creating new transports
func NewController(state *state.State, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
func NewController(state *state.State, federatingDB federatingdb.DB, clock pub.Clock, client httpclient.SigningClient) Controller {
applicationName := config.GetApplicationName()
host := config.GetHost()
proto := config.GetProtocol()
@ -68,20 +68,8 @@ func NewController(state *state.State, federatingDB federatingdb.DB, clock pub.C
clock: clock,
client: client,
trspCache: cache.New[string, *transport](0, 100, 0),
badHosts: cache.New[string, struct{}](0, 1000, 0),
userAgent: fmt.Sprintf("%s (+%s://%s) gotosocial/%s", applicationName, proto, host, version),
}
// Transport cache has TTL=1hr freq=1min
c.trspCache.SetTTL(time.Hour, false)
if !c.trspCache.Start(time.Minute) {
log.Panic(nil, "failed to start transport controller cache")
}
// Bad hosts cache has TTL=15min freq=1min
c.badHosts.SetTTL(15*time.Minute, false)
if !c.badHosts.Start(time.Minute) {
log.Panic(nil, "failed to start transport controller cache")
senders: runtime.GOMAXPROCS(0), // on batch delivery, only ever send GOMAXPROCS at a time.
}
return c