mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 07:22:24 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[performance] improved request batching (removes need for queueing) (#1687)

Browse source 
* revamp http client to not limit requests, instead use sender worker

Signed-off-by: kim <grufwub@gmail.com>

* remove separate sender worker pool, spawn 2*GOMAXPROCS batch senders each time, no need for transport cache sweeping

Signed-off-by: kim <grufwub@gmail.com>

* improve batch senders to keep popping recipients until remote URL found

Signed-off-by: kim <grufwub@gmail.com>

* fix recipient looping issue

Signed-off-by: kim <grufwub@gmail.com>

* fix missing mutex unlock

Signed-off-by: kim <grufwub@gmail.com>

* move request id ctx key to gtscontext, finish filling out more code comments, add basic support for not logging client IP

Signed-off-by: kim <grufwub@gmail.com>

* slight code reformatting

Signed-off-by: kim <grufwub@gmail.com>

* a whitespace

Signed-off-by: kim <grufwub@gmail.com>

* remove unused code

Signed-off-by: kim <grufwub@gmail.com>

* add missing license headers

Signed-off-by: kim <grufwub@gmail.com>

* fix request backoff calculation

Signed-off-by: kim <grufwub@gmail.com>

---------

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2023-04-28 16:45:21 +01:00 • committed by GitHub
commit 6a29c5ffd4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 431 additions and 493 deletions
Download patch file Download diff file Expand all files Collapse all files

19
internal/middleware/logger.go
View file

@ -34,7 +34,7 @@ import (
func Logger() gin.HandlerFunc {
return func(c *gin.Context) {
// Initialize the logging fields
fields := make(kv.Fields, 6, 7)
fields := make(kv.Fields, 5, 7)
// Determine pre-handler time
before := time.Now()
@ -68,11 +68,18 @@ func Logger() gin.HandlerFunc {
// Set request logging fields
fields[0] = kv.Field{"latency", time.Since(before)}
fields[1] = kv.Field{"clientIP", c.ClientIP()}
fields[2] = kv.Field{"userAgent", c.Request.UserAgent()}
fields[3] = kv.Field{"method", c.Request.Method}
fields[4] = kv.Field{"statusCode", code}
fields[5] = kv.Field{"path", path}
fields[1] = kv.Field{"userAgent", c.Request.UserAgent()}
fields[2] = kv.Field{"method", c.Request.Method}
fields[3] = kv.Field{"statusCode", code}
fields[4] = kv.Field{"path", path}
if includeClientIP := true; includeClientIP {
// TODO: make this configurable.
//
// Include clientIP if enabled.
fields = append(fields, kv.Field{
"clientIP", c.ClientIP(),
})
}
// Create log entry with fields
l := log.WithContext(c.Request.Context()).

27
internal/middleware/requestid.go
View file

@ -19,7 +19,6 @@ package middleware
import (
"bufio"
"context"
"crypto/rand"
"encoding/base32"
"encoding/binary"
@ -27,17 +26,11 @@ import (
"sync"
"time"
"codeberg.org/gruf/go-kv"
"github.com/gin-gonic/gin"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
)
type ctxType string
var (
// ridCtxKey is the key underwhich we store request IDs in a context.
ridCtxKey ctxType = "id"
// crand provides buffered reads of random input.
crand = bufio.NewReader(rand.Reader)
mrand sync.Mutex
@ -69,22 +62,8 @@ func generateID() string {
return base32enc.EncodeToString(b)
}
// RequestID fetches the stored request ID from context.
func RequestID(ctx context.Context) string {
id, _ := ctx.Value(ridCtxKey).(string)
return id
}
// AddRequestID returns a gin middleware which adds a unique ID to each request (both response header and context).
func AddRequestID(header string) gin.HandlerFunc {
log.Hook(func(ctx context.Context, kvs []kv.Field) []kv.Field {
if id, _ := ctx.Value(ridCtxKey).(string); id != "" {
// Add stored request ID to log entry fields.
return append(kvs, kv.Field{K: "requestID", V: id})
}
return kvs
})
return func(c *gin.Context) {
// Look for existing ID.
id := c.GetHeader(header)
@ -100,8 +79,8 @@ func AddRequestID(header string) gin.HandlerFunc {
c.Request.Header.Set(header, id)
}
// Store request ID in new request ctx and set new gin request obj.
ctx := context.WithValue(c.Request.Context(), ridCtxKey, id)
// Store request ID in new request context and set on gin ctx.
ctx := gtscontext.SetRequestID(c.Request.Context(), id)
c.Request = c.Request.WithContext(ctx)
// Set the request ID in the rsp header.