[performance] massively improved ActivityPub delivery worker efficiency (#2812)

* add delivery worker type that pulls from queue to httpclient package

* finish up some code commenting, bodge a vendored activity library change, integrate the deliverypool changes into transportcontroller

* hook up queue deletion logic

* support deleting queued http requests by target ID

* don't index APRequest by hostname in the queue

* use gorun

* use the original context's values when wrapping msg type as delivery{}

* actually log in the AP delivery worker ...

* add uncommitted changes

* use errors.AsV2()

* use errorsv2.AsV2()

* finish adding some code comments, add bad host handling to delivery workers

* slightly tweak deliveryworkerpool API, use advanced sender multiplier

* remove PopCtx() method, let others instead rely on Wait()

* shuffle things around to move delivery stuff into transport/ subpkg

* remove dead code

* formatting

* validate request before queueing for delivery

* finish adding code comments, fix up backoff code

* finish adding more code comments

* clamp minimum no. senders to 1

* add start/stop logging to delivery worker, some slight changes

* remove double logging

* use worker ptrs

* expose the embedded log fields in httpclient.Request{}

* ensure request context values are preserved when updating ctx

* add delivery worker tests

* fix linter issues

* ensure delivery worker gets inited in testrig

* fix tests to delivering messages to check worker delivery queue

* update error type to use ptr instead of value receiver

* fix test calling Workers{}.Start() instead of testrig.StartWorkers()

* update docs for advanced-sender-multiplier

* update to the latest activity library version

* add comment about not using httptest.Server{}

internal/federation/federatingactor.go

@@ -89,7 +89,7 @@ func (f *federatingActor) PostInboxScheme(ctx context.Context, w http.ResponseWr
 	// so we specifically have to check for already wrapped with code.
 	//
 	ctx, authenticated, err := f.sideEffectActor.AuthenticatePostInbox(ctx, w, r)
-	if errors.As(err, new(gtserror.WithCode)) {
+	if errorsv2.AsV2[gtserror.WithCode](err) != nil {
 		// If it was already wrapped with an
 		// HTTP code then don't bother rewrapping
 		// it, just return it as-is for caller to
@@ -131,7 +131,7 @@ func (f *federatingActor) PostInboxScheme(ctx context.Context, w http.ResponseWr
 	// Check authorization of the activity; this will include blocks.
 	authorized, err := f.sideEffectActor.AuthorizePostInbox(ctx, w, activity)
 	if err != nil {
-		if errors.As(err, new(errOtherIRIBlocked)) {
+		if errorsv2.AsV2[*errOtherIRIBlocked](err) != nil {
 			// There's no direct block between requester(s) and
 			// receiver. However, one or more of the other IRIs
 			// involved in the request (account replied to, note
@@ -139,7 +139,7 @@ func (f *federatingActor) PostInboxScheme(ctx context.Context, w http.ResponseWr
 			// by the receiver. We don't need to return 403 here,
 			// instead, just return 202 accepted but don't do any
 			// further processing of the activity.
-			return true, nil
+			return true, nil //nolint
 		}
 
 		// Real error has occurred.

internal/federation/federatingactor_test.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"io"
 	"net/url"
 	"testing"
 	"time"
@@ -129,23 +130,27 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
 	suite.NotNil(activity)
 
 	// because we added 1 remote follower for zork, there should be a url in sentMessage
-	var sent [][]byte
+	var sent []byte
 	if !testrig.WaitFor(func() bool {
-		sentI, ok := httpClient.SentMessages.Load(*testRemoteAccount.SharedInboxURI)
-		if ok {
-			sent, ok = sentI.([][]byte)
-			if !ok {
-				panic("SentMessages entry was not []byte")
-			}
-			return true
+		delivery, ok := suite.state.Workers.Delivery.Queue.Pop()
+		if !ok {
+			return false
 		}
-		return false
+		if !testrig.EqualRequestURIs(delivery.Request.URL, *testRemoteAccount.SharedInboxURI) {
+			panic("differing request uris")
+		}
+		sent, err = io.ReadAll(delivery.Request.Body)
+		if err != nil {
+			panic("error reading body: " + err.Error())
+		}
+		return true
+
 	}) {
 		suite.FailNow("timed out waiting for message")
 	}
 
 	dst := new(bytes.Buffer)
-	err = json.Indent(dst, sent[0], "", "  ")
+	err = json.Indent(dst, sent, "", "  ")
 	suite.NoError(err)
 	suite.Equal(`{
   "@context": "https://www.w3.org/ns/activitystreams",

internal/federation/federatingdb/delete.go

@@ -51,7 +51,7 @@ func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error {
 	// in a delete we only get the URI, we can't know if we have a status or a profile or something else,
 	// so we have to try a few different things...
 	if s, err := f.state.DB.GetStatusByURI(ctx, id.String()); err == nil && requestingAcct.ID == s.AccountID {
-		l.Debugf("uri is for STATUS with id: %s", s.ID)
+		l.Debugf("deleting status: %s", s.ID)
 		f.state.Workers.EnqueueFediAPI(ctx, messages.FromFediAPI{
 			APObjectType:     ap.ObjectNote,
 			APActivityType:   ap.ActivityDelete,
@@ -61,7 +61,7 @@ func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error {
 	}
 
 	if a, err := f.state.DB.GetAccountByURI(ctx, id.String()); err == nil && requestingAcct.ID == a.ID {
-		l.Debugf("uri is for ACCOUNT with id %s", a.ID)
+		l.Debugf("deleting account: %s", a.ID)
 		f.state.Workers.EnqueueFediAPI(ctx, messages.FromFediAPI{
 			APObjectType:     ap.ObjectProfile,
 			APActivityType:   ap.ActivityDelete,

internal/federation/federatingprotocol.go

@@ -44,7 +44,7 @@ type errOtherIRIBlocked struct {
 	iriStrs     []string
 }
 
-func (e errOtherIRIBlocked) Error() string {
+func (e *errOtherIRIBlocked) Error() string {
 	iriStrsNice := "[" + strings.Join(e.iriStrs, ", ") + "]"
 	if e.domainBlock {
 		return "domain block exists for one or more of " + iriStrsNice
@@ -67,7 +67,7 @@ func newErrOtherIRIBlocked(
 		e.iriStrs = append(e.iriStrs, iri.String())
 	}
 
-	return e
+	return &e
 }
 
 /*

internal/federation/federatingprotocol_test.go

@@ -21,13 +21,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
 
+	errorsv2 "codeberg.org/gruf/go-errors/v2"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
@@ -101,8 +101,8 @@ func (suite *FederatingProtocolTestSuite) authenticatePostInbox(
 
 	recorder := httptest.NewRecorder()
 	newContext, authed, err := suite.federator.AuthenticatePostInbox(ctx, recorder, request)
-	if withCode := new(gtserror.WithCode); (errors.As(err, withCode) &&
-		(*withCode).Code() >= 500) || (err != nil && (*withCode) == nil) {
+	if withCode := errorsv2.AsV2[gtserror.WithCode](err); // nocollapse
+	(withCode != nil && withCode.Code() >= 500) || (err != nil && withCode == nil) {
 		// NOTE: the behaviour here is a little strange as we have
 		// the competing code styles of the go-fed interface expecting
 		// that any err is a no-go, but authed bool is intended to be