[performance] massively improved ActivityPub delivery worker efficiency (#2812)

* add delivery worker type that pulls from queue to httpclient package

* finish up some code commenting, bodge a vendored activity library change, integrate the deliverypool changes into transportcontroller

* hook up queue deletion logic

* support deleting queued http requests by target ID

* don't index APRequest by hostname in the queue

* use gorun

* use the original context's values when wrapping msg type as delivery{}

* actually log in the AP delivery worker ...

* add uncommitted changes

* use errors.AsV2()

* use errorsv2.AsV2()

* finish adding some code comments, add bad host handling to delivery workers

* slightly tweak deliveryworkerpool API, use advanced sender multiplier

* remove PopCtx() method, let others instead rely on Wait()

* shuffle things around to move delivery stuff into transport/ subpkg

* remove dead code

* formatting

* validate request before queueing for delivery

* finish adding code comments, fix up backoff code

* finish adding more code comments

* clamp minimum no. senders to 1

* add start/stop logging to delivery worker, some slight changes

* remove double logging

* use worker ptrs

* expose the embedded log fields in httpclient.Request{}

* ensure request context values are preserved when updating ctx

* add delivery worker tests

* fix linter issues

* ensure delivery worker gets inited in testrig

* fix tests to delivering messages to check worker delivery queue

* update error type to use ptr instead of value receiver

* fix test calling Workers{}.Start() instead of testrig.StartWorkers()

* update docs for advanced-sender-multiplier

* update to the latest activity library version

* add comment about not using httptest.Server{}

vendor/github.com/superseriousbusiness/activity/pub/side_effect_actor.go

@@ -2,7 +2,6 @@ package pub
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -477,17 +476,12 @@ func (a *SideEffectActor) deliverToRecipients(c context.Context, boxIRI *url.URL
 		return err
 	}
 
-	b, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-
 	tp, err := a.common.NewTransport(c, boxIRI, goFedUserAgent())
 	if err != nil {
 		return err
 	}
 
-	return tp.BatchDeliver(c, b, recipients)
+	return tp.BatchDeliver(c, m, recipients)
 }
 
 // addToOutbox adds the activity to the outbox and creates the activity in the

vendor/github.com/superseriousbusiness/activity/pub/transport.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -44,10 +45,10 @@ type Transport interface {
 	Dereference(c context.Context, iri *url.URL) (*http.Response, error)
 
 	// Deliver sends an ActivityStreams object.
-	Deliver(c context.Context, b []byte, to *url.URL) error
+	Deliver(c context.Context, obj map[string]interface{}, to *url.URL) error
 
 	// BatchDeliver sends an ActivityStreams object to multiple recipients.
-	BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error
+	BatchDeliver(c context.Context, obj map[string]interface{}, recipients []*url.URL) error
 }
 
 // Transport must be implemented by HttpSigTransport.
@@ -138,7 +139,49 @@ func (h HttpSigTransport) Dereference(c context.Context, iri *url.URL) (*http.Re
 }
 
 // Deliver sends a POST request with an HTTP Signature.
-func (h HttpSigTransport) Deliver(c context.Context, b []byte, to *url.URL) error {
+func (h HttpSigTransport) Deliver(c context.Context, data map[string]interface{}, to *url.URL) error {
+	b, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	return h.deliver(c, b, to)
+}
+
+// BatchDeliver sends concurrent POST requests. Returns an error if any of the requests had an error.
+func (h HttpSigTransport) BatchDeliver(c context.Context, data map[string]interface{}, recipients []*url.URL) error {
+	b, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	var wg sync.WaitGroup
+	errCh := make(chan error, len(recipients))
+	for _, recipient := range recipients {
+		wg.Add(1)
+		go func(r *url.URL) {
+			defer wg.Done()
+			if err := h.deliver(c, b, r); err != nil {
+				errCh <- err
+			}
+		}(recipient)
+	}
+	wg.Wait()
+	errs := make([]string, 0, len(recipients))
+outer:
+	for {
+		select {
+		case e := <-errCh:
+			errs = append(errs, e.Error())
+		default:
+			break outer
+		}
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf("batch deliver had at least one failure: %s", strings.Join(errs, "; "))
+	}
+	return nil
+}
+
+func (h HttpSigTransport) deliver(c context.Context, b []byte, to *url.URL) error {
 	req, err := http.NewRequest("POST", to.String(), bytes.NewReader(b))
 	if err != nil {
 		return err
@@ -166,36 +209,6 @@ func (h HttpSigTransport) Deliver(c context.Context, b []byte, to *url.URL) erro
 	return nil
 }
 
-// BatchDeliver sends concurrent POST requests. Returns an error if any of the requests had an error.
-func (h HttpSigTransport) BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error {
-	var wg sync.WaitGroup
-	errCh := make(chan error, len(recipients))
-	for _, recipient := range recipients {
-		wg.Add(1)
-		go func(r *url.URL) {
-			defer wg.Done()
-			if err := h.Deliver(c, b, r); err != nil {
-				errCh <- err
-			}
-		}(recipient)
-	}
-	wg.Wait()
-	errs := make([]string, 0, len(recipients))
-outer:
-	for {
-		select {
-		case e := <-errCh:
-			errs = append(errs, e.Error())
-		default:
-			break outer
-		}
-	}
-	if len(errs) > 0 {
-		return fmt.Errorf("batch deliver had at least one failure: %s", strings.Join(errs, "; "))
-	}
-	return nil
-}
-
 // HttpClient sends http requests, and is an abstraction only needed by the
 // HttpSigTransport. The standard library's Client satisfies this interface.
 type HttpClient interface {