[feature] persist worker queues to db (#3042)

* persist queued worker tasks to database on shutdown, fill worker queues from database on startup

* ensure the tasks are sorted by creation time before pushing them

* add migration to insert WorkerTask{} into database, add test for worker task persistence

* add test for recovering worker queues from database

* quick tweak

* whoops we ended up with double cleaner job scheduling

* insert each task separately, because bun is throwing some reflection error??

* add specific checking of cancelled worker contexts

* add http request signing to deliveries recovered from database

* add test for outgoing public key ID being correctly set on delivery

* replace select with Queue.PopCtx()

* get rid of loop now we don't use it

* remove field now we don't use it

* ensure that signing func is set

* header values weren't being copied over 🤦

* use ptr for httpclient.Request in delivery

* move worker queue filling to later in server init process

* fix rebase issues

* make logging less shouty

* use slices.Delete() instead of copying / reslicing

* have database return tasks in ascending order instead of sorting them

* add a 1 minute timeout to persisting worker queues

internal/transport/deliver.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"io"
 	"net/http"
 	"net/url"
 
@@ -169,6 +170,38 @@ func (t *transport) prepare(
 	}, nil
 }
 
+func (t *transport) SignDelivery(dlv *delivery.Delivery) error {
+	if dlv.Request.GetBody == nil {
+		return gtserror.New("delivery request body not rewindable")
+	}
+
+	// Get a new copy of the request body.
+	body, err := dlv.Request.GetBody()
+	if err != nil {
+		return gtserror.Newf("error getting request body: %w", err)
+	}
+
+	// Read body data into memory.
+	data, err := io.ReadAll(body)
+	if err != nil {
+		return gtserror.Newf("error reading request body: %w", err)
+	}
+
+	// Get signing function for POST data.
+	// (note that delivery is ALWAYS POST).
+	sign := t.signPOST(data)
+
+	// Extract delivery context.
+	ctx := dlv.Request.Context()
+
+	// Update delivery request context with signing details.
+	ctx = gtscontext.SetOutgoingPublicKeyID(ctx, t.pubKeyID)
+	ctx = gtscontext.SetHTTPClientSignFunc(ctx, sign)
+	dlv.Request.Request = dlv.Request.Request.WithContext(ctx)
+
+	return nil
+}
+
 // getObjectID extracts an object ID from 'serialized' ActivityPub object map.
 func getObjectID(obj map[string]interface{}) string {
 	switch t := obj["object"].(type) {