[performance] update remaining worker pools to use queues (#2865)

* start replacing client + federator + media workers with new worker + queue types

* refactor federatingDB.Delete(), drop queued messages when deleting account / status

* move all queue purging to the processor workers

* undo toolchain updates

* code comments, ensure dereferencer worker pool gets started

* update gruf libraries in readme

* start the job scheduler separately to the worker pools

* reshuffle ordering or server.go + remove duplicate worker start / stop

* update go-list version

* fix vendoring

* move queue invalidation to before wipeing / deletion, to ensure queued work not dropped

* add logging to worker processing functions in testrig, don't start workers in unexpected places

* update go-structr to add (+then rely on) QueueCtx{} type

* ensure more worker pools get started properly in tests

* fix remaining broken tests relying on worker queue logic

* fix account test suite queue popping logic, ensure noop workers do not pull from queue

* move back accidentally shuffled account deletion order

* ensure error (non nil!!) gets passed in refactored federatingDB{}.Delete()

* silently drop deletes from accounts not permitted to

* don't warn log on forwarded deletes

* make if else clauses easier to parse

* use getFederatorMsg()

* improved code comment

* improved code comment re: requesting account delete checks

* remove boolean result from worker start / stop since false = already running or already stopped

* remove optional passed-in http.client

* remove worker starting from the admin CLI commands (we don't need to handle side-effects)

* update prune cli to start scheduler but not all of the workers

* fix rebase issues

* remove redundant return statements

* i'm sorry sir linter

internal/messages/messages.go

@@ -20,25 +20,84 @@ package messages
 import (
 	"net/url"
 
+	"codeberg.org/gruf/go-structr"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-// FromClientAPI wraps a message that travels from the client API into the processor.
+// FromClientAPI wraps a message that
+// travels from the client API into the processor.
 type FromClientAPI struct {
-	APObjectType   string
+
+	// APObjectType ...
+	APObjectType string
+
+	// APActivityType ...
 	APActivityType string
-	GTSModel       interface{}
-	OriginAccount  *gtsmodel.Account
-	TargetAccount  *gtsmodel.Account
+
+	// Optional GTS database model
+	// of the Activity / Object.
+	GTSModel interface{}
+
+	// Targeted object URI.
+	TargetURI string
+
+	// Origin is the account that
+	// this message originated from.
+	Origin *gtsmodel.Account
+
+	// Target is the account that
+	// this message is targeting.
+	Target *gtsmodel.Account
 }
 
-// FromFediAPI wraps a message that travels from the federating API into the processor.
-type FromFediAPI struct {
-	APObjectType      string
-	APActivityType    string
-	APIri             *url.URL
-	APObjectModel     interface{}       // Optional AP model of the Object of the Activity. Should be Accountable or Statusable.
-	GTSModel          interface{}       // Optional GTS model of the Activity or Object.
-	RequestingAccount *gtsmodel.Account // Remote account that posted this Activity to the inbox.
-	ReceivingAccount  *gtsmodel.Account // Local account which owns the inbox that this Activity was posted to.
+// ClientMsgIndices defines queue indices this
+// message type should be accessible / stored under.
+func ClientMsgIndices() []structr.IndexConfig {
+	return []structr.IndexConfig{
+		{Fields: "TargetURI", Multiple: true},
+		{Fields: "Origin.ID", Multiple: true},
+		{Fields: "Target.ID", Multiple: true},
+	}
+}
+
+// FromFediAPI wraps a message that
+// travels from the federating API into the processor.
+type FromFediAPI struct {
+
+	// APObjectType ...
+	APObjectType string
+
+	// APActivityType ...
+	APActivityType string
+
+	// Optional ActivityPub ID (IRI)
+	// and / or model of Activity / Object.
+	APIRI    *url.URL
+	APObject interface{}
+
+	// Optional GTS database model
+	// of the Activity / Object.
+	GTSModel interface{}
+
+	// Targeted object URI.
+	TargetURI string
+
+	// Remote account that posted
+	// this Activity to the inbox.
+	Requesting *gtsmodel.Account
+
+	// Local account which owns the inbox
+	// that this Activity was posted to.
+	Receiving *gtsmodel.Account
+}
+
+// FederatorMsgIndices defines queue indices this
+// message type should be accessible / stored under.
+func FederatorMsgIndices() []structr.IndexConfig {
+	return []structr.IndexConfig{
+		{Fields: "APIRI", Multiple: true},
+		{Fields: "TargetURI", Multiple: true},
+		{Fields: "Requesting.ID", Multiple: true},
+		{Fields: "Receiving.ID", Multiple: true},
+	}
 }