[performance] processing media and scheduled jobs improvements (#1482)

* replace media workers with just runners.WorkerPool, move to state structure, use go-sched for global task scheduling

* improved code comment

* fix worker tryUntil function, update go-runners/go-sched

* make preprocess functions package public, use these where possible to stop doubled up processing

* remove separate emoji worker pool

* limit calls to time.Now() during media preprocessing

* use Processor{} to manage singular runtime of processing media

* ensure workers get started when media manager is used

* improved error setting in processing media, fix media test

* port changes from processingmedia to processing emoji

* finish code commenting

* finish code commenting and comment-out client API + federator worker pools until concurrency worker pools replaced

* linterrrrrrrrrrrrrrrr

---------

Signed-off-by: kim <grufwub@gmail.com>

cmd/gotosocial/action/admin/account/account.go

@@ -36,6 +36,7 @@ import (
 var Create action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 	state.Caches.Init()
+	state.Workers.Start()
 
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
@@ -97,6 +98,7 @@ var Create action.GTSAction = func(ctx context.Context) error {
 var Confirm action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 	state.Caches.Init()
+	state.Workers.Start()
 
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
@@ -140,6 +142,7 @@ var Confirm action.GTSAction = func(ctx context.Context) error {
 var Promote action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 	state.Caches.Init()
+	state.Workers.Start()
 
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
@@ -180,6 +183,7 @@ var Promote action.GTSAction = func(ctx context.Context) error {
 var Demote action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 	state.Caches.Init()
+	state.Workers.Start()
 
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
@@ -220,6 +224,7 @@ var Demote action.GTSAction = func(ctx context.Context) error {
 var Disable action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 	state.Caches.Init()
+	state.Workers.Start()
 
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
@@ -260,6 +265,7 @@ var Disable action.GTSAction = func(ctx context.Context) error {
 var Password action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 	state.Caches.Init()
+	state.Workers.Start()
 
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {

cmd/gotosocial/action/admin/media/prune/common.go

@@ -38,21 +38,24 @@ type prune struct {
 func setupPrune(ctx context.Context) (*prune, error) {
 	var state state.State
 	state.Caches.Init()
+	state.Workers.Start()
 
 	dbService, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbservice: %w", err)
 	}
 
-	storage, err := gtsstorage.AutoConfig() //nolint:contextcheck
+	//nolint:contextcheck
+	storage, err := gtsstorage.AutoConfig()
 	if err != nil {
 		return nil, fmt.Errorf("error creating storage backend: %w", err)
 	}
 
-	manager, err := media.NewManager(dbService, storage) //nolint:contextcheck
-	if err != nil {
-		return nil, fmt.Errorf("error instantiating mediamanager: %w", err)
-	}
+	state.DB = dbService
+	state.Storage = storage
+
+	//nolint:contextcheck
+	manager := media.NewManager(&state)
 
 	return &prune{
 		dbService: dbService,
@@ -70,9 +73,5 @@ func (p *prune) shutdown(ctx context.Context) error {
 		return fmt.Errorf("error closing dbservice: %w", err)
 	}
 
-	if err := p.manager.Stop(); err != nil {
-		return fmt.Errorf("error closing media manager: %w", err)
-	}
-
 	return nil
 }

cmd/gotosocial/action/server/server.go

@@ -91,33 +91,35 @@ var Start action.GTSAction = func(ctx context.Context) error {
 		return fmt.Errorf("error creating instance instance: %s", err)
 	}
 
-	// Create the client API and federator worker pools
-	// NOTE: these MUST NOT be used until they are passed to the
-	// processor and it is started. The reason being that the processor
-	// sets the Worker process functions and start the underlying pools
-	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
-	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
-
-	federatingDB := federatingdb.New(dbService, fedWorker)
-
-	// build converters and util
-	typeConverter := typeutils.NewConverter(dbService)
-
 	// Open the storage backend
 	storage, err := gtsstorage.AutoConfig()
 	if err != nil {
 		return fmt.Errorf("error creating storage backend: %w", err)
 	}
 
+	// Set the state storage driver
+	state.Storage = storage
+
 	// Build HTTP client (TODO: add configurables here)
 	client := httpclient.New(httpclient.Config{})
 
+	// Initialize workers.
+	state.Workers.Start()
+	defer state.Workers.Stop()
+
+	// Create the client API and federator worker pools
+	// NOTE: these MUST NOT be used until they are passed to the
+	// processor and it is started. The reason being that the processor
+	// sets the Worker process functions and start the underlying pools
+	// TODO: move these into state.Workers (and maybe reformat worker pools).
+	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
+	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
+
 	// build backend handlers
-	mediaManager, err := media.NewManager(dbService, storage)
-	if err != nil {
-		return fmt.Errorf("error creating media manager: %s", err)
-	}
+	mediaManager := media.NewManager(&state)
 	oauthServer := oauth.New(ctx, dbService)
+	typeConverter := typeutils.NewConverter(dbService)
+	federatingDB := federatingdb.New(dbService, fedWorker, typeConverter)
 	transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, client)
 	federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager)
 

cmd/gotosocial/main.go

@@ -20,9 +20,10 @@ package main
 
 import (
 	"log"
-	"runtime/debug"
+	godebug "runtime/debug"
 	"strings"
 
+	"codeberg.org/gruf/go-debug"
 	"github.com/spf13/cobra"
 
 	_ "github.com/superseriousbusiness/gotosocial/docs"
@@ -60,9 +61,12 @@ func main() {
 
 	// add subcommands
 	rootCmd.AddCommand(serverCommands())
-	rootCmd.AddCommand(testrigCommands())
 	rootCmd.AddCommand(debugCommands())
 	rootCmd.AddCommand(adminCommands())
+	if debug.DEBUG {
+		// only add testrig if debug enabled.
+		rootCmd.AddCommand(testrigCommands())
+	}
 
 	// run
 	if err := rootCmd.Execute(); err != nil {
@@ -73,7 +77,7 @@ func main() {
 // version will build a version string from binary's stored build information.
 func version() string {
 	// Read build information from binary
-	build, ok := debug.ReadBuildInfo()
+	build, ok := godebug.ReadBuildInfo()
 	if !ok {
 		return ""
 	}