[feature] Fetch + create domain permissions from subscriptions nightly (#3635)

* peepeepoopoo

* test domain perm subs

* swagger

* envparsing

* dries your wets

* start on docs

* finish up docs

* copy paste errors

* rename actions package

* rename force -> skipCache

* move obfuscate parse nearer to where err is checked

* make higherPrios a simple slice

* don't use receiver for permsFrom funcs

* add more context to error logs

* defer finished log

* use switch for permType instead of if/else

* thanks linter, love you <3

* validate csv headers before full read

* use bufio scanner

cmd/gotosocial/action/admin/account/account.go

@@ -40,7 +40,8 @@ func initState(ctx context.Context) (*state.State, error) {
 	state.Caches.Init()
 	state.Caches.Start()
 
-	// Set the state DB connection
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this (yet).
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbConn: %w", err)

cmd/gotosocial/action/admin/media/list.go

@@ -127,6 +127,8 @@ func setupList(ctx context.Context) (*list, error) {
 	state.Caches.Init()
 	state.Caches.Start()
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbService, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbservice: %w", err)

cmd/gotosocial/action/admin/media/prune/common.go

@@ -45,10 +45,12 @@ func setupPrune(ctx context.Context) (*prune, error) {
 	state.Caches.Start()
 
 	// Scheduler is required for the
-	// claner, but no other workers
+	// cleaner, but no other workers
 	// are needed for this CLI action.
 	state.Workers.StartScheduler()
 
+	// Set state DB connection.
+	// Don't need Actions for this.
 	dbService, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbservice: %w", err)

cmd/gotosocial/action/admin/trans/export.go

@@ -33,12 +33,12 @@ import (
 var Export action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
-
-	// Set the state DB connection
 	state.DB = dbConn
 
 	exporter := trans.NewExporter(dbConn)

cmd/gotosocial/action/admin/trans/import.go

@@ -33,12 +33,12 @@ import (
 var Import action.GTSAction = func(ctx context.Context) error {
 	var state state.State
 
+	// Only set state DB connection.
+	// Don't need Actions or Workers for this.
 	dbConn, err := bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
-
-	// Set the state DB connection
 	state.DB = dbConn
 
 	importer := trans.NewImporter(dbConn)

cmd/gotosocial/action/server/server.go

@@ -32,6 +32,7 @@ import (
 	"github.com/KimMachineGun/automemlimit/memlimit"
 	"github.com/gin-gonic/gin"
 	"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api"
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
 	"github.com/superseriousbusiness/gotosocial/internal/cleaner"
@@ -44,6 +45,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/metrics"
 	"github.com/superseriousbusiness/gotosocial/internal/middleware"
 	tlprocessor "github.com/superseriousbusiness/gotosocial/internal/processing/timeline"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/tracing"
 	"go.uber.org/automaxprocs/maxprocs"
@@ -164,6 +166,10 @@ var Start action.GTSAction = func(ctx context.Context) error {
 	// Set DB on state.
 	state.DB = dbService
 
+	// Set Actions on state, providing workers to
+	// Actions as well for triggering side effects.
+	state.AdminActions = admin.New(dbService, &state.Workers)
+
 	// Ensure necessary database instance prerequisites exist.
 	if err := dbService.CreateInstanceAccount(ctx); err != nil {
 		return fmt.Errorf("error creating instance account: %s", err)
@@ -283,15 +289,18 @@ var Start action.GTSAction = func(ctx context.Context) error {
 	// Create background cleaner.
 	cleaner := cleaner.New(state)
 
-	// Now schedule background cleaning tasks.
-	if err := cleaner.ScheduleJobs(); err != nil {
-		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
-	}
+	// Create subscriptions fetcher.
+	subscriptions := subscriptions.New(
+		state,
+		transportController,
+		typeConverter,
+	)
 
 	// Create the processor using all the
 	// other services we've created so far.
 	process = processing.NewProcessor(
 		cleaner,
+		subscriptions,
 		typeConverter,
 		federator,
 		oauthServer,
@@ -302,6 +311,16 @@ var Start action.GTSAction = func(ctx context.Context) error {
 		intFilter,
 	)
 
+	// Schedule background cleaning tasks.
+	if err := cleaner.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
+	}
+
+	// Schedule background subscriptions updating.
+	if err := subscriptions.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling subscriptions jobs: %w", err)
+	}
+
 	// Initialize the specialized workers pools.
 	state.Workers.Client.Init(messages.ClientMsgIndices())
 	state.Workers.Federator.Init(messages.FederatorMsgIndices())

cmd/gotosocial/action/testrig/testrig.go

@@ -20,11 +20,9 @@
 package testrig
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
 	"os/signal"
@@ -47,6 +45,7 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/router"
 	"github.com/superseriousbusiness/gotosocial/internal/state"
 	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
 	"github.com/superseriousbusiness/gotosocial/internal/timeline"
 	"github.com/superseriousbusiness/gotosocial/internal/tracing"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@@ -159,16 +158,8 @@ var Start action.GTSAction = func(ctx context.Context) error {
 	testrig.StandardStorageSetup(state.Storage, "./testrig/media")
 
 	// build backend handlers
-	transportController := testrig.NewTestTransportController(state, testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
-		r := io.NopCloser(bytes.NewReader([]byte{}))
-		return &http.Response{
-			StatusCode: 200,
-			Body:       r,
-			Header: http.Header{
-				"Content-Type": req.Header.Values("Accept"),
-			},
-		}, nil
-	}, ""))
+	httpClient := testrig.NewMockHTTPClient(nil, "./testrig/media")
+	transportController := testrig.NewTestTransportController(state, httpClient)
 	mediaManager := testrig.NewTestMediaManager(state)
 	federator := testrig.NewTestFederator(state, transportController, mediaManager)
 
@@ -314,11 +305,23 @@ var Start action.GTSAction = func(ctx context.Context) error {
 	// Create background cleaner.
 	cleaner := cleaner.New(state)
 
-	// Now schedule background cleaning tasks.
+	// Schedule background cleaning tasks.
 	if err := cleaner.ScheduleJobs(); err != nil {
 		return fmt.Errorf("error scheduling cleaner jobs: %w", err)
 	}
 
+	// Create subscriptions fetcher.
+	subscriptions := subscriptions.New(
+		state,
+		transportController,
+		typeConverter,
+	)
+
+	// Schedule background subscriptions updating.
+	if err := subscriptions.ScheduleJobs(); err != nil {
+		return fmt.Errorf("error scheduling subscriptions jobs: %w", err)
+	}
+
 	// Finally start the main http server!
 	if err := route.Start(); err != nil {
 		return fmt.Errorf("error starting router: %w", err)