[feature] Fetch + create domain permissions from subscriptions nightly (#3635)

* peepeepoopoo

* test domain perm subs

* swagger

* envparsing

* dries your wets

* start on docs

* finish up docs

* copy paste errors

* rename actions package

* rename force -> skipCache

* move obfuscate parse nearer to where err is checked

* make higherPrios a simple slice

* don't use receiver for permsFrom funcs

* add more context to error logs

* defer finished log

* use switch for permType instead of if/else

* thanks linter, love you <3

* validate csv headers before full read

* use bufio scanner

internal/subscriptions/domainperms.go

@@ -0,0 +1,811 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions
+
+import (
+	"bufio"
+	"context"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"codeberg.org/gruf/go-kv"
+
+	"github.com/miekg/dns"
+	"github.com/superseriousbusiness/gotosocial/internal/admin"
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/log"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// ScheduleJobs schedules domain permission subscription
+// fetching + updating using configured parameters.
+//
+// Returns an error if `MediaCleanupFrom`
+// is not a valid format (hh:mm:ss).
+func (s *Subscriptions) ScheduleJobs() error {
+	const hourMinute = "15:04"
+
+	var (
+		now            = time.Now()
+		processEvery   = config.GetInstanceSubscriptionsProcessEvery()
+		processFromStr = config.GetInstanceSubscriptionsProcessFrom()
+	)
+
+	// Parse processFromStr as hh:mm.
+	// Resulting time will be on 1 Jan year zero.
+	processFrom, err := time.Parse(hourMinute, processFromStr)
+	if err != nil {
+		return gtserror.Newf(
+			"error parsing '%s' in time format 'hh:mm': %w",
+			processFromStr, err,
+		)
+	}
+
+	// Time travel from
+	// year zero, groovy.
+	firstProcessAt := time.Date(
+		now.Year(),
+		now.Month(),
+		now.Day(),
+		processFrom.Hour(),
+		processFrom.Minute(),
+		0,
+		0,
+		now.Location(),
+	)
+
+	// Ensure first processing is in the future.
+	for firstProcessAt.Before(now) {
+		firstProcessAt = firstProcessAt.Add(processEvery)
+	}
+
+	fn := func(ctx context.Context, start time.Time) {
+		log.Info(ctx, "starting instance subscriptions processing")
+
+		// In blocklist (default) mode, process allows
+		// first to provide immunity to block side effects.
+		//
+		// In allowlist mode, process blocks first to
+		// ensure allowlist doesn't override blocks.
+		var order [2]gtsmodel.DomainPermissionType
+		if config.GetInstanceFederationMode() == config.InstanceFederationModeBlocklist {
+			order = [2]gtsmodel.DomainPermissionType{
+				gtsmodel.DomainPermissionAllow,
+				gtsmodel.DomainPermissionBlock,
+			}
+		} else {
+			order = [2]gtsmodel.DomainPermissionType{
+				gtsmodel.DomainPermissionBlock,
+				gtsmodel.DomainPermissionAllow,
+			}
+		}
+
+		// Fetch + process subscribed perms in order.
+		for _, permType := range order {
+			s.ProcessDomainPermissionSubscriptions(ctx, permType)
+		}
+
+		log.Infof(ctx, "finished instance subscriptions processing after %s", time.Since(start))
+	}
+
+	log.Infof(nil,
+		"scheduling instance subscriptions processing to run every %s, starting from %s; next processing will run at %s",
+		processEvery, processFromStr, firstProcessAt,
+	)
+
+	// Schedule processing to execute according to schedule.
+	if !s.state.Workers.Scheduler.AddRecurring(
+		"@subsprocessing",
+		firstProcessAt,
+		processEvery,
+		fn,
+	) {
+		panic("failed to schedule @subsprocessing")
+	}
+
+	return nil
+}
+
+// ProcessDomainPermissionSubscriptions processes all domain permission
+// subscriptions of the given permission type by, in turn, calling the
+// URI of each subscription, parsing the result into a list of domain
+// permissions, and creating (or skipping) each permission as appropriate.
+func (s *Subscriptions) ProcessDomainPermissionSubscriptions(
+	ctx context.Context,
+	permType gtsmodel.DomainPermissionType,
+) {
+	log.Info(ctx, "start")
+	defer log.Info(ctx, "finished")
+
+	// Get permission subscriptions in priority order (highest -> lowest).
+	permSubs, err := s.state.DB.GetDomainPermissionSubscriptionsByPriority(ctx, permType)
+	if err != nil && !errors.Is(err, db.ErrNoEntries) {
+		// Real db error.
+		log.Errorf(ctx, "db error getting domain perm subs by priority: %v", err)
+		return
+	}
+
+	if len(permSubs) == 0 {
+		// No subscriptions of this
+		// type, so nothing to do.
+		return
+	}
+
+	// Get a transport using the instance account,
+	// we can reuse this for each HTTP call.
+	tsport, err := s.transportController.NewTransportForUsername(ctx, "")
+	if err != nil {
+		log.Errorf(ctx, "error getting transport for instance account: %v", err)
+		return
+	}
+
+	for i, permSub := range permSubs {
+		// Higher priority permission subs = everything
+		// above this permission sub in the slice.
+		higherPrios := permSubs[:i]
+
+		_, err := s.ProcessDomainPermissionSubscription(
+			ctx,
+			permSub,
+			tsport,
+			higherPrios,
+			false, // Not dry. Wet, if you will.
+		)
+		if err != nil {
+			// Real db error.
+			log.Errorf(ctx,
+				"error processing domain permission subscription %s: %v",
+				permSub.URI, err,
+			)
+			return
+		}
+
+		// Update this perm sub.
+		err = s.state.DB.UpdateDomainPermissionSubscription(ctx, permSub)
+		if err != nil {
+			// Real db error.
+			log.Errorf(ctx, "db error updating domain perm sub: %v", err)
+			return
+		}
+	}
+}
+
+// ProcessDomainPermissionSubscription processes one domain permission
+// subscription by dereferencing the URI, parsing the response into a list
+// of permissions, and for each discovered permission either creating an
+// entry in the database, or ignoring it if it's excluded or already
+// covered by a higher-priority subscription.
+//
+// On success, the slice of discovered DomainPermissions will be returned.
+// In case of parsing error, or error on the remote side, permSub.Error
+// will be updated with the calling/parsing error, and `nil, nil` will be
+// returned. In case of an actual db error, `nil, err` will be returned and
+// the caller should handle it.
+//
+// getHigherPrios should be a function for returning a slice of domain
+// permission subscriptions with a higher priority than the given permSub.
+//
+// If dry == true, then the URI will still be called, and permissions
+// will be parsed, but they will not actually be created.
+//
+// Note that while this function modifies fields on the given permSub,
+// it's up to the caller to update it in the database (if desired).
+func (s *Subscriptions) ProcessDomainPermissionSubscription(
+	ctx context.Context,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	tsport transport.Transport,
+	higherPrios []*gtsmodel.DomainPermissionSubscription,
+	dry bool,
+) ([]gtsmodel.DomainPermission, error) {
+	l := log.
+		WithContext(ctx).
+		WithFields(kv.Fields{
+			{"permType", permSub.PermissionType.String()},
+			{"permSubURI", permSub.URI},
+		}...)
+
+	// Set FetchedAt as we're
+	// going to attempt this now.
+	permSub.FetchedAt = time.Now()
+
+	// Call the URI, and only skip
+	// cache if we're doing a dry run.
+	resp, err := tsport.DereferenceDomainPermissions(
+		ctx, permSub, dry,
+	)
+	if err != nil {
+		// Couldn't get this one,
+		// set error + return.
+		errStr := err.Error()
+		l.Warnf("couldn't dereference permSubURI: %+v", err)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	// If the permissions at URI weren't modified
+	// since last time, just update some metadata
+	// to indicate a successful fetch, and return.
+	if resp.Unmodified {
+		l.Debug("received 304 Not Modified from remote")
+		permSub.SuccessfullyFetchedAt = permSub.FetchedAt
+		if permSub.ETag == "" && resp.ETag != "" {
+			// We didn't have an ETag before but
+			// we have one now: probably the remote
+			// added ETag support in the meantime.
+			permSub.ETag = resp.ETag
+		}
+		return nil, nil
+	}
+
+	// At this point we know we got a 200 OK
+	// from the URI, so we've got a live body!
+	// Try to parse the body as a list of wantedPerms
+	// that the subscription wants to create.
+	var wantedPerms []gtsmodel.DomainPermission
+
+	switch permSub.ContentType {
+
+	// text/csv
+	case gtsmodel.DomainPermSubContentTypeCSV:
+		wantedPerms, err = permsFromCSV(l, permSub.PermissionType, resp.Body)
+
+	// application/json
+	case gtsmodel.DomainPermSubContentTypeJSON:
+		wantedPerms, err = permsFromJSON(l, permSub.PermissionType, resp.Body)
+
+	// text/plain
+	case gtsmodel.DomainPermSubContentTypePlain:
+		wantedPerms, err = permsFromPlain(l, permSub.PermissionType, resp.Body)
+	}
+
+	if err != nil {
+		// We retrieved the permissions from remote but
+		// the connection died halfway through transfer,
+		// or we couldn't parse the results, or something.
+		// Just set error and return.
+		errStr := err.Error()
+		l.Warnf("couldn't parse results: %+v", err)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	if len(wantedPerms) == 0 {
+		// Fetch was OK, and parsing was, on the surface at
+		// least, OK, but we didn't get any perms. Consider
+		// this an error as users will probably want to know.
+		const errStr = "fetch successful but parsed zero usable results"
+		l.Warn(errStr)
+		permSub.Error = errStr
+		return nil, nil
+	}
+
+	// This can now be considered a successful fetch.
+	permSub.SuccessfullyFetchedAt = permSub.FetchedAt
+	permSub.ETag = resp.ETag
+	permSub.Error = ""
+
+	// Keep track of which domain perms are
+	// created (or would be, if dry == true).
+	createdPerms := make([]gtsmodel.DomainPermission, 0, len(wantedPerms))
+
+	// Iterate through wantedPerms and
+	// create (or dry create) each one.
+	for _, wantedPerm := range wantedPerms {
+		l = l.WithField("domain", wantedPerm.GetDomain())
+		created, err := s.processDomainPermission(
+			ctx, l,
+			wantedPerm,
+			permSub,
+			higherPrios,
+			dry,
+		)
+		if err != nil {
+			// Proper db error.
+			return nil, err
+		}
+
+		if !created {
+			continue
+		}
+
+		createdPerms = append(createdPerms, wantedPerm)
+	}
+
+	return createdPerms, nil
+}
+
+// processDomainPermission processes one wanted domain
+// permission discovered via a domain permission sub's URI.
+//
+// Error will only be returned in case of an actual database
+// error, else the error will be logged and nil returned.
+func (s *Subscriptions) processDomainPermission(
+	ctx context.Context,
+	l log.Entry,
+	wantedPerm gtsmodel.DomainPermission,
+	permSub *gtsmodel.DomainPermissionSubscription,
+	higherPrios []*gtsmodel.DomainPermissionSubscription,
+	dry bool,
+) (bool, error) {
+	// Set to true if domain permission
+	// actually (would be) created.
+	var created bool
+
+	// If domain is excluded from automatic
+	// permission creation, don't process it.
+	domain := wantedPerm.GetDomain()
+	excluded, err := s.state.DB.IsDomainPermissionExcluded(ctx, domain)
+	if err != nil {
+		// Proper db error.
+		return created, err
+	}
+
+	if excluded {
+		l.Debug("domain is excluded, skipping")
+		return created, nil
+	}
+
+	// Check if a permission already exists for
+	// this domain, and if it's covered already
+	// by a higher-priority subscription.
+	existingPerm, covered, err := s.existingCovered(
+		ctx, permSub.PermissionType, domain, higherPrios,
+	)
+	if err != nil {
+		// Proper db error.
+		return created, err
+	}
+
+	if covered {
+		l.Debug("domain is covered by a higher-priority subscription, skipping")
+		return created, nil
+	}
+
+	// At this point we know we
+	// should create the perm.
+	created = true
+
+	if dry {
+		// Don't do creation or side
+		// effects if we're dry running.
+		return created, nil
+	}
+
+	// Handle perm creation differently depending
+	// on whether or not a perm already existed.
+	existing := !util.IsNil(existingPerm)
+	switch {
+
+	case !existing && *permSub.AsDraft:
+		// No existing perm, create as draft.
+		err = s.state.DB.PutDomainPermissionDraft(
+			ctx,
+			&gtsmodel.DomainPermissionDraft{
+				ID:                 id.NewULID(),
+				PermissionType:     permSub.PermissionType,
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			},
+		)
+
+	case !existing && !*permSub.AsDraft:
+		// No existing perm, create a new one of the
+		// appropriate type, and process side effects.
+		var (
+			insertF func() error
+			action  *gtsmodel.AdminAction
+			actionF admin.ActionF
+		)
+
+		if permSub.PermissionType == gtsmodel.DomainPermissionBlock {
+			// Prepare to insert + process a block.
+			domainBlock := &gtsmodel.DomainBlock{
+				ID:                 id.NewULID(),
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			}
+			insertF = func() error { return s.state.DB.CreateDomainBlock(ctx, domainBlock) }
+
+			action = &gtsmodel.AdminAction{
+				ID:             id.NewULID(),
+				TargetCategory: gtsmodel.AdminActionCategoryDomain,
+				TargetID:       domain,
+				Type:           gtsmodel.AdminActionSuspend,
+				AccountID:      permSub.CreatedByAccountID,
+			}
+			actionF = s.state.AdminActions.DomainBlockF(action.ID, domainBlock)
+
+		} else {
+			// Prepare to insert + process an allow.
+			domainAllow := &gtsmodel.DomainAllow{
+				ID:                 id.NewULID(),
+				Domain:             domain,
+				CreatedByAccountID: permSub.CreatedByAccount.ID,
+				CreatedByAccount:   permSub.CreatedByAccount,
+				PrivateComment:     permSub.URI,
+				PublicComment:      wantedPerm.GetPublicComment(),
+				Obfuscate:          wantedPerm.GetObfuscate(),
+				SubscriptionID:     permSub.ID,
+			}
+			insertF = func() error { return s.state.DB.CreateDomainAllow(ctx, domainAllow) }
+
+			action = &gtsmodel.AdminAction{
+				ID:             id.NewULID(),
+				TargetCategory: gtsmodel.AdminActionCategoryDomain,
+				TargetID:       domain,
+				Type:           gtsmodel.AdminActionUnsuspend,
+				AccountID:      permSub.CreatedByAccountID,
+			}
+			actionF = s.state.AdminActions.DomainAllowF(action.ID, domainAllow)
+		}
+
+		// Insert the new perm in the db.
+		if err = insertF(); err != nil {
+			// Couldn't insert wanted perm,
+			// don't process side effects.
+			break
+		}
+
+		// Run admin action to process
+		// side effects of permission.
+		err = s.state.AdminActions.Run(ctx, action, actionF)
+
+	case existingPerm.GetSubscriptionID() != "" || *permSub.AdoptOrphans:
+		// Perm exists but we should adopt/take
+		// it by copying over desired fields.
+		existingPerm.SetCreatedByAccountID(wantedPerm.GetCreatedByAccountID())
+		existingPerm.SetCreatedByAccount(wantedPerm.GetCreatedByAccount())
+		existingPerm.SetSubscriptionID(permSub.ID)
+		existingPerm.SetObfuscate(wantedPerm.GetObfuscate())
+		existingPerm.SetPrivateComment(wantedPerm.GetPrivateComment())
+		existingPerm.SetPublicComment(wantedPerm.GetPublicComment())
+
+		switch p := existingPerm.(type) {
+		case *gtsmodel.DomainBlock:
+			err = s.state.DB.UpdateDomainBlock(ctx, p)
+		case *gtsmodel.DomainAllow:
+			err = s.state.DB.UpdateDomainAllow(ctx, p)
+		}
+
+	default:
+		// Perm exists but we should leave it alone.
+		l.Debug("domain is covered by a higher-priority subscription, skipping")
+	}
+
+	if err != nil && !errors.Is(err, db.ErrAlreadyExists) {
+		// Proper db error.
+		return created, err
+	}
+
+	created = true
+	return created, nil
+}
+
+func permsFromCSV(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	csvReader := csv.NewReader(body)
+
+	// Read and validate column headers.
+	columnHeaders, err := csvReader.Read()
+	if err != nil {
+		body.Close()
+		return nil, gtserror.NewfAt(3, "error decoding csv column headers: %w", err)
+	}
+
+	if !slices.Equal(
+		columnHeaders,
+		[]string{
+			"#domain",
+			"#severity",
+			"#reject_media",
+			"#reject_reports",
+			"#public_comment",
+			"#obfuscate",
+		},
+	) {
+		body.Close()
+		err := gtserror.NewfAt(3, "unexpected column headers in csv: %+v", columnHeaders)
+		return nil, err
+	}
+
+	// Read remaining CSV records.
+	records, err := csvReader.ReadAll()
+
+	// Totally done
+	// with body now.
+	body.Close()
+
+	// Check for decode error.
+	if err != nil {
+		err := gtserror.NewfAt(3, "error decoding body into csv: %w", err)
+		return nil, err
+	}
+
+	// Make sure we actually
+	// have some records.
+	if len(records) == 0 {
+		return nil, nil
+	}
+
+	// Convert records to permissions slice.
+	perms := make([]gtsmodel.DomainPermission, 0, len(records))
+	for _, record := range records {
+		if len(record) != 6 {
+			l.Warnf("skipping invalid-length record: %+v", record)
+			continue
+		}
+
+		var (
+			domainRaw     = record[0]
+			severity      = record[1]
+			publicComment = record[4]
+			obfuscateStr  = record[5]
+		)
+
+		if severity != "suspend" {
+			l.Warnf("skipping non-suspend record: %+v", record)
+			continue
+		}
+
+		obfuscate, err := strconv.ParseBool(obfuscateStr)
+		if err != nil {
+			l.Warnf("couldn't parse obfuscate field of record: %+v", record)
+			continue
+		}
+
+		// Normalize + validate domain.
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		switch permType {
+		case gtsmodel.DomainPermissionBlock:
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		case gtsmodel.DomainPermissionAllow:
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// Set remaining fields.
+		perm.SetPublicComment(publicComment)
+		perm.SetObfuscate(&obfuscate)
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func permsFromJSON(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	var (
+		dec      = json.NewDecoder(body)
+		apiPerms = make([]*apimodel.DomainPermission, 0)
+	)
+
+	// Read body into memory as
+	// slice of domain permissions.
+	if err := dec.Decode(&apiPerms); err != nil {
+		_ = body.Close() // ensure closed.
+		return nil, gtserror.NewfAt(3, "error decoding into json: %w", err)
+	}
+
+	// Perform a secondary decode just to ensure we drained the
+	// entirety of the data source. Error indicates either extra
+	// trailing garbage, or multiple JSON values (invalid data).
+	if err := dec.Decode(&struct{}{}); err != io.EOF {
+		_ = body.Close() // ensure closed.
+		return nil, gtserror.NewfAt(3, "data remaining after json")
+	}
+
+	// Done with body.
+	_ = body.Close()
+
+	// Convert apimodel perms to barebones internal perms.
+	perms := make([]gtsmodel.DomainPermission, 0, len(apiPerms))
+	for _, apiPerm := range apiPerms {
+
+		// Normalize + validate domain.
+		domainRaw := apiPerm.Domain.Domain
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		switch permType {
+		case gtsmodel.DomainPermissionBlock:
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		case gtsmodel.DomainPermissionAllow:
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// Set remaining fields.
+		perm.SetPublicComment(apiPerm.PublicComment)
+		perm.SetObfuscate(&apiPerm.Obfuscate)
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func permsFromPlain(
+	l log.Entry,
+	permType gtsmodel.DomainPermissionType,
+	body io.ReadCloser,
+) ([]gtsmodel.DomainPermission, error) {
+	// Scan + split by line.
+	sc := bufio.NewScanner(body)
+
+	// Read into domains
+	// line by line.
+	var domains []string
+	for sc.Scan() {
+		domains = append(domains, sc.Text())
+	}
+
+	// Whatever happened, we're
+	// done with the body now.
+	body.Close()
+
+	// Check if error reading body.
+	if err := sc.Err(); err != nil {
+		return nil, gtserror.NewfAt(3, "error decoding into plain: %w", err)
+	}
+
+	// Convert raw domains to permissions.
+	perms := make([]gtsmodel.DomainPermission, 0, len(domains))
+	for _, domainRaw := range domains {
+
+		// Normalize + validate domain.
+		domain, err := validateDomain(domainRaw)
+		if err != nil {
+			l.Warnf("skipping invalid domain %s: %+v", domainRaw, err)
+			continue
+		}
+
+		// Instantiate the permission
+		// as either block or allow.
+		var perm gtsmodel.DomainPermission
+		switch permType {
+		case gtsmodel.DomainPermissionBlock:
+			perm = &gtsmodel.DomainBlock{Domain: domain}
+		case gtsmodel.DomainPermissionAllow:
+			perm = &gtsmodel.DomainAllow{Domain: domain}
+		}
+
+		// We're done.
+		perms = append(perms, perm)
+	}
+
+	return perms, nil
+}
+
+func validateDomain(domain string) (string, error) {
+	// Basic validation.
+	if _, ok := dns.IsDomainName(domain); !ok {
+		err := fmt.Errorf("invalid domain name")
+		return "", err
+	}
+
+	// Convert to punycode.
+	domain, err := util.Punify(domain)
+	if err != nil {
+		err := fmt.Errorf("could not punify domain: %w", err)
+		return "", err
+	}
+
+	// Check for invalid characters
+	// after the punification process.
+	if strings.ContainsAny(domain, "*, \n") {
+		err := fmt.Errorf("invalid char(s) in domain")
+		return "", err
+	}
+
+	return domain, nil
+}
+
+func (s *Subscriptions) existingCovered(
+	ctx context.Context,
+	permType gtsmodel.DomainPermissionType,
+	domain string,
+	higherPrios []*gtsmodel.DomainPermissionSubscription,
+) (
+	existingPerm gtsmodel.DomainPermission,
+	covered bool,
+	err error,
+) {
+	// Check for existing perm
+	// of appropriate type.
+	var dbErr error
+	switch permType {
+	case gtsmodel.DomainPermissionBlock:
+		existingPerm, dbErr = s.state.DB.GetDomainBlock(ctx, domain)
+	case gtsmodel.DomainPermissionAllow:
+		existingPerm, dbErr = s.state.DB.GetDomainAllow(ctx, domain)
+	}
+
+	if dbErr != nil && !errors.Is(dbErr, db.ErrNoEntries) {
+		// Real db error.
+		err = dbErr
+		return
+	}
+
+	if util.IsNil(existingPerm) {
+		// Can't be covered if
+		// no existing perm.
+		return
+	}
+
+	subscriptionID := existingPerm.GetSubscriptionID()
+	if subscriptionID == "" {
+		// Can't be covered if
+		// no subscription ID.
+		return
+	}
+
+	// Covered if subscription ID is in the slice
+	// of higher-priority permission subscriptions.
+	covered = slices.ContainsFunc(
+		higherPrios,
+		func(permSub *gtsmodel.DomainPermissionSubscription) bool {
+			return permSub.ID == subscriptionID
+		},
+	)
+
+	return
+}

internal/subscriptions/subscriptions.go

@@ -0,0 +1,42 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions
+
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/state"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+type Subscriptions struct {
+	state               *state.State
+	transportController transport.Controller
+	tc                  *typeutils.Converter
+}
+
+func New(
+	state *state.State,
+	transportController transport.Controller,
+	tc *typeutils.Converter,
+) *Subscriptions {
+	return &Subscriptions{
+		state:               state,
+		transportController: transportController,
+		tc:                  tc,
+	}
+}

internal/subscriptions/subscriptions_test.go

@@ -0,0 +1,538 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subscriptions_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+const (
+	rMediaPath    = "../../testrig/media"
+	rTemplatePath = "../../web/template"
+)
+
+type SubscriptionsTestSuite struct {
+	suite.Suite
+
+	testAccounts map[string]*gtsmodel.Account
+}
+
+func (suite *SubscriptionsTestSuite) SetupSuite() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
+	suite.testAccounts = testrig.NewTestAccounts()
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksCSV() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have some perms now.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("bigbums6969", permSub.ETag)
+	suite.EqualValues(3, count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksJSON() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a JSON list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.json",
+			ContentType:        gtsmodel.DomainPermSubContentTypeJSON,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have some perms now.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("don't modify me daddy", permSub.ETag)
+	suite.EqualValues(3, count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksPlain() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plain list of baddies.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.txt",
+			ContentType:        gtsmodel.DomainPermSubContentTypePlain,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now have blocks for
+	// each domain on the subscribed list.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		var (
+			perm gtsmodel.DomainPermission
+			err  error
+		)
+		if !testrig.WaitFor(func() bool {
+			perm, err = testStructs.State.DB.GetDomainBlock(ctx, domain)
+			return err == nil
+		}) {
+			suite.FailNowf("", "timed out waiting for domain %s", domain)
+		}
+
+		suite.Equal(testSubscription.ID, perm.GetSubscriptionID())
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have some perms now.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("this is a legit etag i swear", permSub.ETag)
+	suite.EqualValues(3, count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksCSVETag() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		// Include the ETag so it gets sent with the request.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+			ETag:               "bigbums6969",
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// We should now NOT have blocks for the domains
+	// on the list, as the remote will have returned
+	// 304, indicating we should do nothing.
+	for _, domain := range []string{
+		"bumfaces.net",
+		"peepee.poopoo",
+		"nothanks.com",
+	} {
+		_, err := testStructs.State.DB.GetDomainBlock(ctx, domain)
+		if !errors.Is(err, db.ErrNoEntries) {
+			suite.FailNowf("", "domain perm %s created when it shouldn't be")
+		}
+	}
+
+	// The just-fetched perm sub should
+	// have ETag and count etc set now.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have no perms.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Equal("bigbums6969", permSub.ETag)
+	suite.Zero(count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.WithinDuration(time.Now(), permSub.SuccessfullyFetchedAt, 1*time.Minute)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocks404() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a CSV list of baddies.
+		// The endpoint will return a 404 so we can test erroring.
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/does_not_exist.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have no perms.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Zero(count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`DereferenceDomainPermissions: GET request to https://lists.example.org/does_not_exist.csv failed: status="" body="{"error":"not found"}"`, permSub.Error)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypeCSV() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plaintext list of baddies,
+		// but try to parse as CSV content type (shouldn't work).
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.txt",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have no perms.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Zero(count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`ProcessDomainPermissionSubscription: unexpected column headers in csv: [bumfaces.net]`, permSub.Error)
+}
+
+func (suite *SubscriptionsTestSuite) TestDomainBlocksWrongContentTypePlain() {
+	var (
+		ctx           = context.Background()
+		testStructs   = testrig.SetupTestStructs(rMediaPath, rTemplatePath)
+		testAccount   = suite.testAccounts["admin_account"]
+		subscriptions = subscriptions.New(
+			testStructs.State,
+			testStructs.TransportController,
+			testStructs.TypeConverter,
+		)
+
+		// Create a subscription for a plaintext list of baddies,
+		// but try to parse as CSV content type (shouldn't work).
+		testSubscription = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypePlain,
+		}
+	)
+	defer testrig.TearDownTestStructs(testStructs)
+
+	// Store test subscription.
+	if err := testStructs.State.DB.PutDomainPermissionSubscription(
+		ctx, testSubscription,
+	); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Process all subscriptions.
+	subscriptions.ProcessDomainPermissionSubscriptions(ctx, testSubscription.PermissionType)
+
+	// The just-fetched perm sub should have an error set on it.
+	permSub, err := testStructs.State.DB.GetDomainPermissionSubscriptionByID(
+		ctx, testSubscription.ID,
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Should have no perms.
+	count, err := testStructs.State.DB.CountDomainPermissionSubscriptionPerms(ctx, permSub.ID)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	suite.Zero(count)
+	suite.WithinDuration(time.Now(), permSub.FetchedAt, 1*time.Minute)
+	suite.Zero(permSub.SuccessfullyFetchedAt)
+	suite.Equal(`fetch successful but parsed zero usable results`, permSub.Error)
+}
+
+func TestSubscriptionTestSuite(t *testing.T) {
+	suite.Run(t, new(SubscriptionsTestSuite))
+}