[feature] Fetch + create domain permissions from subscriptions nightly (#3635)

* peepeepoopoo

* test domain perm subs

* swagger

* envparsing

* dries your wets

* start on docs

* finish up docs

* copy paste errors

* rename actions package

* rename force -> skipCache

* move obfuscate parse nearer to where err is checked

* make higherPrios a simple slice

* don't use receiver for permsFrom funcs

* add more context to error logs

* defer finished log

* use switch for permType instead of if/else

* thanks linter, love you <3

* validate csv headers before full read

* use bufio scanner

internal/api/client/admin/admin.go

@@ -46,6 +46,7 @@ const (
 	DomainPermissionSubscriptionsPathWithID  = DomainPermissionSubscriptionsPath + "/:" + apiutil.IDKey
 	DomainPermissionSubscriptionsPreviewPath = DomainPermissionSubscriptionsPath + "/preview"
 	DomainPermissionSubscriptionRemovePath   = DomainPermissionSubscriptionsPathWithID + "/remove"
+	DomainPermissionSubscriptionTestPath     = DomainPermissionSubscriptionsPathWithID + "/test"
 	DomainKeysExpirePath                     = BasePath + "/domain_keys_expire"
 	HeaderAllowsPath                         = BasePath + "/header_allows"
 	HeaderAllowsPathWithID                   = HeaderAllowsPath + "/:" + apiutil.IDKey
@@ -129,6 +130,7 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
 	attachHandler(http.MethodGet, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionGETHandler)
 	attachHandler(http.MethodPatch, DomainPermissionSubscriptionsPathWithID, m.DomainPermissionSubscriptionPATCHHandler)
 	attachHandler(http.MethodPost, DomainPermissionSubscriptionRemovePath, m.DomainPermissionSubscriptionRemovePOSTHandler)
+	attachHandler(http.MethodPost, DomainPermissionSubscriptionTestPath, m.DomainPermissionSubscriptionTestPOSTHandler)
 
 	// header filtering administration routes
 	attachHandler(http.MethodGet, HeaderAllowsPathWithID, m.HeaderFilterAllowGET)

internal/api/client/admin/admin_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"
+	adminactions "github.com/superseriousbusiness/gotosocial/internal/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -91,6 +92,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
 
 	suite.db = testrig.NewTestDB(&suite.state)
 	suite.state.DB = suite.db
+	suite.state.AdminActions = adminactions.New(suite.state.DB, &suite.state.Workers)
 	suite.storage = testrig.NewInMemoryStorage()
 	suite.state.Storage = suite.storage
 

internal/api/client/admin/domainpermissionsubscriptiontest.go

@@ -0,0 +1,118 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// DomainPermissionSubscriptionTestPOSTHandler swagger:operation POST /api/v1/admin/domain_permission_subscriptions/{id}/test domainPermissionSubscriptionTest
+//
+// Test one domain permission subscription by making your instance fetch and parse it *without creating permissions*.
+//
+// The response body will be a list of domain permissions that *would* be created by this subscription, OR an error message.
+//
+// This is useful in cases where you want to check that your instance can actually fetch + parse a list.
+//
+//	---
+//	tags:
+//	- admin
+//
+//	produces:
+//	- application/json
+//
+//	parameters:
+//	-
+//		name: id
+//		required: true
+//		in: path
+//		description: ID of the domain permission draft.
+//		type: string
+//
+//	security:
+//	- OAuth2 Bearer:
+//		- admin
+//
+//	responses:
+//		'200':
+//			description: >-
+//				Either an array of domain permissions, OR an error message of the form
+//				`{"error":"[ERROR MESSAGE HERE]"}` indicating why the list could not be fetched.
+//			schema:
+//				type: array
+//				items:
+//					"$ref": "#/definitions/domain"
+//		'400':
+//			description: bad request
+//		'401':
+//			description: unauthorized
+//		'403':
+//			description: forbidden
+//		'406':
+//			description: not acceptable
+//		'409':
+//			description: conflict
+//		'500':
+//			description: internal server error
+func (m *Module) DomainPermissionSubscriptionTestPOSTHandler(c *gin.Context) {
+	authed, err := oauth.Authed(c, true, true, true, true)
+	if err != nil {
+		apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	if !*authed.User.Admin {
+		err := fmt.Errorf("user %s not an admin", authed.User.ID)
+		apiutil.ErrorHandler(c, gtserror.NewErrorForbidden(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	if authed.Account.IsMoving() {
+		apiutil.ForbiddenAfterMove(c)
+		return
+	}
+
+	if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil {
+		apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1)
+		return
+	}
+
+	id, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey))
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	resp, errWithCode := m.processor.Admin().DomainPermissionSubscriptionTest(
+		c.Request.Context(),
+		authed.Account,
+		id,
+	)
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
+		return
+	}
+
+	apiutil.JSON(c, http.StatusOK, resp)
+}

internal/api/client/admin/domainpermissionsubscruptiontest_test.go

@@ -0,0 +1,125 @@
+// GoToSocial
+// Copyright (C) GoToSocial Authors admin@gotosocial.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package admin_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
+	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+type DomainPermissionSubscriptionTestTestSuite struct {
+	AdminStandardTestSuite
+}
+
+func (suite *DomainPermissionSubscriptionTestTestSuite) TestDomainPermissionSubscriptionTest() {
+	var (
+		ctx         = context.Background()
+		testAccount = suite.testAccounts["admin_account"]
+		permSub     = &gtsmodel.DomainPermissionSubscription{
+			ID:                 "01JGE681TQSBPAV59GZXPKE62H",
+			Priority:           255,
+			Title:              "whatever!",
+			PermissionType:     gtsmodel.DomainPermissionBlock,
+			AsDraft:            util.Ptr(false),
+			AdoptOrphans:       util.Ptr(true),
+			CreatedByAccountID: testAccount.ID,
+			CreatedByAccount:   testAccount,
+			URI:                "https://lists.example.org/baddies.csv",
+			ContentType:        gtsmodel.DomainPermSubContentTypeCSV,
+		}
+	)
+
+	// Create a subscription for a CSV list of baddies.
+	err := suite.state.DB.PutDomainPermissionSubscription(ctx, permSub)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Prepare the request to the /test endpoint.
+	subPath := strings.ReplaceAll(
+		admin.DomainPermissionSubscriptionTestPath,
+		":id", permSub.ID,
+	)
+	path := "/api" + subPath
+	recorder := httptest.NewRecorder()
+	ginCtx := suite.newContext(recorder, http.MethodPost, nil, path, "application/json")
+	ginCtx.Params = gin.Params{
+		gin.Param{
+			Key:   apiutil.IDKey,
+			Value: permSub.ID,
+		},
+	}
+
+	// Trigger the handler.
+	suite.adminModule.DomainPermissionSubscriptionTestPOSTHandler(ginCtx)
+	suite.Equal(http.StatusOK, recorder.Code)
+
+	// Read the body back.
+	b, err := io.ReadAll(recorder.Body)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	dst := new(bytes.Buffer)
+	if err := json.Indent(dst, b, "", "  "); err != nil {
+		suite.FailNow(err.Error())
+	}
+
+	// Ensure expected.
+	suite.Equal(`[
+  {
+    "domain": "bumfaces.net",
+    "public_comment": "big jerks"
+  },
+  {
+    "domain": "peepee.poopoo",
+    "public_comment": "harassment"
+  },
+  {
+    "domain": "nothanks.com"
+  }
+]`, dst.String())
+
+	// No permissions should be created
+	// since this is a dry run / test.
+	blocked, err := suite.state.DB.AreDomainsBlocked(
+		ctx,
+		[]string{"bumfaces.net", "peepee.poopoo", "nothanks.com"},
+	)
+	if err != nil {
+		suite.FailNow(err.Error())
+	}
+	suite.False(blocked)
+}
+
+func TestDomainPermissionSubscriptionTestTestSuite(t *testing.T) {
+	suite.Run(t, &DomainPermissionSubscriptionTestTestSuite{})
+}