mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 01:32:25 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Fetch + create domain permissions from subscriptions nightly (#3635)

Browse source 
* peepeepoopoo

* test domain perm subs

* swagger

* envparsing

* dries your wets

* start on docs

* finish up docs

* copy paste errors

* rename actions package

* rename force -> skipCache

* move obfuscate parse nearer to where err is checked

* make higherPrios a simple slice

* don't use receiver for permsFrom funcs

* add more context to error logs

* defer finished log

* use switch for permType instead of if/else

* thanks linter, love you <3

* validate csv headers before full read

* use bufio scanner
This commit is contained in:
tobi 2025-01-08 11:29:40 +01:00 • committed by GitHub
commit 451803b230
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
95 changed files with 3320 additions and 626 deletions
Download patch file Download diff file Expand all files Collapse all files

2
testrig/config.go
View file

@ -99,6 +99,8 @@ func testDefaults() config.Configuration {
TagStr: "en-gb",
},
},
InstanceSubscriptionsProcessFrom: "23:00", // 11pm,
InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
AccountsRegistrationOpen: true,
AccountsReasonRequired: true,

14
testrig/processor.go
View file

@ -26,15 +26,27 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
// NewTestProcessor returns a Processor suitable for testing purposes.
// The passed in state will have its worker functions set appropriately,
// but the state will not be initialized.
func NewTestProcessor(state *state.State, federator *federation.Federator, emailSender email.Sender, mediaManager *media.Manager) *processing.Processor {
func NewTestProcessor(
state *state.State,
federator *federation.Federator,
emailSender email.Sender,
mediaManager *media.Manager,
) *processing.Processor {
return processing.NewProcessor(
cleaner.New(state),
subscriptions.New(
state,
federator.TransportController(),
typeutils.NewConverter(state),
),
typeutils.NewConverter(state),
federator,
NewTestOauthServer(state.DB),

31
testrig/teststructs.go
View file

@ -18,6 +18,7 @@
package testrig
import (
"github.com/superseriousbusiness/gotosocial/internal/admin"
"github.com/superseriousbusiness/gotosocial/internal/cleaner"
"github.com/superseriousbusiness/gotosocial/internal/email"
"github.com/superseriousbusiness/gotosocial/internal/filter/interaction"
@ -25,6 +26,8 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/processing/common"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/subscriptions"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@ -38,12 +41,13 @@ import (
// and worker queues, which was causing issues
// when running all tests at once.
type TestStructs struct {
State *state.State
Common *common.Processor
Processor *processing.Processor
HTTPClient *MockHTTPClient
TypeConverter *typeutils.Converter
EmailSender email.Sender
State *state.State
Common *common.Processor
Processor *processing.Processor
HTTPClient *MockHTTPClient
TypeConverter *typeutils.Converter
EmailSender email.Sender
TransportController transport.Controller
}
func SetupTestStructs(
@ -56,6 +60,7 @@ func SetupTestStructs(
db := NewTestDB(&state)
state.DB = db
state.AdminActions = admin.New(db, &state.Workers)
storage := NewInMemoryStorage()
state.Storage = storage
@ -89,6 +94,7 @@ func SetupTestStructs(
processor := processing.NewProcessor(
cleaner.New(&state),
subscriptions.New(&state, transportController, typeconverter),
typeconverter,
federator,
oauthServer,
@ -105,12 +111,13 @@ func SetupTestStructs(
StandardStorageSetup(storage, rMediaPath)
return &TestStructs{
State: &state,
Common: &common,
Processor: processor,
HTTPClient: httpClient,
TypeConverter: typeconverter,
EmailSender: emailSender,
State: &state,
Common: &common,
Processor: processor,
HTTPClient: httpClient,
TypeConverter: typeconverter,
EmailSender: emailSender,
TransportController: transportController,
}
}

124
testrig/transportcontroller.go
View file

@ -41,6 +41,8 @@ import (
const (
applicationJSON = "application/json"
applicationActivityJSON = "application/activity+json"
textCSV = "text/csv"
textPlain = "text/plain"
)
// NewTestTransportController returns a test transport controller with the given http client.
@ -101,6 +103,7 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
responseBytes = []byte(`{"error":"404 not found"}`)
responseContentType = applicationJSON
responseContentLength = len(responseBytes)
extraHeaders = make(map[string]string, 0)
reqURLString = req.URL.String()
)
@ -124,11 +127,13 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
responseContentType = applicationJSON
responseContentLength = len(responseBytes)
} else if strings.Contains(reqURLString, ".well-known/webfinger") {
responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
} else if strings.Contains(reqURLString, ".weird-webfinger-location/webfinger") {
responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = WebfingerResponse(req)
} else if strings.Contains(reqURLString, ".well-known/host-meta") {
responseCode, responseBytes, responseContentType, responseContentLength = HostMetaResponse(req)
responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = HostMetaResponse(req)
} else if strings.Contains(reqURLString, "lists.example.org") {
responseCode, responseBytes, responseContentType, responseContentLength, extraHeaders = DomainPermissionSubscriptionResponse(req)
} else if note, ok := mockHTTPClient.TestRemoteStatuses[reqURLString]; ok {
// the request is for a note that we have stored
noteI, err := streams.Serialize(note)
@ -239,14 +244,23 @@ func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relat
}
log.Debugf(nil, "returning response %s", string(responseBytes))
reader := bytes.NewReader(responseBytes)
readCloser := io.NopCloser(reader)
header := http.Header{
"Content-Type": {responseContentType},
}
for k, v := range extraHeaders {
header.Add(k, v)
}
return &http.Response{
Request: req,
StatusCode: responseCode,
Body: readCloser,
ContentLength: int64(responseContentLength),
Header: http.Header{"Content-Type": {responseContentType}},
Header: header,
}, nil
}
@ -261,7 +275,13 @@ func (m *MockHTTPClient) DoSigned(req *http.Request, sign httpclient.SignFunc) (
return m.do(req)
}
func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
func HostMetaResponse(req *http.Request) (
responseCode int,
responseBytes []byte,
responseContentType string,
responseContentLength int,
extraHeaders map[string]string,
) {
var hm *apimodel.HostMeta
if req.URL.String() == "https://misconfigured-instance.com/.well-known/host-meta" {
@ -297,7 +317,13 @@ func HostMetaResponse(req *http.Request) (responseCode int, responseBytes []byte
return
}
func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
func WebfingerResponse(req *http.Request) (
responseCode int,
responseBytes []byte,
responseContentType string,
responseContentLength int,
extraHeaders map[string]string,
) {
var wfr *apimodel.WellKnownResponse
switch req.URL.String() {
@ -410,3 +436,89 @@ func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byt
responseContentLength = len(wfrJSON)
return
}
func DomainPermissionSubscriptionResponse(req *http.Request) (
responseCode int,
responseBytes []byte,
responseContentType string,
responseContentLength int,
extraHeaders map[string]string,
) {
const (
csvResp = `#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate
bumfaces.net,suspend,false,false,big jerks,false
peepee.poopoo,suspend,false,false,harassment,false
nothanks.com,suspend,false,false,,false`
csvRespETag = "bigbums6969"
textResp = `bumfaces.net
peepee.poopoo
nothanks.com`
textRespETag = "this is a legit etag i swear"
jsonResp = `[
{
"domain": "bumfaces.net",
"suspended_at": "2020-05-13T13:29:12.000Z",
"public_comment": "big jerks"
},
{
"domain": "peepee.poopoo",
"suspended_at": "2020-05-13T13:29:12.000Z",
"public_comment": "harassment"
},
{
"domain": "nothanks.com",
"suspended_at": "2020-05-13T13:29:12.000Z"
}
]`
jsonRespETag = "don't modify me daddy"
)
switch req.URL.String() {
case "https://lists.example.org/baddies.csv":
extraHeaders = map[string]string{"ETag": csvRespETag}
if req.Header.Get("If-None-Match") == csvRespETag {
// Cached.
responseCode = http.StatusNotModified
} else {
responseBytes = []byte(csvResp)
responseContentType = textCSV
responseCode = http.StatusOK
}
responseContentLength = len(responseBytes)
case "https://lists.example.org/baddies.txt":
extraHeaders = map[string]string{"ETag": textRespETag}
if req.Header.Get("If-None-Match") == textRespETag {
// Cached.
responseCode = http.StatusNotModified
} else {
responseBytes = []byte(textResp)
responseContentType = textPlain
responseCode = http.StatusOK
}
responseContentLength = len(responseBytes)
case "https://lists.example.org/baddies.json":
extraHeaders = map[string]string{"ETag": jsonRespETag}
if req.Header.Get("If-None-Match") == jsonRespETag {
// Cached.
responseCode = http.StatusNotModified
} else {
responseBytes = []byte(jsonResp)
responseContentType = applicationJSON
responseCode = http.StatusOK
}
responseContentLength = len(responseBytes)
default:
responseCode = http.StatusNotFound
responseBytes = []byte(`{"error":"not found"}`)
responseContentType = applicationJSON
responseContentLength = len(responseBytes)
}
return
}