mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-10-29 07:22:24 -05:00
[security] transport.Controller{} and transport.Transport{} security and performance improvements (#564)
* cache transports in controller by privkey-generated pubkey, add retry logic to transport requests
Signed-off-by: kim <grufwub@gmail.com>
* update code comments, defer mutex unlocks
Signed-off-by: kim <grufwub@gmail.com>
* add count to 'performing request' log message
Signed-off-by: kim <grufwub@gmail.com>
* reduce repeated conversions of same url.URL object
Signed-off-by: kim <grufwub@gmail.com>
* move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue
Signed-off-by: kim <grufwub@gmail.com>
* fix security advisories regarding max outgoing conns, max rsp body size
- implemented by a new httpclient.Client{} that wraps an underlying
client with a queue to limit connections, and limit reader wrapping
a response body with a configured maximum size
- update pub.HttpClient args passed around to be this new httpclient.Client{}
Signed-off-by: kim <grufwub@gmail.com>
* add httpclient tests, move ip validation to separate package + change mechanism
Signed-off-by: kim <grufwub@gmail.com>
* fix merge conflicts
Signed-off-by: kim <grufwub@gmail.com>
* use singular mutex in transport rather than separate signer mus
Signed-off-by: kim <grufwub@gmail.com>
* improved useragent string
Signed-off-by: kim <grufwub@gmail.com>
* add note regarding missing test
Signed-off-by: kim <grufwub@gmail.com>
* remove useragent field from transport (instead store in controller)
Signed-off-by: kim <grufwub@gmail.com>
* shutup linter
Signed-off-by: kim <grufwub@gmail.com>
* reset other signing headers on each loop iteration
Signed-off-by: kim <grufwub@gmail.com>
* respect request ctx during retry-backoff sleep period
Signed-off-by: kim <grufwub@gmail.com>
* use external pkg with docs explaining performance "hack"
Signed-off-by: kim <grufwub@gmail.com>
* use http package constants instead of string method literals
Signed-off-by: kim <grufwub@gmail.com>
* add license file headers
Signed-off-by: kim <grufwub@gmail.com>
* update code comment to match new func names
Signed-off-by: kim <grufwub@gmail.com>
* updates to user-agent string
Signed-off-by: kim <grufwub@gmail.com>
* update signed testrig models to fit with new transport logic (instead uses separate signer now)
Signed-off-by: kim <grufwub@gmail.com>
* fuck you linter
Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
GitHub
parent
4ac508f037
commit
223025fc27
61 changed files with 1801 additions and 435 deletions
|
|
@ -1,13 +1,13 @@
|
|||
package testrig
|
||||
|
||||
import (
|
||||
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/messages"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/worker"
|
||||
)
|
||||
|
||||
// NewTestFederatingDB returns a federating DB with the underlying db
|
||||
func NewTestFederatingDB(db db.DB, fedWorker *worker.Worker[messages.FromFederator]) federatingdb.DB {
|
||||
func NewTestFederatingDB(db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator]) federatingdb.DB {
|
||||
return federatingdb.New(db, fedWorker)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,15 +20,15 @@ package testrig
|
|||
|
||||
import (
|
||||
"codeberg.org/gruf/go-store/kv"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/federation"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/messages"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/transport"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/worker"
|
||||
)
|
||||
|
||||
// NewTestFederator returns a federator with the given database and (mock!!) transport controller.
|
||||
func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager, fedWorker *worker.Worker[messages.FromFederator]) federation.Federator {
|
||||
func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager, fedWorker *concurrency.WorkerPool[messages.FromFederator]) federation.Federator {
|
||||
return federation.NewFederator(db, NewTestFederatingDB(db, fedWorker), tc, NewTestTypeConverter(db), mediaManager)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,16 +20,16 @@ package testrig
|
|||
|
||||
import (
|
||||
"codeberg.org/gruf/go-store/kv"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/email"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/federation"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/messages"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/processing"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/worker"
|
||||
)
|
||||
|
||||
// NewTestProcessor returns a Processor suitable for testing purposes
|
||||
func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager, clientWorker *worker.Worker[messages.FromClientAPI], fedWorker *worker.Worker[messages.FromFederator]) processing.Processor {
|
||||
func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager, clientWorker *concurrency.WorkerPool[messages.FromClientAPI], fedWorker *concurrency.WorkerPool[messages.FromFederator]) processing.Processor {
|
||||
return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), mediaManager, storage, db, emailSender, clientWorker, fedWorker)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,6 @@ package testrig
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
|
|
@ -29,7 +27,6 @@ import (
|
|||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
|
@ -42,8 +39,7 @@ import (
|
|||
"github.com/superseriousbusiness/activity/streams/vocab"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/ap"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/messages"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/worker"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/transport"
|
||||
)
|
||||
|
||||
// NewTestTokens returns a map of tokens keyed according to which account the token belongs to.
|
||||
|
|
@ -1855,86 +1851,71 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
|
|||
}
|
||||
}
|
||||
|
||||
// GetSignatureForActivity does some sneaky sneaky work with a mock http client and a test transport controller, in order to derive
|
||||
// the HTTP Signature for the given activity, public key ID, private key, and destination.
|
||||
func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey crypto.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
|
||||
// create a client that basically just pulls the signature out of the request and sets it
|
||||
client := &mockHTTPClient{
|
||||
do: func(req *http.Request) (*http.Response, error) {
|
||||
signatureHeader = req.Header.Get("Signature")
|
||||
digestHeader = req.Header.Get("Digest")
|
||||
dateHeader = req.Header.Get("Date")
|
||||
r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
|
||||
return &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: r,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
// Create temporary federator worker for transport controller
|
||||
fedWorker := worker.New[messages.FromFederator](-1, -1)
|
||||
_ = fedWorker.Start()
|
||||
defer func() { _ = fedWorker.Stop() }()
|
||||
|
||||
// use the client to create a new transport
|
||||
c := NewTestTransportController(client, NewTestDB(), fedWorker)
|
||||
tp, err := c.NewTransport(pubKeyID, privkey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// GetSignatureForActivity prepares a mock HTTP request as if it were going to deliver activity to destination signed for privkey and pubKeyID, signs the request and returns the header values.
|
||||
func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey *rsa.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
|
||||
// convert the activity into json bytes
|
||||
m, err := activity.Serialize()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
bytes, err := json.Marshal(m)
|
||||
b, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// trigger the delivery function for the underlying signature transport, which will trigger the 'do' function of the recorder above
|
||||
if err := tp.SigTransport().Deliver(context.Background(), bytes, destination); err != nil {
|
||||
// Prepare HTTP request signer
|
||||
sig, err := transport.NewPOSTSigner(120)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Prepare a mock request ready for signing
|
||||
r, err := http.NewRequest("POST", destination.String(), bytes.NewReader(b))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
r.Header.Set("Host", destination.Host)
|
||||
r.Header.Set("Date", time.Now().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
|
||||
|
||||
// Sign this new HTTP request
|
||||
if err := sig.SignRequest(privkey, pubKeyID, r, b); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Load signed data from request
|
||||
signatureHeader = r.Header.Get("Signature")
|
||||
digestHeader = r.Header.Get("Digest")
|
||||
dateHeader = r.Header.Get("Date")
|
||||
|
||||
// headers should now be populated
|
||||
return
|
||||
}
|
||||
|
||||
// GetSignatureForDereference does some sneaky sneaky work with a mock http client and a test transport controller, in order to derive
|
||||
// the HTTP Signature for the given derefence GET request using public key ID, private key, and destination.
|
||||
func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
|
||||
// create a client that basically just pulls the signature out of the request and sets it
|
||||
client := &mockHTTPClient{
|
||||
do: func(req *http.Request) (*http.Response, error) {
|
||||
signatureHeader = req.Header.Get("Signature")
|
||||
dateHeader = req.Header.Get("Date")
|
||||
r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
|
||||
return &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: r,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
// Create temporary federator worker for transport controller
|
||||
fedWorker := worker.New[messages.FromFederator](-1, -1)
|
||||
_ = fedWorker.Start()
|
||||
defer func() { _ = fedWorker.Stop() }()
|
||||
|
||||
// use the client to create a new transport
|
||||
c := NewTestTransportController(client, NewTestDB(), fedWorker)
|
||||
tp, err := c.NewTransport(pubKeyID, privkey)
|
||||
// GetSignatureForDereference prepares a mock HTTP request as if it were going to dereference destination signed for privkey and pubKeyID, signs the request and returns the header values.
|
||||
func GetSignatureForDereference(pubKeyID string, privkey *rsa.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
|
||||
// Prepare HTTP request signer
|
||||
sig, err := transport.NewGETSigner(120)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// trigger the dereference function for the underlying signature transport, which will trigger the 'do' function of the recorder above
|
||||
if _, err := tp.SigTransport().Dereference(context.Background(), destination); err != nil {
|
||||
// Prepare a mock request ready for signing
|
||||
r, err := http.NewRequest("GET", destination.String(), nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
r.Header.Set("Host", destination.Host)
|
||||
r.Header.Set("Date", time.Now().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
|
||||
|
||||
// Sign this new HTTP request
|
||||
if err := sig.SignRequest(privkey, pubKeyID, r, nil); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Load signed data from request
|
||||
signatureHeader = r.Header.Get("Signature")
|
||||
digestHeader = r.Header.Get("Digest")
|
||||
dateHeader = r.Header.Get("Date")
|
||||
|
||||
// headers should now be populated
|
||||
return
|
||||
|
|
|
|||
|
|
@ -24,11 +24,11 @@ import (
|
|||
"net/http"
|
||||
|
||||
"github.com/superseriousbusiness/activity/pub"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/federation"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/messages"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/transport"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/worker"
|
||||
)
|
||||
|
||||
// NewTestTransportController returns a test transport controller with the given http client.
|
||||
|
|
@ -40,7 +40,7 @@ import (
|
|||
// Unlike the other test interfaces provided in this package, you'll probably want to call this function
|
||||
// PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
|
||||
// basis.
|
||||
func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *worker.Worker[messages.FromFederator]) transport.Controller {
|
||||
func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *concurrency.WorkerPool[messages.FromFederator]) transport.Controller {
|
||||
return transport.NewController(db, NewTestFederatingDB(db, fedWorker), &federation.Clock{}, client)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue