mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2026-01-05 22:13:16 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[security] transport.Controller{} and transport.Transport{} security and performance improvements (#564)

Browse source 
* cache transports in controller by privkey-generated pubkey, add retry logic to transport requests

Signed-off-by: kim <grufwub@gmail.com>

* update code comments, defer mutex unlocks

Signed-off-by: kim <grufwub@gmail.com>

* add count to 'performing request' log message

Signed-off-by: kim <grufwub@gmail.com>

* reduce repeated conversions of same url.URL object

Signed-off-by: kim <grufwub@gmail.com>

* move worker.Worker to concurrency subpackage, add WorkQueue type, limit transport http client use by WorkQueue

Signed-off-by: kim <grufwub@gmail.com>

* fix security advisories regarding max outgoing conns, max rsp body size

- implemented by a new httpclient.Client{} that wraps an underlying
  client with a queue to limit connections, and limit reader wrapping
  a response body with a configured maximum size
- update pub.HttpClient args passed around to be this new httpclient.Client{}

Signed-off-by: kim <grufwub@gmail.com>

* add httpclient tests, move ip validation to separate package + change mechanism

Signed-off-by: kim <grufwub@gmail.com>

* fix merge conflicts

Signed-off-by: kim <grufwub@gmail.com>

* use singular mutex in transport rather than separate signer mus

Signed-off-by: kim <grufwub@gmail.com>

* improved useragent string

Signed-off-by: kim <grufwub@gmail.com>

* add note regarding missing test

Signed-off-by: kim <grufwub@gmail.com>

* remove useragent field from transport (instead store in controller)

Signed-off-by: kim <grufwub@gmail.com>

* shutup linter

Signed-off-by: kim <grufwub@gmail.com>

* reset other signing headers on each loop iteration

Signed-off-by: kim <grufwub@gmail.com>

* respect request ctx during retry-backoff sleep period

Signed-off-by: kim <grufwub@gmail.com>

* use external pkg with docs explaining performance "hack"

Signed-off-by: kim <grufwub@gmail.com>

* use http package constants instead of string method literals

Signed-off-by: kim <grufwub@gmail.com>

* add license file headers

Signed-off-by: kim <grufwub@gmail.com>

* update code comment to match new func names

Signed-off-by: kim <grufwub@gmail.com>

* updates to user-agent string

Signed-off-by: kim <grufwub@gmail.com>

* update signed testrig models to fit with new transport logic (instead uses separate signer now)

Signed-off-by: kim <grufwub@gmail.com>

* fuck you linter

Signed-off-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2022-05-15 10:16:43 +01:00 • committed by GitHub
commit 223025fc27
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
61 changed files with 1801 additions and 435 deletions
Download patch file Download diff file Expand all files Collapse all files

107
testrig/testmodels.go
View file

@ -20,8 +20,6 @@ package testrig
import (
"bytes"
"context"
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
@ -29,7 +27,6 @@ import (
"encoding/json"
"encoding/pem"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
@ -42,8 +39,7 @@ import (
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/worker"
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
// NewTestTokens returns a map of tokens keyed according to which account the token belongs to.
@ -1855,86 +1851,71 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin
}
}
// GetSignatureForActivity does some sneaky sneaky work with a mock http client and a test transport controller, in order to derive
// the HTTP Signature for the given activity, public key ID, private key, and destination.
func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey crypto.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
// create a client that basically just pulls the signature out of the request and sets it
client := &mockHTTPClient{
do: func(req *http.Request) (*http.Response, error) {
signatureHeader = req.Header.Get("Signature")
digestHeader = req.Header.Get("Digest")
dateHeader = req.Header.Get("Date")
r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
return &http.Response{
StatusCode: 200,
Body: r,
}, nil
},
}
// Create temporary federator worker for transport controller
fedWorker := worker.New[messages.FromFederator](-1, -1)
_ = fedWorker.Start()
defer func() { _ = fedWorker.Stop() }()
// use the client to create a new transport
c := NewTestTransportController(client, NewTestDB(), fedWorker)
tp, err := c.NewTransport(pubKeyID, privkey)
if err != nil {
panic(err)
}
// GetSignatureForActivity prepares a mock HTTP request as if it were going to deliver activity to destination signed for privkey and pubKeyID, signs the request and returns the header values.
func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey *rsa.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
// convert the activity into json bytes
m, err := activity.Serialize()
if err != nil {
panic(err)
}
bytes, err := json.Marshal(m)
b, err := json.Marshal(m)
if err != nil {
panic(err)
}
// trigger the delivery function for the underlying signature transport, which will trigger the 'do' function of the recorder above
if err := tp.SigTransport().Deliver(context.Background(), bytes, destination); err != nil {
// Prepare HTTP request signer
sig, err := transport.NewPOSTSigner(120)
if err != nil {
panic(err)
}
// Prepare a mock request ready for signing
r, err := http.NewRequest("POST", destination.String(), bytes.NewReader(b))
if err != nil {
panic(err)
}
r.Header.Set("Host", destination.Host)
r.Header.Set("Date", time.Now().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
// Sign this new HTTP request
if err := sig.SignRequest(privkey, pubKeyID, r, b); err != nil {
panic(err)
}
// Load signed data from request
signatureHeader = r.Header.Get("Signature")
digestHeader = r.Header.Get("Digest")
dateHeader = r.Header.Get("Date")
// headers should now be populated
return
}
// GetSignatureForDereference does some sneaky sneaky work with a mock http client and a test transport controller, in order to derive
// the HTTP Signature for the given derefence GET request using public key ID, private key, and destination.
func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
// create a client that basically just pulls the signature out of the request and sets it
client := &mockHTTPClient{
do: func(req *http.Request) (*http.Response, error) {
signatureHeader = req.Header.Get("Signature")
dateHeader = req.Header.Get("Date")
r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
return &http.Response{
StatusCode: 200,
Body: r,
}, nil
},
}
// Create temporary federator worker for transport controller
fedWorker := worker.New[messages.FromFederator](-1, -1)
_ = fedWorker.Start()
defer func() { _ = fedWorker.Stop() }()
// use the client to create a new transport
c := NewTestTransportController(client, NewTestDB(), fedWorker)
tp, err := c.NewTransport(pubKeyID, privkey)
// GetSignatureForDereference prepares a mock HTTP request as if it were going to dereference destination signed for privkey and pubKeyID, signs the request and returns the header values.
func GetSignatureForDereference(pubKeyID string, privkey *rsa.PrivateKey, destination *url.URL) (signatureHeader string, digestHeader string, dateHeader string) {
// Prepare HTTP request signer
sig, err := transport.NewGETSigner(120)
if err != nil {
panic(err)
}
// trigger the dereference function for the underlying signature transport, which will trigger the 'do' function of the recorder above
if _, err := tp.SigTransport().Dereference(context.Background(), destination); err != nil {
// Prepare a mock request ready for signing
r, err := http.NewRequest("GET", destination.String(), nil)
if err != nil {
panic(err)
}
r.Header.Set("Host", destination.Host)
r.Header.Set("Date", time.Now().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
// Sign this new HTTP request
if err := sig.SignRequest(privkey, pubKeyID, r, nil); err != nil {
panic(err)
}
// Load signed data from request
signatureHeader = r.Header.Get("Signature")
digestHeader = r.Header.Get("Digest")
dateHeader = r.Header.Get("Date")
// headers should now be populated
return