[feature] Allow users to skip http client tls verification for testing purposes (with appropriately loud warnings) (#2052)

internal/config/config.go

@@ -168,9 +168,10 @@ type Configuration struct {
 }
 
 type HTTPClientConfiguration struct {
-	AllowIPs []string      `name:"allow-ips"`
-	BlockIPs []string      `name:"block-ips"`
-	Timeout  time.Duration `name:"timeout"`
+	AllowIPs              []string      `name:"allow-ips"`
+	BlockIPs              []string      `name:"block-ips"`
+	Timeout               time.Duration `name:"timeout"`
+	TLSInsecureSkipVerify bool          `name:"tls-insecure-skip-verify"`
 }
 
 type CacheConfiguration struct {

internal/config/defaults.go

@@ -234,9 +234,10 @@ var Defaults = Configuration{
 	},
 
 	HTTPClient: HTTPClientConfiguration{
-		AllowIPs: make([]string, 0),
-		BlockIPs: make([]string, 0),
-		Timeout:  10 * time.Second,
+		AllowIPs:              make([]string, 0),
+		BlockIPs:              make([]string, 0),
+		Timeout:               10 * time.Second,
+		TLSInsecureSkipVerify: false,
 	},
 
 	AdminMediaPruneDryRun: true,

internal/config/flags.go

@@ -60,6 +60,7 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
 		cmd.PersistentFlags().StringSlice(HTTPClientAllowIPsFlag(), cfg.HTTPClient.AllowIPs, "no usage string")
 		cmd.PersistentFlags().StringSlice(HTTPClientBlockIPsFlag(), cfg.HTTPClient.BlockIPs, "no usage string")
 		cmd.PersistentFlags().Duration(HTTPClientTimeoutFlag(), cfg.HTTPClient.Timeout, "no usage string")
+		cmd.PersistentFlags().Bool(HTTPClientTLSInsecureSkipVerifyFlag(), cfg.HTTPClient.TLSInsecureSkipVerify, "no usage string")
 	})
 }
 

internal/config/helpers.gen.go

@@ -2399,6 +2399,31 @@ func GetHTTPClientTimeout() time.Duration { return global.GetHTTPClientTimeout()
 // SetHTTPClientTimeout safely sets the value for global configuration 'HTTPClient.Timeout' field
 func SetHTTPClientTimeout(v time.Duration) { global.SetHTTPClientTimeout(v) }
 
+// GetHTTPClientTLSInsecureSkipVerify safely fetches the Configuration value for state's 'HTTPClient.TLSInsecureSkipVerify' field
+func (st *ConfigState) GetHTTPClientTLSInsecureSkipVerify() (v bool) {
+	st.mutex.RLock()
+	v = st.config.HTTPClient.TLSInsecureSkipVerify
+	st.mutex.RUnlock()
+	return
+}
+
+// SetHTTPClientTLSInsecureSkipVerify safely sets the Configuration value for state's 'HTTPClient.TLSInsecureSkipVerify' field
+func (st *ConfigState) SetHTTPClientTLSInsecureSkipVerify(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.HTTPClient.TLSInsecureSkipVerify = v
+	st.reloadToViper()
+}
+
+// HTTPClientTLSInsecureSkipVerifyFlag returns the flag name for the 'HTTPClient.TLSInsecureSkipVerify' field
+func HTTPClientTLSInsecureSkipVerifyFlag() string { return "httpclient-tls-insecure-skip-verify" }
+
+// GetHTTPClientTLSInsecureSkipVerify safely fetches the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field
+func GetHTTPClientTLSInsecureSkipVerify() bool { return global.GetHTTPClientTLSInsecureSkipVerify() }
+
+// SetHTTPClientTLSInsecureSkipVerify safely sets the value for global configuration 'HTTPClient.TLSInsecureSkipVerify' field
+func SetHTTPClientTLSInsecureSkipVerify(v bool) { global.SetHTTPClientTLSInsecureSkipVerify(v) }
+
 // GetCacheGTSAccountMaxSize safely fetches the Configuration value for state's 'Cache.GTS.AccountMaxSize' field
 func (st *ConfigState) GetCacheGTSAccountMaxSize() (v int) {
 	st.mutex.RLock()

internal/httpclient/client.go

@@ -19,6 +19,7 @@ package httpclient
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -86,6 +87,14 @@ type Config struct {
 
 	// BlockRanges blocks outgoing communiciations to given IP nets.
 	BlockRanges []netip.Prefix
+
+	// TLSInsecureSkipVerify can be set to true to
+	// skip validation of remote TLS certificates.
+	//
+	// THIS SHOULD BE USED FOR TESTING ONLY, IF YOU
+	// TURN THIS ON WHILE RUNNING IN PRODUCTION YOU
+	// ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS!
+	TLSInsecureSkipVerify bool
 }
 
 // Client wraps an underlying http.Client{} to provide the following:
@@ -139,11 +148,26 @@ func New(cfg Config) *Client {
 	c.client.Timeout = cfg.Timeout
 	c.bodyMax = cfg.MaxBodySize
 
+	// Prepare TLS config for transport.
+	tlsClientConfig := &tls.Config{
+		InsecureSkipVerify: cfg.TLSInsecureSkipVerify, //nolint:gosec
+	}
+
+	if tlsClientConfig.InsecureSkipVerify {
+		// Warn against playing silly buggers.
+		log.Warn(nil, "http-client.tls-insecure-skip-verify was set to TRUE. "+
+			"*****THIS SHOULD BE USED FOR TESTING ONLY, IF YOU TURN THIS ON WHILE "+
+			"RUNNING IN PRODUCTION YOU ARE LEAVING YOUR SERVER WIDE OPEN TO ATTACKS! "+
+			"IF IN DOUBT, STOP YOUR SERVER *NOW* AND ADJUST YOUR CONFIGURATION!*****",
+		)
+	}
+
 	// Set underlying HTTP client roundtripper.
 	c.client.Transport = &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		ForceAttemptHTTP2:     true,
 		DialContext:           d.DialContext,
+		TLSClientConfig:       tlsClientConfig,
 		MaxIdleConns:          cfg.MaxIdleConns,
 		IdleConnTimeout:       90 * time.Second,
 		TLSHandshakeTimeout:   10 * time.Second,