[feature] Allow turning empty user-agent filtering off

cmd/gotosocial/action/server/server.go

@@ -438,7 +438,7 @@ func Start(ctx context.Context) error {
 		// the logger, otherwise won't be accessible.
 		middleware.Logger(config.GetLogClientIP()),
 		middleware.HeaderFilter(state),
-		middleware.UserAgent(),
+		middleware.UserAgentOrTeapot(),
 		middleware.CORS(),
 		middleware.ExtraHeaders(),
 	}...)

cmd/gotosocial/action/testrig/testrig.go

@@ -200,7 +200,7 @@ func Start(ctx context.Context) error {
 	middlewares = append(middlewares, []gin.HandlerFunc{
 		middleware.Logger(config.GetLogClientIP()),
 		middleware.HeaderFilter(state),
-		middleware.UserAgent(),
+		middleware.UserAgentOrTeapot(),
 		middleware.CORS(),
 		middleware.ExtraHeaders(),
 	}...)

docs/configuration/instance.md

@@ -229,4 +229,15 @@ instance-stats-mode: ""
 # Options: [true, false]
 # Default: true
 instance-allow-backdating-statuses: true
+
+# Bool. If set to true, then any HTTP requests coming into the instance,
+# whether by client, web browser, or server-to-server requests, will be
+# rejected if they do not identify themselves by setting a value on the
+# request's User-Agent header. Since almost all HTTP clients provide
+# *something* as a User-Agent value, leaving this set to "true" will
+# likely not cause issues, but you can turn it off if necessary.
+#
+# Options: [true, false]
+# Default: true
+instance-reject-empty-user-agents: true
 ```

example/config.yaml

@@ -529,6 +529,17 @@ instance-stats-mode: ""
 # Default: true
 instance-allow-backdating-statuses: true
 
+# Bool. If set to true, then any HTTP requests coming into the instance,
+# whether by client, web browser, or server-to-server requests, will be
+# rejected if they do not identify themselves by setting a value on the
+# request's User-Agent header. Since almost all HTTP clients provide
+# *something* as a User-Agent value, leaving this set to "true" will
+# likely not cause issues, but you can turn it off if necessary.
+#
+# Options: [true, false]
+# Default: true
+instance-reject-empty-user-agents: true
+
 ###########################
 ##### ACCOUNTS CONFIG #####
 ###########################

internal/config/config.go

@@ -105,6 +105,7 @@ type Configuration struct {
 	InstanceSubscriptionsProcessEvery time.Duration      `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."`
 	InstanceStatsMode                 string             `name:"instance-stats-mode" usage:"Allows you to customize the way stats are served to crawlers: one of '', 'serve', 'zero', 'baffle'. Home page stats remain unchanged."`
 	InstanceAllowBackdatingStatuses   bool               `name:"instance-allow-backdating-statuses" usage:"Allow local accounts to backdate statuses using the scheduled_at param to /api/v1/statuses"`
+	InstanceRejectEmptyUserAgents     bool               `name:"instance-reject-empty-user-agents" usage:"Reject all incoming HTTP requests that do not have a User-Agent header set"`
 
 	AccountsRegistrationOpen         bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."`
 	AccountsReasonRequired           bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"`

internal/config/helpers.gen.go

@@ -77,6 +77,7 @@ const (
 	InstanceSubscriptionsProcessEveryFlag          = "instance-subscriptions-process-every"
 	InstanceStatsModeFlag                          = "instance-stats-mode"
 	InstanceAllowBackdatingStatusesFlag            = "instance-allow-backdating-statuses"
+	InstanceRejectEmptyUserAgentsFlag              = "instance-reject-empty-user-agents"
 	AccountsRegistrationOpenFlag                   = "accounts-registration-open"
 	AccountsReasonRequiredFlag                     = "accounts-reason-required"
 	AccountsRegistrationDailyLimitFlag             = "accounts-registration-daily-limit"
@@ -277,6 +278,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
 	flags.Duration("instance-subscriptions-process-every", cfg.InstanceSubscriptionsProcessEvery, "Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from.")
 	flags.String("instance-stats-mode", cfg.InstanceStatsMode, "Allows you to customize the way stats are served to crawlers: one of '', 'serve', 'zero', 'baffle'. Home page stats remain unchanged.")
 	flags.Bool("instance-allow-backdating-statuses", cfg.InstanceAllowBackdatingStatuses, "Allow local accounts to backdate statuses using the scheduled_at param to /api/v1/statuses")
+	flags.Bool("instance-reject-empty-user-agents", cfg.InstanceRejectEmptyUserAgents, "Reject all incoming HTTP requests that do not have a User-Agent header set")
 	flags.Bool("accounts-registration-open", cfg.AccountsRegistrationOpen, "Allow anyone to submit an account signup request. If false, server will be invite-only.")
 	flags.Bool("accounts-reason-required", cfg.AccountsReasonRequired, "Do new account signups require a reason to be submitted on registration?")
 	flags.Int("accounts-registration-daily-limit", cfg.AccountsRegistrationDailyLimit, "Limit amount of approved account sign-ups allowed per 24hrs before registration is closed. 0 or less = no limit.")
@@ -420,7 +422,7 @@ func (cfg *Configuration) RegisterFlags(flags *pflag.FlagSet) {
 }
 
 func (cfg *Configuration) MarshalMap() map[string]any {
-	cfgmap := make(map[string]any, 197)
+	cfgmap := make(map[string]any, 198)
 	cfgmap["log-level"] = cfg.LogLevel
 	cfgmap["log-format"] = cfg.LogFormat
 	cfgmap["log-timestamp-format"] = cfg.LogTimestampFormat
@@ -469,6 +471,7 @@ func (cfg *Configuration) MarshalMap() map[string]any {
 	cfgmap["instance-subscriptions-process-every"] = cfg.InstanceSubscriptionsProcessEvery
 	cfgmap["instance-stats-mode"] = cfg.InstanceStatsMode
 	cfgmap["instance-allow-backdating-statuses"] = cfg.InstanceAllowBackdatingStatuses
+	cfgmap["instance-reject-empty-user-agents"] = cfg.InstanceRejectEmptyUserAgents
 	cfgmap["accounts-registration-open"] = cfg.AccountsRegistrationOpen
 	cfgmap["accounts-reason-required"] = cfg.AccountsReasonRequired
 	cfgmap["accounts-registration-daily-limit"] = cfg.AccountsRegistrationDailyLimit
@@ -1019,6 +1022,14 @@ func (cfg *Configuration) UnmarshalMap(cfgmap map[string]any) error {
 		}
 	}
 
+	if ival, ok := cfgmap["instance-reject-empty-user-agents"]; ok {
+		var err error
+		cfg.InstanceRejectEmptyUserAgents, err = cast.ToBoolE(ival)
+		if err != nil {
+			return fmt.Errorf("error casting %#v -> bool for 'instance-reject-empty-user-agents': %w", ival, err)
+		}
+	}
+
 	if ival, ok := cfgmap["accounts-registration-open"]; ok {
 		var err error
 		cfg.AccountsRegistrationOpen, err = cast.ToBoolE(ival)
@@ -3302,6 +3313,28 @@ func GetInstanceAllowBackdatingStatuses() bool { return global.GetInstanceAllowB
 // SetInstanceAllowBackdatingStatuses safely sets the value for global configuration 'InstanceAllowBackdatingStatuses' field
 func SetInstanceAllowBackdatingStatuses(v bool) { global.SetInstanceAllowBackdatingStatuses(v) }
 
+// GetInstanceRejectEmptyUserAgents safely fetches the Configuration value for state's 'InstanceRejectEmptyUserAgents' field
+func (st *ConfigState) GetInstanceRejectEmptyUserAgents() (v bool) {
+	st.mutex.RLock()
+	v = st.config.InstanceRejectEmptyUserAgents
+	st.mutex.RUnlock()
+	return
+}
+
+// SetInstanceRejectEmptyUserAgents safely sets the Configuration value for state's 'InstanceRejectEmptyUserAgents' field
+func (st *ConfigState) SetInstanceRejectEmptyUserAgents(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.InstanceRejectEmptyUserAgents = v
+	st.reloadToViper()
+}
+
+// GetInstanceRejectEmptyUserAgents safely fetches the value for global configuration 'InstanceRejectEmptyUserAgents' field
+func GetInstanceRejectEmptyUserAgents() bool { return global.GetInstanceRejectEmptyUserAgents() }
+
+// SetInstanceRejectEmptyUserAgents safely sets the value for global configuration 'InstanceRejectEmptyUserAgents' field
+func SetInstanceRejectEmptyUserAgents(v bool) { global.SetInstanceRejectEmptyUserAgents(v) }
+
 // GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field
 func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) {
 	st.mutex.RLock()

internal/middleware/useragent.go

@@ -21,19 +21,49 @@ import (
 	"net/http"
 
 	apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util"
+	"code.superseriousbusiness.org/gotosocial/internal/config"
+	"code.superseriousbusiness.org/gotosocial/internal/log"
 	"github.com/gin-gonic/gin"
 )
 
-// UserAgent returns a gin middleware which aborts requests with
+// UserAgentOrTeapot returns a gin middleware
-// empty user agent strings, returning code 418 - I'm a teapot.
+// which aborts requests with empty user agent
-func UserAgent() gin.HandlerFunc {
+// strings, returning code 418 - I'm a teapot.
-	// todo: make this configurable
+//
-	var rsp = []byte(`{"error": "I'm a teapot: no user-agent sent with request"}`)
+// If `instance-reject-empty-user-agents` is
+// false, it just logs a debug msg instead.
+func UserAgentOrTeapot() gin.HandlerFunc {
+
+	// Build variables outside the handler
+	// so they're not instantiated every
+	// time a request is processed.
+	var (
+		rsp         = []byte(`{"error": "I'm a teapot: no user-agent sent with request"}`)
+		rejectEmpty = config.GetInstanceRejectEmptyUserAgents()
+	)
+
 	return func(c *gin.Context) {
-		if ua := c.Request.UserAgent(); ua == "" {
+		ua := c.Request.UserAgent()
-			apiutil.Data(c,
+		if ua != "" {
-				http.StatusTeapot, apiutil.AppJSON, rsp)
+			// All good.
+			return
+		}
+
+		if !rejectEmpty {
+			// No user-agent was
+			// set but that's OK.
+			log.Debugf(
+				c.Request.Context(),
+				"allowing request with empty User-Agent from client %s",
+				c.ClientIP(),
+			)
+			return
+		}
+
+		// No user-agent set and that's not ok!
+		//
+		// Give them a taste of the ol' teapot.
+		apiutil.Data(c, http.StatusTeapot, apiutil.AppJSON, rsp)
 		c.Abort()
 	}
-	}
 }

test/envparsing.sh

@@ -127,6 +127,7 @@ EXPECT=$(cat << "EOF"
         "nl",
         "en-GB"
     ],
+    "instance-reject-empty-user-agents": false,
     "instance-stats-mode": "baffle",
     "instance-subscriptions-process-every": 86400000000000,
     "instance-subscriptions-process-from": "23:00",
@@ -264,6 +265,7 @@ GTS_INSTANCE_FEDERATION_SPAM_FILTER=true \
 GTS_INSTANCE_DELIVER_TO_SHARED_INBOXES=false \
 GTS_INSTANCE_INJECT_MASTODON_VERSION=true \
 GTS_INSTANCE_LANGUAGES="nl,en-gb" \
+GTS_INSTANCE_REJECT_EMPTY_USER_AGENTS="false" \
 GTS_INSTANCE_STATS_MODE="baffle" \
 GTS_ACCOUNTS_ALLOW_CUSTOM_CSS=true \
 GTS_ACCOUNTS_CUSTOM_CSS_LENGTH=5000 \

testrig/config.go

@@ -107,6 +107,7 @@ func testDefaults() config.Configuration {
 		InstanceSubscriptionsProcessFrom:  "23:00",        // 11pm,
 		InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day.
 		InstanceAllowBackdatingStatuses:   true,
+		InstanceRejectEmptyUserAgents:     false,
 
 		AccountsRegistrationOpen:         true,
 		AccountsReasonRequired:           true,