[bug] respect X-Robots-Tag and robots.txt on api/v1/instance and nodeinfo (#3756)

* feat: check X-Robots-Tag

when accessing /api/v1/instance or /nodeinfo endpoints respect
X-Robots-Tag

* chore: go fmt ./...

* Check robots.txt as well, add tests

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>

vendor/github.com/temoto/robotstxt/fuzz.go

@@ -0,0 +1,29 @@
+// +build gofuzz
+
+package robotstxt
+
+import "testing/quick"
+
+func Fuzz(data []byte) int {
+	r, err := FromBytes(data)
+	if err != nil {
+		if r != nil {
+			panic("r != nil on error")
+		}
+		return 0
+	}
+
+	// FindGroup must never return nil
+	f1 := func(agent string) bool { return r.FindGroup(agent) != nil }
+	if err := quick.Check(f1, nil); err != nil {
+		panic(err)
+	}
+
+	// just check TestAgent doesn't panic
+	f2 := func(path, agent string) bool { r.TestAgent(path, agent); return true }
+	if err := quick.Check(f2, nil); err != nil {
+		panic(err)
+	}
+
+	return 1
+}