[bug] respect X-Robots-Tag and robots.txt on api/v1/instance and nodeinfo (#3756)

* feat: check X-Robots-Tag

when accessing /api/v1/instance or /nodeinfo endpoints respect
X-Robots-Tag

* chore: go fmt ./...

* Check robots.txt as well, add tests

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>

vendor/github.com/temoto/robotstxt/.travis.yml

@@ -0,0 +1,30 @@
+cache:
+  go: true
+  directories:
+  - $HOME/.cache
+  - $HOME/bin
+  - $HOME/gopath/pkg/mod
+language: go
+go:
+- 1.11
+- 1.12
+- 1.13
+- 1.14
+- 1.x
+- master
+install: true
+script: GO111MODULE=on go test -race
+
+matrix:
+  include:
+  - go: 1.x
+    env: task=coverage
+    script: GO111MODULE=on go test -race -covermode=atomic -coverprofile=coverage.txt
+    after_success: bash <(curl -s https://codecov.io/bash)
+  - go: 1.x
+    env: task=bench
+    script: GO111MODULE=on ./script/bench
+  - go: 1.x
+    install: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b $HOME/bin v1.19.1
+    env: task=clean
+    script: GO111MODULE=on ./script/clean