[feature] Add meta robots tag; allow robots to index profile card if user is Discoverable (#842)

* rework robots.txt response

* don't let robots snippet from statuses/threads

* allow robots to index if user is Discoverable

* add license text

internal/web/profile.go

@@ -82,6 +82,12 @@ func (m *Module) profileGETHandler(c *gin.Context) {
 		return
 	}
 
+	// only allow search engines / robots to view this page if account is discoverable
+	var robotsMeta string
+	if account.Discoverable {
+		robotsMeta = robotsAllowSome
+	}
+
 	// we should only show the 'back to top' button if the
 	// profile visitor is paging through statuses
 	showBackToTop := false
@@ -112,6 +118,7 @@ func (m *Module) profileGETHandler(c *gin.Context) {
 		"instance":         instance,
 		"account":          account,
 		"ogMeta":           ogBase(instance).withAccount(account),
+		"robotsMeta":       robotsMeta,
 		"statuses":         statusResp.Items,
 		"statuses_next":    statusResp.NextLink,
 		"show_back_to_top": showBackToTop,

internal/web/robots.go

@@ -0,0 +1,24 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package web
+
+// https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#robotsmeta
+const (
+	robotsAllowSome = "nofollow, noarchive, nositelinkssearchbox, max-image-preview:standard"
+)