[chore] The Big Middleware and API Refactor (tm) (#1250)

* interim commit: start refactoring middlewares into package under router

* another interim commit, this is becoming a big job

* another fucking massive interim commit

* refactor bookmarks to new style

* ambassador, wiz zeze commits you are spoiling uz

* she compiles, we're getting there

* we're just normal men; we're just innocent men

* apiutil

* whoopsie

* i'm glad noone reads commit msgs haha :blob_sweat:

* use that weirdo go-bytesize library for maxMultipartMemory

* fix media module paths

internal/api/security/robots.go

@@ -1,57 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package security
-
-import (
-	"net/http"
-
-	"github.com/gin-gonic/gin"
-)
-
-const robotsString = `User-agent: *
-Crawl-delay: 500
-# api stuff
-Disallow: /api/
-# auth/login stuff
-Disallow: /auth/
-Disallow: /oauth/
-Disallow: /check_your_email
-Disallow: /wait_for_approval
-Disallow: /account_disabled
-# well known stuff
-Disallow: /.well-known/
-# files
-Disallow: /fileserver/
-# s2s AP stuff
-Disallow: /users/
-Disallow: /emoji/
-# panels
-Disallow: /admin
-Disallow: /user
-Disallow: /settings/
-`
-
-// RobotsGETHandler returns a decent robots.txt that prevents crawling
-// the api, auth pages, settings pages, etc.
-//
-// More granular robots meta tags are then applied for web pages
-// depending on user preferences (see internal/web).
-func (m *Module) RobotsGETHandler(c *gin.Context) {
-	c.String(http.StatusOK, robotsString)
-}