work on emojis

PROGRESS.md

@@ -144,6 +144,7 @@
   * [ ] Custom Emojis
     * [ ] /api/v1/custom_emojis GET                         (Show this server's custom emoji)
   * [ ] Admin
+    * [x] /api/v1/admin/custom_emojis POST                  (Upload a custom emoji for instance-wide usage)
     * [ ] /api/v1/admin/accounts GET                        (View accounts filtered by criteria)
     * [ ] /api/v1/admin/accounts/:id GET                    (View admin level info about an account)
     * [ ] /api/v1/admin/accounts/:id/action POST            (Perform an admin action on account)

internal/apimodule/admin/admin.go

@@ -0,0 +1,88 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/apimodule"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/mastotypes"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/router"
+)
+
+const (
+	idKey                 = "id"
+	basePath              = "/api/v1/admin"
+	emojiPath             = basePath + "/custom_emojis"
+	basePathWithID        = basePath + "/:" + idKey
+	verifyPath            = basePath + "/verify_credentials"
+	updateCredentialsPath = basePath + "/update_credentials"
+)
+
+type adminModule struct {
+	config         *config.Config
+	db             db.DB
+	mediaHandler   media.MediaHandler
+	mastoConverter mastotypes.Converter
+	log            *logrus.Logger
+}
+
+// New returns a new account module
+func New(config *config.Config, db db.DB, mediaHandler media.MediaHandler, mastoConverter mastotypes.Converter, log *logrus.Logger) apimodule.ClientAPIModule {
+	return &adminModule{
+		config:         config,
+		db:             db,
+		mediaHandler:   mediaHandler,
+		mastoConverter: mastoConverter,
+		log:            log,
+	}
+}
+
+// Route attaches all routes from this module to the given router
+func (m *adminModule) Route(r router.Router) error {
+	r.AttachHandler(http.MethodPost, emojiPath, m.emojiCreatePOSTHandler)
+	return nil
+}
+
+func (m *adminModule) CreateTables(db db.DB) error {
+	models := []interface{}{
+		&gtsmodel.User{},
+		&gtsmodel.Account{},
+		&gtsmodel.Follow{},
+		&gtsmodel.FollowRequest{},
+		&gtsmodel.Status{},
+		&gtsmodel.Application{},
+		&gtsmodel.EmailDomainBlock{},
+		&gtsmodel.MediaAttachment{},
+		&gtsmodel.Emoji{},
+	}
+
+	for _, m := range models {
+		if err := db.CreateTable(m); err != nil {
+			return fmt.Errorf("error creating table: %s", err)
+		}
+	}
+	return nil
+}

internal/apimodule/admin/emojicreate.go

@@ -0,0 +1,130 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	mastotypes "github.com/superseriousbusiness/gotosocial/internal/mastotypes/mastomodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (m *adminModule) emojiCreatePOSTHandler(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "emojiCreatePOSTHandler",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
+
+	// make sure we're authed with an admin account
+	authed, err := oauth.MustAuth(c, true, true, true, true) // posting a status is serious business so we want *everything*
+	if err != nil {
+		l.Debugf("couldn't auth: %s", err)
+		c.JSON(http.StatusForbidden, gin.H{"error": err.Error()})
+		return
+	}
+	if !authed.User.Admin {
+		l.Debugf("user %s not an admin", authed.User.ID)
+		c.JSON(http.StatusForbidden, gin.H{"error": "not an admin"})
+		return
+	}
+
+	// extract the media create form from the request context
+	l.Tracef("parsing request form: %+v", c.Request.Form)
+	form := &mastotypes.EmojiCreateRequest{}
+	if err := c.ShouldBind(form); err != nil {
+		l.Debugf("error parsing form %+v: %s", c.Request.Form, err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not parse form: %s", err)})
+		return
+	}
+
+	// Give the fields on the request form a first pass to make sure the request is superficially valid.
+	l.Tracef("validating form %+v", form)
+	if err := validateCreateEmoji(form); err != nil {
+		l.Debugf("error validating form: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	// open the emoji and extract the bytes from it
+	f, err := form.Image.Open()
+	if err != nil {
+		l.Debugf("error opening emoji: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not open provided emoji: %s", err)})
+		return
+	}
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		l.Debugf("error reading emoji: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not read provided emoji: %s", err)})
+		return
+	}
+	if size == 0 {
+		l.Debug("could not read provided emoji: size 0 bytes")
+		c.JSON(http.StatusBadRequest, gin.H{"error": "could not read provided emoji: size 0 bytes"})
+		return
+	}
+
+	// allow the mediaHandler to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using
+	emoji, err := m.mediaHandler.ProcessLocalEmoji(buf.Bytes(), form.Shortcode)
+	if err != nil {
+		l.Debugf("error reading emoji: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not process emoji: %s", err)})
+		return
+	}
+
+	mastoEmoji, err := m.mastoConverter.EmojiToMasto(emoji)
+	if err != nil {
+		l.Debugf("error converting emoji to mastotype: %s", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("could not convert emoji: %s", err)})
+		return
+	}
+
+	if err := m.db.Put(emoji); err != nil {
+		l.Debugf("database error while processing emoji: %s", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("database error while processing emoji: %s", err)})
+		return
+	}
+
+	c.JSON(http.StatusOK, mastoEmoji)
+}
+
+func validateCreateEmoji(form *mastotypes.EmojiCreateRequest) error {
+	// check there actually is an image attached and it's not size 0
+	if form.Image == nil || form.Image.Size == 0 {
+		return errors.New("no emoji given")
+	}
+
+	// a very superficial check to see if the media size limit is exceeded
+	if form.Image.Size > media.EmojiMaxBytes {
+		return fmt.Errorf("file size limit exceeded: limit is %d bytes but emoji was %d bytes", media.EmojiMaxBytes, form.Image.Size)
+	}
+
+	return util.ValidateEmojiShortcode(form.Shortcode)
+}

internal/apimodule/fileserver/fileserver.go

@@ -36,9 +36,7 @@ const (
 	mediaTypeKey = "media_type"
 	mediaSizeKey = "media_size"
 	fileNameKey  = "file_name"
-	shortcodeKey = "shortcode"
 
-	emojisPath = "emojis"
 	filesPath  = "files"
 )
 
@@ -66,7 +64,6 @@ func New(config *config.Config, db db.DB, storage storage.Storage, log *logrus.L
 // Route satisfies the RESTAPIModule interface
 func (m *fileServer) Route(s router.Router) error {
 	s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, accountIDKey, mediaTypeKey, mediaSizeKey, fileNameKey), m.ServeFile)
-	s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, accountIDKey, mediaTypeKey, mediaSizeKey, fileNameKey), m.serveEmoji)
 	return nil
 }
 

internal/apimodule/fileserver/serveemoji.go

@@ -1,7 +0,0 @@
-package fileserver
-
-import "github.com/gin-gonic/gin"
-
-func (m *fileServer) serveEmoji(c *gin.Context) {
-
-}

internal/apimodule/fileserver/servefile.go

@@ -75,12 +75,24 @@ func (m *fileServer) ServeFile(c *gin.Context) {
 
 	// Only serve media types that are defined in our internal media module
 	switch mediaType {
-	case media.MediaHeader, media.MediaAvatar, media.MediaAttachment, media.MediaEmoji:
+	case media.MediaHeader, media.MediaAvatar, media.MediaAttachment:
-	default:
+		m.serveAttachment(c, accountID, mediaType, mediaSize, fileName)
-		l.Debugf("mediatype %s not recognized", mediaType)
+		return
-		c.String(http.StatusNotFound, "404 page not found")
+	case media.MediaEmoji:
+		m.serveEmoji(c, accountID, mediaType, mediaSize, fileName)
 		return
 	}
+	l.Debugf("mediatype %s not recognized", mediaType)
+	c.String(http.StatusNotFound, "404 page not found")
+}
+
+func (m *fileServer) serveAttachment(c *gin.Context, accountID string, mediaType string, mediaSize string, fileName string) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "serveAttachment",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
 
 	// This corresponds to original-sized image as it was uploaded, small (which is the thumbnail), or static
 	switch mediaSize {
@@ -147,3 +159,83 @@ func (m *fileServer) ServeFile(c *gin.Context) {
 	// finally we can return with all the information we derived above
 	c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(attachmentBytes), map[string]string{})
 }
+
+func (m *fileServer) serveEmoji(c *gin.Context, accountID string, mediaType string, mediaSize string, fileName string) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "serveEmoji",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
+
+	// This corresponds to original-sized emoji as it was uploaded, or static
+	switch mediaSize {
+	case media.MediaOriginal, media.MediaStatic:
+	default:
+		l.Debugf("mediasize %s not recognized", mediaSize)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// derive the media id and the file extension from the last part of the request
+	spl := strings.Split(fileName, ".")
+	if len(spl) != 2 {
+		l.Debugf("filename %s not parseable", fileName)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+	wantedEmojiID := spl[0]
+	fileExtension := spl[1]
+	if wantedEmojiID == "" || fileExtension == "" {
+		l.Debugf("filename %s not parseable", fileName)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// now we know the attachment ID that the caller is asking for we can use it to pull the attachment out of the db
+	emoji := &gtsmodel.Emoji{}
+	if err := m.db.GetByID(wantedEmojiID, emoji); err != nil {
+		l.Debugf("emoji with id %s not retrievable: %s", wantedEmojiID, err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// make sure the instance account id owns the requested emoji
+	instanceAccount := &gtsmodel.Account{}
+	if err := m.db.GetWhere("username", m.config.Host, instanceAccount); err != nil {
+		l.Debugf("error fetching instance account: %s", err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+	if accountID != instanceAccount.ID {
+		l.Debugf("account %s does not own emoji with id %s", accountID, wantedEmojiID)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// now we can start preparing the response depending on whether we're serving a thumbnail or a larger attachment
+	var storagePath string
+	var contentType string
+	var contentLength int
+	switch mediaSize {
+	case media.MediaOriginal:
+		storagePath = emoji.ImagePath
+		contentType = emoji.ImageContentType
+		contentLength = emoji.ImageFileSize
+	case media.MediaStatic:
+		storagePath = emoji.ImageStaticPath
+		contentType = "image/png"
+		contentLength = emoji.ImageStaticFileSize
+	}
+
+	// use the path listed on the emoji we pulled out of the database to retrieve the object from storage
+	emojiBytes, err := m.storage.RetrieveFileFrom(storagePath)
+	if err != nil {
+		l.Debugf("error retrieving emoji from storage: %s", err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// finally we can return with all the information we derived above
+	c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(emojiBytes), map[string]string{})
+}

internal/apimodule/status/status.go

@@ -20,6 +20,7 @@ package status
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule"
@@ -75,7 +76,7 @@ func New(config *config.Config, db db.DB, oauthServer oauth.Server, mediaHandler
 
 // Route attaches all routes from this module to the given router
 func (m *statusModule) Route(r router.Router) error {
-	// r.AttachHandler(http.MethodPost, basePath, m.accountCreatePOSTHandler)
+	r.AttachHandler(http.MethodPost, basePath, m.statusCreatePOSTHandler)
 	// r.AttachHandler(http.MethodGet, basePathWithID, m.muxHandler)
 	return nil
 }
@@ -102,3 +103,14 @@ func (m *statusModule) CreateTables(db db.DB) error {
 	}
 	return nil
 }
+
+// func (m *statusModule) muxHandler(c *gin.Context) {
+// 	ru := c.Request.RequestURI
+// 	if strings.HasPrefix(ru, verifyPath) {
+// 		m.accountVerifyGETHandler(c)
+// 	} else if strings.HasPrefix(ru, updateCredentialsPath) {
+// 		m.accountUpdateCredentialsPATCHHandler(c)
+// 	} else {
+// 		m.accountGETHandler(c)
+// 	}
+// }

internal/apimodule/status/statuscreate.go

@@ -232,6 +232,16 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
 		return
 	}
 
+	mastoEmojis := []mastotypes.Emoji{}
+	for _, gtse := range newStatus.GTSEmojis {
+		me, err := m.mastoConverter.EmojiToMasto(gtse)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		mastoEmojis = append(mastoEmojis, me)
+	}
+
 	mastoStatus := &mastotypes.Status{
 		ID:                 newStatus.ID,
 		CreatedAt:          newStatus.CreatedAt.Format(time.RFC3339),
@@ -248,6 +258,8 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
 		Account:            mastoAccount,
 		MediaAttachments:   mastoAttachments,
 		Mentions:           mastoMentions,
+		Tags:               nil,
+		Emojis:             mastoEmojis,
 		Text:               form.Status,
 	}
 	c.JSON(http.StatusOK, mastoStatus)
@@ -320,12 +332,15 @@ func parseVisibility(form *advancedStatusCreateForm, accountDefaultVis gtsmodel.
 	// Advanced takes priority if it's set.
 	// If it's not set, take whatever masto visibility is set.
 	// If *that's* not set either, then just take the account default.
+	// If that's also not set, take the default for the whole instance.
 	if form.VisibilityAdvanced != nil {
 		gtsBasicVis = *form.VisibilityAdvanced
 	} else if form.Visibility != "" {
 		gtsBasicVis = util.ParseGTSVisFromMastoVis(form.Visibility)
-	} else {
+	} else if accountDefaultVis != "" {
 		gtsBasicVis = accountDefaultVis
+	} else {
+		gtsBasicVis = gtsmodel.VisibilityDefault
 	}
 
 	switch gtsBasicVis {

internal/apimodule/status/statuscreate_test.go

@@ -161,6 +161,47 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
 	assert.Equal(suite.T(), mastomodel.VisibilityPrivate, statusReply.Visibility)
 }
 
+func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
+
+	t := suite.testTokens["local_account_1"]
+	oauthToken := oauth.PGTokenToOauthToken(t)
+
+	// setup
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
+	ctx.Set(oauth.SessionAuthorizedToken, oauthToken)
+	ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
+	ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
+	ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", basePath), nil) // the endpoint we're hitting
+	ctx.Request.Form = url.Values{
+		"status":              {"here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: \n here's an emoji that isn't in the db: :test_emoji: "},
+	}
+	suite.statusModule.statusCreatePOSTHandler(ctx)
+
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	assert.NoError(suite.T(), err)
+
+	statusReply := &mastomodel.Status{}
+	err = json.Unmarshal(b, statusReply)
+	assert.NoError(suite.T(), err)
+
+	assert.Equal(suite.T(), "", statusReply.SpoilerText)
+	assert.Equal(suite.T(), "here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: \n here's an emoji that isn't in the db: :test_emoji: ", statusReply.Content)
+
+	assert.Len(suite.T(), statusReply.Emojis, 1)
+	mastoEmoji := statusReply.Emojis[0]
+	gtsEmoji := testrig.NewTestEmojis()["rainbow"]
+
+	assert.Equal(suite.T(), gtsEmoji.Shortcode, mastoEmoji.Shortcode)
+	assert.Equal(suite.T(), gtsEmoji.ImageURL, mastoEmoji.URL)
+	assert.Equal(suite.T(), gtsEmoji.ImageStaticURL, mastoEmoji.StaticURL)
+}
+
 // Try to reply to a status that doesn't exist
 func (suite *StatusCreateTestSuite) TestReplyToNonexistentStatus() {
 	t := suite.testTokens["local_account_1"]

internal/db/gtsmodel/emoji.go

@@ -25,9 +25,9 @@ type Emoji struct {
 	ID string `pg:"type:uuid,default:gen_random_uuid(),pk,notnull"`
 	// String shortcode for this emoji -- the part that's between colons. This should be lowercase a-z_
 	// eg., 'blob_hug' 'purple_heart' Must be unique with domain.
-	Shortcode string `pg:"notnull,unique:shortcodedomain"`
+	Shortcode string `pg:",notnull,unique:shortcodedomain"`
-	// Origin domain of this emoji, eg 'example.org', 'queer.party'. Null for local emojis.
+	// Origin domain of this emoji, eg 'example.org', 'queer.party'. empty string for local emojis.
-	Domain string `pg:",unique:shortcodedomain"`
+	Domain string `pg:",notnull,default:'',use_zero,unique:shortcodedomain"`
 	// When was this emoji created. Must be unique with shortcode.
 	CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
 	// When was this emoji updated

internal/db/gtsmodel/status.go

@@ -49,7 +49,7 @@ type Status struct {
 	// cw string for this status
 	ContentWarning string
 	// visibility entry for this status
-	Visibility Visibility
+	Visibility Visibility `pg:",notnull"`
 	// mark the status as sensitive?
 	Sensitive bool
 	// what language is this status written in?
@@ -95,6 +95,8 @@ const (
 	VisibilityMutualsOnly Visibility = "mutuals_only"
 	// This status is visible only to mentioned recipients
 	VisibilityDirect Visibility = "direct"
+	// Default visibility to use when no other setting can be found
+	VisibilityDefault Visibility = "public"
 )
 
 type VisibilityAdvanced struct {

internal/distributor/distributor.go

@@ -19,7 +19,6 @@
 package distributor
 
 import (
-	"github.com/go-fed/activity/pub"
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
 )
@@ -43,7 +42,7 @@ type Distributor interface {
 
 // distributor just implements the Distributor interface
 type distributor struct {
-	federator     pub.FederatingActor
+	// federator     pub.FederatingActor
 	fromClientAPI chan FromClientAPI
 	toClientAPI   chan ToClientAPI
 	stop          chan interface{}
@@ -51,9 +50,9 @@ type distributor struct {
 }
 
 // New returns a new Distributor that uses the given federator and logger
-func New(federator pub.FederatingActor, log *logrus.Logger) Distributor {
+func New(log *logrus.Logger) Distributor {
 	return &distributor{
-		federator:     federator,
+		// federator:     federator,
 		fromClientAPI: make(chan FromClientAPI, 100),
 		toClientAPI:   make(chan ToClientAPI, 100),
 		stop:          make(chan interface{}),

internal/gotosocial/actions.go

@@ -29,13 +29,16 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/action"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule/account"
+	"github.com/superseriousbusiness/gotosocial/internal/apimodule/admin"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule/app"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule/auth"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule/fileserver"
 	mediaModule "github.com/superseriousbusiness/gotosocial/internal/apimodule/media"
+	"github.com/superseriousbusiness/gotosocial/internal/apimodule/status"
 	"github.com/superseriousbusiness/gotosocial/internal/cache"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/distributor"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/mastotypes"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -51,10 +54,6 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
 
-	if err := dbService.CreateInstanceAccount(); err != nil {
-		return fmt.Errorf("error creating instance account: %s", err)
-	}
-
 	router, err := router.New(c, log)
 	if err != nil {
 		return fmt.Errorf("error creating router: %s", err)
@@ -68,6 +67,10 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
 	// build backend handlers
 	mediaHandler := media.New(c, dbService, storageBackend, log)
 	oauthServer := oauth.New(dbService, log)
+	distributor := distributor.New(log)
+	if err := distributor.Start(); err != nil {
+		return fmt.Errorf("error starting distributor: %s", err)
+	}
 
 	// build converters and util
 	mastoConverter := mastotypes.New(c, dbService)
@@ -78,6 +81,8 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
 	appsModule := app.New(oauthServer, dbService, mastoConverter, log)
 	mm := mediaModule.New(dbService, mediaHandler, mastoConverter, c, log)
 	fileServerModule := fileserver.New(c, dbService, storageBackend, log)
+	adminModule := admin.New(c, dbService, mediaHandler, mastoConverter, log)
+	statusModule := status.New(c, dbService, oauthServer, mediaHandler, mastoConverter, distributor, log)
 
 	apiModules := []apimodule.ClientAPIModule{
 		authModule, // this one has to go first so the other modules use its middleware
@@ -85,6 +90,8 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
 		appsModule,
 		mm,
 		fileServerModule,
+		adminModule,
+		statusModule,
 	}
 
 	for _, m := range apiModules {
@@ -96,6 +103,10 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
 		}
 	}
 
+	if err := dbService.CreateInstanceAccount(); err != nil {
+		return fmt.Errorf("error creating instance account: %s", err)
+	}
+
 	gts, err := New(dbService, &cache.MockCache{}, router, federation.New(dbService, log), c)
 	if err != nil {
 		return fmt.Errorf("error creating gotosocial service: %s", err)

internal/mastotypes/converter.go

@@ -52,9 +52,14 @@ type Converter interface {
 	// fields sanitized so that it can be served to non-authorized accounts without revealing any private information.
 	AppToMastoPublic(application *gtsmodel.Application) (*mastotypes.Application, error)
 
+	// AttachmentToMasto converts a gts model media attacahment into its mastodon representation for serialization on the API.
 	AttachmentToMasto(attachment *gtsmodel.MediaAttachment) (mastotypes.Attachment, error)
 
+	// MentionToMasto converts a gts model mention into its mastodon (frontend) representation for serialization on the API.
 	MentionToMasto(m *gtsmodel.Mention) (mastotypes.Mention, error)
+
+	// EmojiToMasto converts a gts model emoji into its mastodon (frontend) representation for serialization on the API.
+	EmojiToMasto(e *gtsmodel.Emoji) (mastotypes.Emoji, error)
 }
 
 type converter struct {
@@ -62,6 +67,7 @@ type converter struct {
 	db     db.DB
 }
 
+// New returns a new Converter
 func New(config *config.Config, db db.DB) Converter {
 	return &converter{
 		config: config,
@@ -290,3 +296,13 @@ func (c *converter) MentionToMasto(m *gtsmodel.Mention) (mastotypes.Mention, err
 		Acct:     acct,
 	}, nil
 }
+
+func (c *converter) EmojiToMasto(e *gtsmodel.Emoji) (mastotypes.Emoji, error) {
+	return mastotypes.Emoji{
+		Shortcode:       e.Shortcode,
+		URL:             e.ImageURL,
+		StaticURL:       e.ImageStaticURL,
+		VisibleInPicker: e.VisibleInPicker,
+		Category:        e.CategoryID,
+	}, nil
+}

internal/mastotypes/mastomodel/emoji.go

@@ -18,6 +18,8 @@
 
 package mastotypes
 
+import "mime/multipart"
+
 // Emoji represents a custom emoji. See https://docs.joinmastodon.org/entities/emoji/
 type Emoji struct {
 	// REQUIRED
@@ -36,3 +38,11 @@ type Emoji struct {
 	// Used for sorting custom emoji in the picker.
 	Category string `json:"category,omitempty"`
 }
+
+// EmojiCreateRequest represents a request to create a custom emoji made through the admin API.
+type EmojiCreateRequest struct {
+	// Desired shortcode for the emoji, without surrounding colons. This must be unique for the domain.
+	Shortcode string                `form:"shortcode" validation:"required"`
+	// Image file to use for the emoji. Must be png or gif and no larger than 50kb.
+	Image     *multipart.FileHeader `form:"image" validation:"required"`
+}

internal/media/media.go

@@ -33,15 +33,23 @@ import (
 )
 
 const (
+	// Key for small/thumbnail versions of media
 	MediaSmall      = "small"
+	// Key for original/fullsize versions of media and emoji
 	MediaOriginal   = "original"
+	// Key for static (non-animated) versions of emoji
 	MediaStatic     = "static"
+	// Key for media attachments
 	MediaAttachment = "attachment"
+	// Key for profile header
 	MediaHeader     = "header"
+	// Key for profile avatar
 	MediaAvatar     = "avatar"
+	// Key for emoji type
 	MediaEmoji      = "emoji"
 
-	emojiMaxBytes = 51200
+	// Maximum permitted bytes of an emoji upload (50kb)
+	EmojiMaxBytes = 51200
 )
 
 // MediaHandler provides an interface for parsing, storing, and retrieving media objects like photos, videos, and gifs.
@@ -55,6 +63,11 @@ type MediaHandler interface {
 	// puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media,
 	// and then returns information to the caller about the attachment.
 	ProcessLocalAttachment(attachment []byte, accountID string) (*gtsmodel.MediaAttachment, error)
+
+	// ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new
+	// *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct
+	// in the database.
+	ProcessLocalEmoji(emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error)
 }
 
 type mediaHandler struct {
@@ -165,8 +178,8 @@ func (mh *mediaHandler) ProcessLocalEmoji(emojiBytes []byte, shortcode string) (
 	if len(emojiBytes) == 0 {
 		return nil, errors.New("emoji was of size 0")
 	}
-	if len(emojiBytes) > emojiMaxBytes {
+	if len(emojiBytes) > EmojiMaxBytes {
-		return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), emojiMaxBytes)
+		return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), EmojiMaxBytes)
 	}
 
 	// clean any exif data from image/png type but leave gifs alone
@@ -227,7 +240,7 @@ func (mh *mediaHandler) ProcessLocalEmoji(emojiBytes []byte, shortcode string) (
 	}
 
 	// store the static
-	if err := mh.storage.StoreFileAt(emojiPath, static.image); err != nil {
+	if err := mh.storage.StoreFileAt(emojiStaticPath, static.image); err != nil {
 		return nil, fmt.Errorf("storage error: %s", err)
 	}
 

internal/media/media_test.go

@@ -115,6 +115,11 @@ func (suite *MediaTestSuite) SetupTest() {
 			logrus.Panicf("db connection error: %s", err)
 		}
 	}
+
+	err := suite.db.CreateInstanceAccount()
+	if err != nil {
+		logrus.Panic(err)
+	}
 }
 
 // TearDownTest drops tables to make sure there's no data in the db
@@ -151,6 +156,15 @@ func (suite *MediaTestSuite) TestSetHeaderOrAvatarForAccountID() {
 	//TODO: add more checks here, cba right now!
 }
 
+func (suite *MediaTestSuite) TestProcessLocalEmoji() {
+	f, err := ioutil.ReadFile("./test/rainbow-original.png")
+	assert.NoError(suite.T(), err)
+
+	emoji, err := suite.mediaHandler.ProcessLocalEmoji(f, "rainbow")
+	assert.NoError(suite.T(), err)
+	suite.log.Debugf("%+v", emoji)
+}
+
 // TODO: add tests for sad path, gif, png....
 
 func TestMediaTestSuite(t *testing.T) {

internal/media/util.go

@@ -248,6 +248,7 @@ func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMet
 	}, nil
 }
 
+// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png.
 func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) {
 	var i image.Image
 	var err error

internal/util/parse.go

@@ -25,6 +25,7 @@ import (
 	mastotypes "github.com/superseriousbusiness/gotosocial/internal/mastotypes/mastomodel"
 )
 
+// URIs contains a bunch of URIs and URLs for a user, host, account, etc.
 type URIs struct {
 	HostURL     string
 	UserURL     string
@@ -38,6 +39,7 @@ type URIs struct {
 	CollectionURI string
 }
 
+// GenerateURIs throws together a bunch of URIs for the given username, with the given protocol and host.
 func GenerateURIs(username string, protocol string, host string) *URIs {
 	hostURL := fmt.Sprintf("%s://%s", protocol, host)
 	userURL := fmt.Sprintf("%s/@%s", hostURL, username)
@@ -74,8 +76,6 @@ func ParseGTSVisFromMastoVis(m mastotypes.Visibility) gtsmodel.Visibility {
 		return gtsmodel.VisibilityFollowersOnly
 	case mastotypes.VisibilityDirect:
 		return gtsmodel.VisibilityDirect
-	default:
-		break
 	}
 	return ""
 }
@@ -91,8 +91,6 @@ func ParseMastoVisFromGTSVis(m gtsmodel.Visibility) mastotypes.Visibility {
 		return mastotypes.VisibilityPrivate
 	case gtsmodel.VisibilityDirect:
 		return mastotypes.VisibilityDirect
-	default:
-		break
 	}
 	return ""
 }

internal/util/regexes.go

@@ -0,0 +1,36 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package util
+
+import "regexp"
+
+var (
+	// mention regex can be played around with here: https://regex101.com/r/qwM9D3/1
+	mentionRegexString = `(?: |^|\W)(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?: |\n)`
+	mentionRegex       = regexp.MustCompile(mentionRegexString)
+	// hashtag regex can be played with here: https://regex101.com/r/Vhy8pg/1
+	hashtagRegexString = `(?: |^|\W)?#([a-zA-Z0-9]{1,30})(?:\b|\r)`
+	hashtagRegex       = regexp.MustCompile(hashtagRegexString)
+	// emoji regex can be played with here: https://regex101.com/r/478XGM/1
+	emojiRegexString = `(?: |^|\W)?:([a-zA-Z0-9_]{2,30}):(?:\b|\r)?`
+	emojiRegex       = regexp.MustCompile(emojiRegexString)
+	// emoji shortcode regex can be played with here: https://regex101.com/r/zMDRaG/1
+	emojiShortcodeString = `^[a-z0-9_]{2,30}$`
+	emojiShortcodeRegex  = regexp.MustCompile(emojiShortcodeString)
+)

internal/util/status.go

@@ -19,22 +19,9 @@
 package util
 
 import (
-	"regexp"
 	"strings"
 )
 
-var (
-	// mention regex can be played around with here: https://regex101.com/r/qwM9D3/1
-	mentionRegexString = `(?: |^|\W)(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?: |\n)`
-	mentionRegex       = regexp.MustCompile(mentionRegexString)
-	// hashtag regex can be played with here: https://regex101.com/r/Vhy8pg/1
-	hashtagRegexString = `(?: |^|\W)?#([a-zA-Z0-9]{1,30})(?:\b|\r)`
-	hashtagRegex       = regexp.MustCompile(hashtagRegexString)
-	// emoji regex can be played with here: https://regex101.com/r/478XGM/1
-	emojiRegexString = `(?: |^|\W)?:([a-zA-Z0-9_]{2,30}):(?:\b|\r)?`
-	emojiRegex       = regexp.MustCompile(emojiRegexString)
-)
-
 // DeriveMentions takes a plaintext (ie., not html-formatted) status,
 // and applies a regex to it to return a deduplicated list of accounts
 // mentioned in that status.

internal/util/validation.go

@@ -142,3 +142,13 @@ func ValidatePrivacy(privacy string) error {
 	// TODO: add some validation logic here -- length, characters, etc
 	return nil
 }
+
+// ValidateEmojiShortcode just runs the given shortcode through the regular expression
+// for emoji shortcodes, to figure out whether it's a valid shortcode, ie., 2-30 characters,
+// lowercase a-z, numbers, and underscores.
+func ValidateEmojiShortcode(shortcode string) error {
+	if !emojiShortcodeRegex.MatchString(shortcode) {
+		return fmt.Errorf("shortcode %s did not pass validation, must be between 2 and 30 characters, lowercase letters, numbers, and underscores only", shortcode)
+	}
+	return nil
+}

testrig/db.go

@@ -40,6 +40,7 @@ var testModels []interface{} = []interface{}{
 	&gtsmodel.Status{},
 	&gtsmodel.Tag{},
 	&gtsmodel.User{},
+	&gtsmodel.Emoji{},
 	&oauth.Token{},
 	&oauth.Client{},
 }
@@ -106,6 +107,12 @@ func StandardDBSetup(db db.DB) {
 		}
 	}
 
+	for _, v := range NewTestEmojis() {
+		if err := db.Put(v); err != nil {
+			panic(err)
+		}
+	}
+
 	if err := db.CreateInstanceAccount(); err != nil {
 		panic(err)
 	}

testrig/distributor.go

@@ -21,5 +21,5 @@ package testrig
 import "github.com/superseriousbusiness/gotosocial/internal/distributor"
 
 func NewTestDistributor() distributor.Distributor {
-	return distributor.New(nil, NewTestLog())
+	return distributor.New(NewTestLog())
 }

testrig/storage.go

@@ -36,9 +36,9 @@ func NewTestStorage() storage.Storage {
 
 // StandardStorageSetup populates the storage with standard test entries from the given directory.
 func StandardStorageSetup(s storage.Storage, relativePath string) {
-	stored := NewTestStored()
+	storedA := NewTestStoredAttachments()
 	a := NewTestAttachments()
-	for k, paths := range stored {
+	for k, paths := range storedA {
 		attachmentInfo, ok := a[k]
 		if !ok {
 			panic(fmt.Errorf("key %s not found in test attachments", k))
@@ -62,6 +62,33 @@ func StandardStorageSetup(s storage.Storage, relativePath string) {
 			panic(err)
 		}
 	}
+
+	storedE := NewTestStoredEmoji()
+	e := NewTestEmojis()
+	for k, paths := range storedE {
+		emojiInfo, ok := e[k]
+		if !ok {
+			panic(fmt.Errorf("key %s not found in test emojis", k))
+		}
+		filenameOriginal := paths.original
+		filenameStatic := paths.static
+		pathOriginal := emojiInfo.ImagePath
+		pathStatic := emojiInfo.ImageStaticPath
+		bOriginal, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameOriginal))
+		if err != nil {
+			panic(err)
+		}
+		if err := s.StoreFileAt(pathOriginal, bOriginal); err != nil {
+			panic(err)
+		}
+		bStatic, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameStatic))
+		if err != nil {
+			panic(err)
+		}
+		if err := s.StoreFileAt(pathStatic, bStatic); err != nil {
+			panic(err)
+		}
+	}
 }
 
 // StandardStorageTeardown deletes everything in storage so that it's clean for the next test

testrig/testmodels.go

@@ -206,6 +206,10 @@ func NewTestUsers() map[string]*gtsmodel.User {
 // NewTestAccounts returns a map of accounts keyed by what type of account they are.
 func NewTestAccounts() map[string]*gtsmodel.Account {
 	accounts := map[string]*gtsmodel.Account{
+		"instance_account": {
+			ID:       "39b745a3-774d-4b65-8bb2-b63d9e20a343",
+			Username: "localhost:8080",
+		},
 		"unconfirmed_account": {
 			ID:                    "59e197f5-87cd-4be8-ac7c-09082ccc4b4d",
 			Username:              "weed_lord420",
@@ -610,14 +614,41 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 	}
 }
 
-type paths struct {
+func NewTestEmojis() map[string]*gtsmodel.Emoji {
-	original string
+	return map[string]*gtsmodel.Emoji{
-	small    string
+		"rainbow": {
+			ID:                   "a96ec4f3-1cae-47e4-a508-f9d66a6b221b",
+			Shortcode:            "rainbow",
+			Domain:               "",
+			CreatedAt:            time.Now(),
+			UpdatedAt:            time.Now(),
+			ImageRemoteURL:       "",
+			ImageStaticRemoteURL: "",
+			ImageURL:             "http://localhost:8080/fileserver/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/original/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
+			ImagePath:            "/tmp/gotosocial/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/original/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
+			ImageStaticURL:       "http://localhost:8080/fileserver/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/static/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
+			ImageStaticPath:      "/tmp/gotosocial/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/static/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png",
+			ImageContentType:     "image/png",
+			ImageFileSize:        36702,
+			ImageStaticFileSize:  10413,
+			ImageUpdatedAt:       time.Now(),
+			Disabled:             false,
+			URI:                  "http://localhost:8080/emoji/a96ec4f3-1cae-47e4-a508-f9d66a6b221b",
+			VisibleInPicker:      true,
+			CategoryID:           "",
+		},
+	}
 }
 
-// NewTestStored returns a map of filenames, keyed according to which attachment they pertain to.
+type filenames struct {
-func NewTestStored() map[string]paths {
+	original string
-	return map[string]paths{
+	small    string
+	static   string
+}
+
+// NewTestStoredAttachments returns a map of filenames, keyed according to which attachment they pertain to.
+func NewTestStoredAttachments() map[string]filenames {
+	return map[string]filenames{
 		"admin_account_status_1_attachment_1": {
 			original: "welcome-original.jpeg",
 			small:    "welcome-small.jpeg",
@@ -633,6 +664,15 @@ func NewTestStored() map[string]paths {
 	}
 }
 
+func NewTestStoredEmoji() map[string]filenames {
+	return map[string]filenames{
+		"rainbow": {
+			original: "rainbow-original.png",
+			static:   "rainbow-static.png",
+		},
+	}
+}
+
 // NewTestStatuses returns a map of statuses keyed according to which account
 // and status they are.
 func NewTestStatuses() map[string]*gtsmodel.Status {

testrig/util.go

@@ -47,7 +47,6 @@ func CreateMultipartFormData(fieldName string, fileName string, extraFields map[
 		return b, nil, err
 	}
 
-	if extraFields != nil {
 	for k, v := range extraFields {
 		f, err := w.CreateFormField(k)
 		if err != nil {
@@ -57,7 +56,6 @@ func CreateMultipartFormData(fieldName string, fileName string, extraFields map[
 			return b, nil, err
 		}
 	}
-	}
 
 	if err := w.Close(); err != nil {
 		return b, nil, err