work on emojis

internal/apimodule/admin/admin.go

@@ -0,0 +1,88 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/apimodule"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/mastotypes"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/router"
+)
+
+const (
+	idKey                 = "id"
+	basePath              = "/api/v1/admin"
+	emojiPath             = basePath + "/custom_emojis"
+	basePathWithID        = basePath + "/:" + idKey
+	verifyPath            = basePath + "/verify_credentials"
+	updateCredentialsPath = basePath + "/update_credentials"
+)
+
+type adminModule struct {
+	config         *config.Config
+	db             db.DB
+	mediaHandler   media.MediaHandler
+	mastoConverter mastotypes.Converter
+	log            *logrus.Logger
+}
+
+// New returns a new account module
+func New(config *config.Config, db db.DB, mediaHandler media.MediaHandler, mastoConverter mastotypes.Converter, log *logrus.Logger) apimodule.ClientAPIModule {
+	return &adminModule{
+		config:         config,
+		db:             db,
+		mediaHandler:   mediaHandler,
+		mastoConverter: mastoConverter,
+		log:            log,
+	}
+}
+
+// Route attaches all routes from this module to the given router
+func (m *adminModule) Route(r router.Router) error {
+	r.AttachHandler(http.MethodPost, emojiPath, m.emojiCreatePOSTHandler)
+	return nil
+}
+
+func (m *adminModule) CreateTables(db db.DB) error {
+	models := []interface{}{
+		&gtsmodel.User{},
+		&gtsmodel.Account{},
+		&gtsmodel.Follow{},
+		&gtsmodel.FollowRequest{},
+		&gtsmodel.Status{},
+		&gtsmodel.Application{},
+		&gtsmodel.EmailDomainBlock{},
+		&gtsmodel.MediaAttachment{},
+		&gtsmodel.Emoji{},
+	}
+
+	for _, m := range models {
+		if err := db.CreateTable(m); err != nil {
+			return fmt.Errorf("error creating table: %s", err)
+		}
+	}
+	return nil
+}

internal/apimodule/admin/emojicreate.go

@@ -0,0 +1,130 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	mastotypes "github.com/superseriousbusiness/gotosocial/internal/mastotypes/mastomodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (m *adminModule) emojiCreatePOSTHandler(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "emojiCreatePOSTHandler",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
+
+	// make sure we're authed with an admin account
+	authed, err := oauth.MustAuth(c, true, true, true, true) // posting a status is serious business so we want *everything*
+	if err != nil {
+		l.Debugf("couldn't auth: %s", err)
+		c.JSON(http.StatusForbidden, gin.H{"error": err.Error()})
+		return
+	}
+	if !authed.User.Admin {
+		l.Debugf("user %s not an admin", authed.User.ID)
+		c.JSON(http.StatusForbidden, gin.H{"error": "not an admin"})
+		return
+	}
+
+	// extract the media create form from the request context
+	l.Tracef("parsing request form: %+v", c.Request.Form)
+	form := &mastotypes.EmojiCreateRequest{}
+	if err := c.ShouldBind(form); err != nil {
+		l.Debugf("error parsing form %+v: %s", c.Request.Form, err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not parse form: %s", err)})
+		return
+	}
+
+	// Give the fields on the request form a first pass to make sure the request is superficially valid.
+	l.Tracef("validating form %+v", form)
+	if err := validateCreateEmoji(form); err != nil {
+		l.Debugf("error validating form: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	// open the emoji and extract the bytes from it
+	f, err := form.Image.Open()
+	if err != nil {
+		l.Debugf("error opening emoji: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not open provided emoji: %s", err)})
+		return
+	}
+	buf := new(bytes.Buffer)
+	size, err := io.Copy(buf, f)
+	if err != nil {
+		l.Debugf("error reading emoji: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not read provided emoji: %s", err)})
+		return
+	}
+	if size == 0 {
+		l.Debug("could not read provided emoji: size 0 bytes")
+		c.JSON(http.StatusBadRequest, gin.H{"error": "could not read provided emoji: size 0 bytes"})
+		return
+	}
+
+	// allow the mediaHandler to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using
+	emoji, err := m.mediaHandler.ProcessLocalEmoji(buf.Bytes(), form.Shortcode)
+	if err != nil {
+		l.Debugf("error reading emoji: %s", err)
+		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not process emoji: %s", err)})
+		return
+	}
+
+	mastoEmoji, err := m.mastoConverter.EmojiToMasto(emoji)
+	if err != nil {
+		l.Debugf("error converting emoji to mastotype: %s", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("could not convert emoji: %s", err)})
+		return
+	}
+
+	if err := m.db.Put(emoji); err != nil {
+		l.Debugf("database error while processing emoji: %s", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("database error while processing emoji: %s", err)})
+		return
+	}
+
+	c.JSON(http.StatusOK, mastoEmoji)
+}
+
+func validateCreateEmoji(form *mastotypes.EmojiCreateRequest) error {
+	// check there actually is an image attached and it's not size 0
+	if form.Image == nil || form.Image.Size == 0 {
+		return errors.New("no emoji given")
+	}
+
+	// a very superficial check to see if the media size limit is exceeded
+	if form.Image.Size > media.EmojiMaxBytes {
+		return fmt.Errorf("file size limit exceeded: limit is %d bytes but emoji was %d bytes", media.EmojiMaxBytes, form.Image.Size)
+	}
+
+	return util.ValidateEmojiShortcode(form.Shortcode)
+}

internal/apimodule/fileserver/fileserver.go

@@ -36,9 +36,7 @@ const (
 	mediaTypeKey = "media_type"
 	mediaSizeKey = "media_size"
 	fileNameKey  = "file_name"
-	shortcodeKey = "shortcode"
 
-	emojisPath = "emojis"
 	filesPath  = "files"
 )
 
@@ -66,7 +64,6 @@ func New(config *config.Config, db db.DB, storage storage.Storage, log *logrus.L
 // Route satisfies the RESTAPIModule interface
 func (m *fileServer) Route(s router.Router) error {
 	s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, accountIDKey, mediaTypeKey, mediaSizeKey, fileNameKey), m.ServeFile)
-	s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, accountIDKey, mediaTypeKey, mediaSizeKey, fileNameKey), m.serveEmoji)
 	return nil
 }
 

internal/apimodule/fileserver/serveemoji.go

@@ -1,7 +0,0 @@
-package fileserver
-
-import "github.com/gin-gonic/gin"
-
-func (m *fileServer) serveEmoji(c *gin.Context) {
-
-}

internal/apimodule/fileserver/servefile.go

@@ -75,12 +75,24 @@ func (m *fileServer) ServeFile(c *gin.Context) {
 
 	// Only serve media types that are defined in our internal media module
 	switch mediaType {
-	case media.MediaHeader, media.MediaAvatar, media.MediaAttachment, media.MediaEmoji:
-	default:
-		l.Debugf("mediatype %s not recognized", mediaType)
-		c.String(http.StatusNotFound, "404 page not found")
+	case media.MediaHeader, media.MediaAvatar, media.MediaAttachment:
+		m.serveAttachment(c, accountID, mediaType, mediaSize, fileName)
+		return
+	case media.MediaEmoji:
+		m.serveEmoji(c, accountID, mediaType, mediaSize, fileName)
 		return
 	}
+	l.Debugf("mediatype %s not recognized", mediaType)
+	c.String(http.StatusNotFound, "404 page not found")
+}
+
+func (m *fileServer) serveAttachment(c *gin.Context, accountID string, mediaType string, mediaSize string, fileName string) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "serveAttachment",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
 
 	// This corresponds to original-sized image as it was uploaded, small (which is the thumbnail), or static
 	switch mediaSize {
@@ -147,3 +159,83 @@ func (m *fileServer) ServeFile(c *gin.Context) {
 	// finally we can return with all the information we derived above
 	c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(attachmentBytes), map[string]string{})
 }
+
+func (m *fileServer) serveEmoji(c *gin.Context, accountID string, mediaType string, mediaSize string, fileName string) {
+	l := m.log.WithFields(logrus.Fields{
+		"func":        "serveEmoji",
+		"request_uri": c.Request.RequestURI,
+		"user_agent":  c.Request.UserAgent(),
+		"origin_ip":   c.ClientIP(),
+	})
+
+	// This corresponds to original-sized emoji as it was uploaded, or static
+	switch mediaSize {
+	case media.MediaOriginal, media.MediaStatic:
+	default:
+		l.Debugf("mediasize %s not recognized", mediaSize)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// derive the media id and the file extension from the last part of the request
+	spl := strings.Split(fileName, ".")
+	if len(spl) != 2 {
+		l.Debugf("filename %s not parseable", fileName)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+	wantedEmojiID := spl[0]
+	fileExtension := spl[1]
+	if wantedEmojiID == "" || fileExtension == "" {
+		l.Debugf("filename %s not parseable", fileName)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// now we know the attachment ID that the caller is asking for we can use it to pull the attachment out of the db
+	emoji := &gtsmodel.Emoji{}
+	if err := m.db.GetByID(wantedEmojiID, emoji); err != nil {
+		l.Debugf("emoji with id %s not retrievable: %s", wantedEmojiID, err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// make sure the instance account id owns the requested emoji
+	instanceAccount := &gtsmodel.Account{}
+	if err := m.db.GetWhere("username", m.config.Host, instanceAccount); err != nil {
+		l.Debugf("error fetching instance account: %s", err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+	if accountID != instanceAccount.ID {
+		l.Debugf("account %s does not own emoji with id %s", accountID, wantedEmojiID)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// now we can start preparing the response depending on whether we're serving a thumbnail or a larger attachment
+	var storagePath string
+	var contentType string
+	var contentLength int
+	switch mediaSize {
+	case media.MediaOriginal:
+		storagePath = emoji.ImagePath
+		contentType = emoji.ImageContentType
+		contentLength = emoji.ImageFileSize
+	case media.MediaStatic:
+		storagePath = emoji.ImageStaticPath
+		contentType = "image/png"
+		contentLength = emoji.ImageStaticFileSize
+	}
+
+	// use the path listed on the emoji we pulled out of the database to retrieve the object from storage
+	emojiBytes, err := m.storage.RetrieveFileFrom(storagePath)
+	if err != nil {
+		l.Debugf("error retrieving emoji from storage: %s", err)
+		c.String(http.StatusNotFound, "404 page not found")
+		return
+	}
+
+	// finally we can return with all the information we derived above
+	c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(emojiBytes), map[string]string{})
+}

internal/apimodule/status/status.go

@@ -20,6 +20,7 @@ package status
 
 import (
 	"fmt"
+	"net/http"
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/apimodule"
@@ -75,7 +76,7 @@ func New(config *config.Config, db db.DB, oauthServer oauth.Server, mediaHandler
 
 // Route attaches all routes from this module to the given router
 func (m *statusModule) Route(r router.Router) error {
-	// r.AttachHandler(http.MethodPost, basePath, m.accountCreatePOSTHandler)
+	r.AttachHandler(http.MethodPost, basePath, m.statusCreatePOSTHandler)
 	// r.AttachHandler(http.MethodGet, basePathWithID, m.muxHandler)
 	return nil
 }
@@ -102,3 +103,14 @@ func (m *statusModule) CreateTables(db db.DB) error {
 	}
 	return nil
 }
+
+// func (m *statusModule) muxHandler(c *gin.Context) {
+// 	ru := c.Request.RequestURI
+// 	if strings.HasPrefix(ru, verifyPath) {
+// 		m.accountVerifyGETHandler(c)
+// 	} else if strings.HasPrefix(ru, updateCredentialsPath) {
+// 		m.accountUpdateCredentialsPATCHHandler(c)
+// 	} else {
+// 		m.accountGETHandler(c)
+// 	}
+// }

internal/apimodule/status/statuscreate.go

@@ -232,6 +232,16 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
 		return
 	}
 
+	mastoEmojis := []mastotypes.Emoji{}
+	for _, gtse := range newStatus.GTSEmojis {
+		me, err := m.mastoConverter.EmojiToMasto(gtse)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		mastoEmojis = append(mastoEmojis, me)
+	}
+
 	mastoStatus := &mastotypes.Status{
 		ID:                 newStatus.ID,
 		CreatedAt:          newStatus.CreatedAt.Format(time.RFC3339),
@@ -248,6 +258,8 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
 		Account:            mastoAccount,
 		MediaAttachments:   mastoAttachments,
 		Mentions:           mastoMentions,
+		Tags:               nil,
+		Emojis:             mastoEmojis,
 		Text:               form.Status,
 	}
 	c.JSON(http.StatusOK, mastoStatus)
@@ -320,12 +332,15 @@ func parseVisibility(form *advancedStatusCreateForm, accountDefaultVis gtsmodel.
 	// Advanced takes priority if it's set.
 	// If it's not set, take whatever masto visibility is set.
 	// If *that's* not set either, then just take the account default.
+	// If that's also not set, take the default for the whole instance.
 	if form.VisibilityAdvanced != nil {
 		gtsBasicVis = *form.VisibilityAdvanced
 	} else if form.Visibility != "" {
 		gtsBasicVis = util.ParseGTSVisFromMastoVis(form.Visibility)
-	} else {
+	} else if accountDefaultVis != "" {
 		gtsBasicVis = accountDefaultVis
+	} else {
+		gtsBasicVis = gtsmodel.VisibilityDefault
 	}
 
 	switch gtsBasicVis {

internal/apimodule/status/statuscreate_test.go

@@ -161,6 +161,47 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
 	assert.Equal(suite.T(), mastomodel.VisibilityPrivate, statusReply.Visibility)
 }
 
+func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
+
+	t := suite.testTokens["local_account_1"]
+	oauthToken := oauth.PGTokenToOauthToken(t)
+
+	// setup
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
+	ctx.Set(oauth.SessionAuthorizedToken, oauthToken)
+	ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
+	ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
+	ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", basePath), nil) // the endpoint we're hitting
+	ctx.Request.Form = url.Values{
+		"status":              {"here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: \n here's an emoji that isn't in the db: :test_emoji: "},
+	}
+	suite.statusModule.statusCreatePOSTHandler(ctx)
+
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	assert.NoError(suite.T(), err)
+
+	statusReply := &mastomodel.Status{}
+	err = json.Unmarshal(b, statusReply)
+	assert.NoError(suite.T(), err)
+
+	assert.Equal(suite.T(), "", statusReply.SpoilerText)
+	assert.Equal(suite.T(), "here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: \n here's an emoji that isn't in the db: :test_emoji: ", statusReply.Content)
+
+	assert.Len(suite.T(), statusReply.Emojis, 1)
+	mastoEmoji := statusReply.Emojis[0]
+	gtsEmoji := testrig.NewTestEmojis()["rainbow"]
+
+	assert.Equal(suite.T(), gtsEmoji.Shortcode, mastoEmoji.Shortcode)
+	assert.Equal(suite.T(), gtsEmoji.ImageURL, mastoEmoji.URL)
+	assert.Equal(suite.T(), gtsEmoji.ImageStaticURL, mastoEmoji.StaticURL)
+}
+
 // Try to reply to a status that doesn't exist
 func (suite *StatusCreateTestSuite) TestReplyToNonexistentStatus() {
 	t := suite.testTokens["local_account_1"]