diff --git a/PROGRESS.md b/PROGRESS.md index 89ada4aa7..9a41805c8 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -144,6 +144,7 @@ * [ ] Custom Emojis * [ ] /api/v1/custom_emojis GET (Show this server's custom emoji) * [ ] Admin + * [x] /api/v1/admin/custom_emojis POST (Upload a custom emoji for instance-wide usage) * [ ] /api/v1/admin/accounts GET (View accounts filtered by criteria) * [ ] /api/v1/admin/accounts/:id GET (View admin level info about an account) * [ ] /api/v1/admin/accounts/:id/action POST (Perform an admin action on account) diff --git a/internal/apimodule/admin/admin.go b/internal/apimodule/admin/admin.go new file mode 100644 index 000000000..81d00116f --- /dev/null +++ b/internal/apimodule/admin/admin.go @@ -0,0 +1,88 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package admin + +import ( + "fmt" + "net/http" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/apimodule" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/mastotypes" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/router" +) + +const ( + idKey = "id" + basePath = "/api/v1/admin" + emojiPath = basePath + "/custom_emojis" + basePathWithID = basePath + "/:" + idKey + verifyPath = basePath + "/verify_credentials" + updateCredentialsPath = basePath + "/update_credentials" +) + +type adminModule struct { + config *config.Config + db db.DB + mediaHandler media.MediaHandler + mastoConverter mastotypes.Converter + log *logrus.Logger +} + +// New returns a new account module +func New(config *config.Config, db db.DB, mediaHandler media.MediaHandler, mastoConverter mastotypes.Converter, log *logrus.Logger) apimodule.ClientAPIModule { + return &adminModule{ + config: config, + db: db, + mediaHandler: mediaHandler, + mastoConverter: mastoConverter, + log: log, + } +} + +// Route attaches all routes from this module to the given router +func (m *adminModule) Route(r router.Router) error { + r.AttachHandler(http.MethodPost, emojiPath, m.emojiCreatePOSTHandler) + return nil +} + +func (m *adminModule) CreateTables(db db.DB) error { + models := []interface{}{ + >smodel.User{}, + >smodel.Account{}, + >smodel.Follow{}, + >smodel.FollowRequest{}, + >smodel.Status{}, + >smodel.Application{}, + >smodel.EmailDomainBlock{}, + >smodel.MediaAttachment{}, + >smodel.Emoji{}, + } + + for _, m := range models { + if err := db.CreateTable(m); err != nil { + return fmt.Errorf("error creating table: %s", err) + } + } + return nil +} diff --git a/internal/apimodule/admin/emojicreate.go b/internal/apimodule/admin/emojicreate.go new file mode 100644 index 000000000..91457c07c --- /dev/null +++ b/internal/apimodule/admin/emojicreate.go @@ -0,0 +1,130 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package admin + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + mastotypes "github.com/superseriousbusiness/gotosocial/internal/mastotypes/mastomodel" + "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +func (m *adminModule) emojiCreatePOSTHandler(c *gin.Context) { + l := m.log.WithFields(logrus.Fields{ + "func": "emojiCreatePOSTHandler", + "request_uri": c.Request.RequestURI, + "user_agent": c.Request.UserAgent(), + "origin_ip": c.ClientIP(), + }) + + // make sure we're authed with an admin account + authed, err := oauth.MustAuth(c, true, true, true, true) // posting a status is serious business so we want *everything* + if err != nil { + l.Debugf("couldn't auth: %s", err) + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } + if !authed.User.Admin { + l.Debugf("user %s not an admin", authed.User.ID) + c.JSON(http.StatusForbidden, gin.H{"error": "not an admin"}) + return + } + + // extract the media create form from the request context + l.Tracef("parsing request form: %+v", c.Request.Form) + form := &mastotypes.EmojiCreateRequest{} + if err := c.ShouldBind(form); err != nil { + l.Debugf("error parsing form %+v: %s", c.Request.Form, err) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not parse form: %s", err)}) + return + } + + // Give the fields on the request form a first pass to make sure the request is superficially valid. + l.Tracef("validating form %+v", form) + if err := validateCreateEmoji(form); err != nil { + l.Debugf("error validating form: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + // open the emoji and extract the bytes from it + f, err := form.Image.Open() + if err != nil { + l.Debugf("error opening emoji: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not open provided emoji: %s", err)}) + return + } + buf := new(bytes.Buffer) + size, err := io.Copy(buf, f) + if err != nil { + l.Debugf("error reading emoji: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not read provided emoji: %s", err)}) + return + } + if size == 0 { + l.Debug("could not read provided emoji: size 0 bytes") + c.JSON(http.StatusBadRequest, gin.H{"error": "could not read provided emoji: size 0 bytes"}) + return + } + + // allow the mediaHandler to work its magic of processing the emoji bytes, and putting them in whatever storage backend we're using + emoji, err := m.mediaHandler.ProcessLocalEmoji(buf.Bytes(), form.Shortcode) + if err != nil { + l.Debugf("error reading emoji: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not process emoji: %s", err)}) + return + } + + mastoEmoji, err := m.mastoConverter.EmojiToMasto(emoji) + if err != nil { + l.Debugf("error converting emoji to mastotype: %s", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("could not convert emoji: %s", err)}) + return + } + + if err := m.db.Put(emoji); err != nil { + l.Debugf("database error while processing emoji: %s", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("database error while processing emoji: %s", err)}) + return + } + + c.JSON(http.StatusOK, mastoEmoji) +} + +func validateCreateEmoji(form *mastotypes.EmojiCreateRequest) error { + // check there actually is an image attached and it's not size 0 + if form.Image == nil || form.Image.Size == 0 { + return errors.New("no emoji given") + } + + // a very superficial check to see if the media size limit is exceeded + if form.Image.Size > media.EmojiMaxBytes { + return fmt.Errorf("file size limit exceeded: limit is %d bytes but emoji was %d bytes", media.EmojiMaxBytes, form.Image.Size) + } + + return util.ValidateEmojiShortcode(form.Shortcode) +} diff --git a/internal/apimodule/fileserver/fileserver.go b/internal/apimodule/fileserver/fileserver.go index 825afecf4..e57e27627 100644 --- a/internal/apimodule/fileserver/fileserver.go +++ b/internal/apimodule/fileserver/fileserver.go @@ -36,9 +36,7 @@ const ( mediaTypeKey = "media_type" mediaSizeKey = "media_size" fileNameKey = "file_name" - shortcodeKey = "shortcode" - emojisPath = "emojis" filesPath = "files" ) @@ -66,7 +64,6 @@ func New(config *config.Config, db db.DB, storage storage.Storage, log *logrus.L // Route satisfies the RESTAPIModule interface func (m *fileServer) Route(s router.Router) error { s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, accountIDKey, mediaTypeKey, mediaSizeKey, fileNameKey), m.ServeFile) - s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, accountIDKey, mediaTypeKey, mediaSizeKey, fileNameKey), m.serveEmoji) return nil } diff --git a/internal/apimodule/fileserver/serveemoji.go b/internal/apimodule/fileserver/serveemoji.go deleted file mode 100644 index 062e59afe..000000000 --- a/internal/apimodule/fileserver/serveemoji.go +++ /dev/null @@ -1,7 +0,0 @@ -package fileserver - -import "github.com/gin-gonic/gin" - -func (m *fileServer) serveEmoji(c *gin.Context) { - -} diff --git a/internal/apimodule/fileserver/servefile.go b/internal/apimodule/fileserver/servefile.go index 5813f4b33..66bc18542 100644 --- a/internal/apimodule/fileserver/servefile.go +++ b/internal/apimodule/fileserver/servefile.go @@ -75,12 +75,24 @@ func (m *fileServer) ServeFile(c *gin.Context) { // Only serve media types that are defined in our internal media module switch mediaType { - case media.MediaHeader, media.MediaAvatar, media.MediaAttachment, media.MediaEmoji: - default: - l.Debugf("mediatype %s not recognized", mediaType) - c.String(http.StatusNotFound, "404 page not found") + case media.MediaHeader, media.MediaAvatar, media.MediaAttachment: + m.serveAttachment(c, accountID, mediaType, mediaSize, fileName) + return + case media.MediaEmoji: + m.serveEmoji(c, accountID, mediaType, mediaSize, fileName) return } + l.Debugf("mediatype %s not recognized", mediaType) + c.String(http.StatusNotFound, "404 page not found") +} + +func (m *fileServer) serveAttachment(c *gin.Context, accountID string, mediaType string, mediaSize string, fileName string) { + l := m.log.WithFields(logrus.Fields{ + "func": "serveAttachment", + "request_uri": c.Request.RequestURI, + "user_agent": c.Request.UserAgent(), + "origin_ip": c.ClientIP(), + }) // This corresponds to original-sized image as it was uploaded, small (which is the thumbnail), or static switch mediaSize { @@ -147,3 +159,83 @@ func (m *fileServer) ServeFile(c *gin.Context) { // finally we can return with all the information we derived above c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(attachmentBytes), map[string]string{}) } + +func (m *fileServer) serveEmoji(c *gin.Context, accountID string, mediaType string, mediaSize string, fileName string) { + l := m.log.WithFields(logrus.Fields{ + "func": "serveEmoji", + "request_uri": c.Request.RequestURI, + "user_agent": c.Request.UserAgent(), + "origin_ip": c.ClientIP(), + }) + + // This corresponds to original-sized emoji as it was uploaded, or static + switch mediaSize { + case media.MediaOriginal, media.MediaStatic: + default: + l.Debugf("mediasize %s not recognized", mediaSize) + c.String(http.StatusNotFound, "404 page not found") + return + } + + // derive the media id and the file extension from the last part of the request + spl := strings.Split(fileName, ".") + if len(spl) != 2 { + l.Debugf("filename %s not parseable", fileName) + c.String(http.StatusNotFound, "404 page not found") + return + } + wantedEmojiID := spl[0] + fileExtension := spl[1] + if wantedEmojiID == "" || fileExtension == "" { + l.Debugf("filename %s not parseable", fileName) + c.String(http.StatusNotFound, "404 page not found") + return + } + + // now we know the attachment ID that the caller is asking for we can use it to pull the attachment out of the db + emoji := >smodel.Emoji{} + if err := m.db.GetByID(wantedEmojiID, emoji); err != nil { + l.Debugf("emoji with id %s not retrievable: %s", wantedEmojiID, err) + c.String(http.StatusNotFound, "404 page not found") + return + } + + // make sure the instance account id owns the requested emoji + instanceAccount := >smodel.Account{} + if err := m.db.GetWhere("username", m.config.Host, instanceAccount); err != nil { + l.Debugf("error fetching instance account: %s", err) + c.String(http.StatusNotFound, "404 page not found") + return + } + if accountID != instanceAccount.ID { + l.Debugf("account %s does not own emoji with id %s", accountID, wantedEmojiID) + c.String(http.StatusNotFound, "404 page not found") + return + } + + // now we can start preparing the response depending on whether we're serving a thumbnail or a larger attachment + var storagePath string + var contentType string + var contentLength int + switch mediaSize { + case media.MediaOriginal: + storagePath = emoji.ImagePath + contentType = emoji.ImageContentType + contentLength = emoji.ImageFileSize + case media.MediaStatic: + storagePath = emoji.ImageStaticPath + contentType = "image/png" + contentLength = emoji.ImageStaticFileSize + } + + // use the path listed on the emoji we pulled out of the database to retrieve the object from storage + emojiBytes, err := m.storage.RetrieveFileFrom(storagePath) + if err != nil { + l.Debugf("error retrieving emoji from storage: %s", err) + c.String(http.StatusNotFound, "404 page not found") + return + } + + // finally we can return with all the information we derived above + c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(emojiBytes), map[string]string{}) +} diff --git a/internal/apimodule/status/status.go b/internal/apimodule/status/status.go index 02ae77f7c..123ca0a56 100644 --- a/internal/apimodule/status/status.go +++ b/internal/apimodule/status/status.go @@ -20,6 +20,7 @@ package status import ( "fmt" + "net/http" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/apimodule" @@ -75,7 +76,7 @@ func New(config *config.Config, db db.DB, oauthServer oauth.Server, mediaHandler // Route attaches all routes from this module to the given router func (m *statusModule) Route(r router.Router) error { - // r.AttachHandler(http.MethodPost, basePath, m.accountCreatePOSTHandler) + r.AttachHandler(http.MethodPost, basePath, m.statusCreatePOSTHandler) // r.AttachHandler(http.MethodGet, basePathWithID, m.muxHandler) return nil } @@ -102,3 +103,14 @@ func (m *statusModule) CreateTables(db db.DB) error { } return nil } + +// func (m *statusModule) muxHandler(c *gin.Context) { +// ru := c.Request.RequestURI +// if strings.HasPrefix(ru, verifyPath) { +// m.accountVerifyGETHandler(c) +// } else if strings.HasPrefix(ru, updateCredentialsPath) { +// m.accountUpdateCredentialsPATCHHandler(c) +// } else { +// m.accountGETHandler(c) +// } +// } diff --git a/internal/apimodule/status/statuscreate.go b/internal/apimodule/status/statuscreate.go index 17360b125..1ae3d7135 100644 --- a/internal/apimodule/status/statuscreate.go +++ b/internal/apimodule/status/statuscreate.go @@ -232,6 +232,16 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) { return } + mastoEmojis := []mastotypes.Emoji{} + for _, gtse := range newStatus.GTSEmojis { + me, err := m.mastoConverter.EmojiToMasto(gtse) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + mastoEmojis = append(mastoEmojis, me) + } + mastoStatus := &mastotypes.Status{ ID: newStatus.ID, CreatedAt: newStatus.CreatedAt.Format(time.RFC3339), @@ -248,6 +258,8 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) { Account: mastoAccount, MediaAttachments: mastoAttachments, Mentions: mastoMentions, + Tags: nil, + Emojis: mastoEmojis, Text: form.Status, } c.JSON(http.StatusOK, mastoStatus) @@ -320,12 +332,15 @@ func parseVisibility(form *advancedStatusCreateForm, accountDefaultVis gtsmodel. // Advanced takes priority if it's set. // If it's not set, take whatever masto visibility is set. // If *that's* not set either, then just take the account default. + // If that's also not set, take the default for the whole instance. if form.VisibilityAdvanced != nil { gtsBasicVis = *form.VisibilityAdvanced } else if form.Visibility != "" { gtsBasicVis = util.ParseGTSVisFromMastoVis(form.Visibility) - } else { + } else if accountDefaultVis != "" { gtsBasicVis = accountDefaultVis + } else { + gtsBasicVis = gtsmodel.VisibilityDefault } switch gtsBasicVis { diff --git a/internal/apimodule/status/statuscreate_test.go b/internal/apimodule/status/statuscreate_test.go index c87ba9e36..03b3d2d33 100644 --- a/internal/apimodule/status/statuscreate_test.go +++ b/internal/apimodule/status/statuscreate_test.go @@ -161,6 +161,47 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() { assert.Equal(suite.T(), mastomodel.VisibilityPrivate, statusReply.Visibility) } +func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() { + + t := suite.testTokens["local_account_1"] + oauthToken := oauth.PGTokenToOauthToken(t) + + // setup + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) + ctx.Set(oauth.SessionAuthorizedToken, oauthToken) + ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"]) + ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"]) + ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", basePath), nil) // the endpoint we're hitting + ctx.Request.Form = url.Values{ + "status": {"here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: \n here's an emoji that isn't in the db: :test_emoji: "}, + } + suite.statusModule.statusCreatePOSTHandler(ctx) + + suite.EqualValues(http.StatusOK, recorder.Code) + + result := recorder.Result() + defer result.Body.Close() + b, err := ioutil.ReadAll(result.Body) + assert.NoError(suite.T(), err) + + statusReply := &mastomodel.Status{} + err = json.Unmarshal(b, statusReply) + assert.NoError(suite.T(), err) + + assert.Equal(suite.T(), "", statusReply.SpoilerText) + assert.Equal(suite.T(), "here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow: \n here's an emoji that isn't in the db: :test_emoji: ", statusReply.Content) + + assert.Len(suite.T(), statusReply.Emojis, 1) + mastoEmoji := statusReply.Emojis[0] + gtsEmoji := testrig.NewTestEmojis()["rainbow"] + + assert.Equal(suite.T(), gtsEmoji.Shortcode, mastoEmoji.Shortcode) + assert.Equal(suite.T(), gtsEmoji.ImageURL, mastoEmoji.URL) + assert.Equal(suite.T(), gtsEmoji.ImageStaticURL, mastoEmoji.StaticURL) +} + // Try to reply to a status that doesn't exist func (suite *StatusCreateTestSuite) TestReplyToNonexistentStatus() { t := suite.testTokens["local_account_1"] diff --git a/internal/db/gtsmodel/emoji.go b/internal/db/gtsmodel/emoji.go index fbd5aedf9..da1e2e02c 100644 --- a/internal/db/gtsmodel/emoji.go +++ b/internal/db/gtsmodel/emoji.go @@ -25,9 +25,9 @@ type Emoji struct { ID string `pg:"type:uuid,default:gen_random_uuid(),pk,notnull"` // String shortcode for this emoji -- the part that's between colons. This should be lowercase a-z_ // eg., 'blob_hug' 'purple_heart' Must be unique with domain. - Shortcode string `pg:"notnull,unique:shortcodedomain"` - // Origin domain of this emoji, eg 'example.org', 'queer.party'. Null for local emojis. - Domain string `pg:",unique:shortcodedomain"` + Shortcode string `pg:",notnull,unique:shortcodedomain"` + // Origin domain of this emoji, eg 'example.org', 'queer.party'. empty string for local emojis. + Domain string `pg:",notnull,default:'',use_zero,unique:shortcodedomain"` // When was this emoji created. Must be unique with shortcode. CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"` // When was this emoji updated diff --git a/internal/db/gtsmodel/status.go b/internal/db/gtsmodel/status.go index 43792e9f7..04ddc8f35 100644 --- a/internal/db/gtsmodel/status.go +++ b/internal/db/gtsmodel/status.go @@ -49,7 +49,7 @@ type Status struct { // cw string for this status ContentWarning string // visibility entry for this status - Visibility Visibility + Visibility Visibility `pg:",notnull"` // mark the status as sensitive? Sensitive bool // what language is this status written in? @@ -95,6 +95,8 @@ const ( VisibilityMutualsOnly Visibility = "mutuals_only" // This status is visible only to mentioned recipients VisibilityDirect Visibility = "direct" + // Default visibility to use when no other setting can be found + VisibilityDefault Visibility = "public" ) type VisibilityAdvanced struct { diff --git a/internal/distributor/distributor.go b/internal/distributor/distributor.go index 027b32279..74b69c5b0 100644 --- a/internal/distributor/distributor.go +++ b/internal/distributor/distributor.go @@ -19,7 +19,6 @@ package distributor import ( - "github.com/go-fed/activity/pub" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel" ) @@ -43,7 +42,7 @@ type Distributor interface { // distributor just implements the Distributor interface type distributor struct { - federator pub.FederatingActor + // federator pub.FederatingActor fromClientAPI chan FromClientAPI toClientAPI chan ToClientAPI stop chan interface{} @@ -51,9 +50,9 @@ type distributor struct { } // New returns a new Distributor that uses the given federator and logger -func New(federator pub.FederatingActor, log *logrus.Logger) Distributor { +func New(log *logrus.Logger) Distributor { return &distributor{ - federator: federator, + // federator: federator, fromClientAPI: make(chan FromClientAPI, 100), toClientAPI: make(chan ToClientAPI, 100), stop: make(chan interface{}), diff --git a/internal/gotosocial/actions.go b/internal/gotosocial/actions.go index ee336a249..3f12d7b65 100644 --- a/internal/gotosocial/actions.go +++ b/internal/gotosocial/actions.go @@ -29,13 +29,16 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/action" "github.com/superseriousbusiness/gotosocial/internal/apimodule" "github.com/superseriousbusiness/gotosocial/internal/apimodule/account" + "github.com/superseriousbusiness/gotosocial/internal/apimodule/admin" "github.com/superseriousbusiness/gotosocial/internal/apimodule/app" "github.com/superseriousbusiness/gotosocial/internal/apimodule/auth" "github.com/superseriousbusiness/gotosocial/internal/apimodule/fileserver" mediaModule "github.com/superseriousbusiness/gotosocial/internal/apimodule/media" + "github.com/superseriousbusiness/gotosocial/internal/apimodule/status" "github.com/superseriousbusiness/gotosocial/internal/cache" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/distributor" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/mastotypes" "github.com/superseriousbusiness/gotosocial/internal/media" @@ -51,10 +54,6 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr return fmt.Errorf("error creating dbservice: %s", err) } - if err := dbService.CreateInstanceAccount(); err != nil { - return fmt.Errorf("error creating instance account: %s", err) - } - router, err := router.New(c, log) if err != nil { return fmt.Errorf("error creating router: %s", err) @@ -68,6 +67,10 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr // build backend handlers mediaHandler := media.New(c, dbService, storageBackend, log) oauthServer := oauth.New(dbService, log) + distributor := distributor.New(log) + if err := distributor.Start(); err != nil { + return fmt.Errorf("error starting distributor: %s", err) + } // build converters and util mastoConverter := mastotypes.New(c, dbService) @@ -78,6 +81,8 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr appsModule := app.New(oauthServer, dbService, mastoConverter, log) mm := mediaModule.New(dbService, mediaHandler, mastoConverter, c, log) fileServerModule := fileserver.New(c, dbService, storageBackend, log) + adminModule := admin.New(c, dbService, mediaHandler, mastoConverter, log) + statusModule := status.New(c, dbService, oauthServer, mediaHandler, mastoConverter, distributor, log) apiModules := []apimodule.ClientAPIModule{ authModule, // this one has to go first so the other modules use its middleware @@ -85,6 +90,8 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr appsModule, mm, fileServerModule, + adminModule, + statusModule, } for _, m := range apiModules { @@ -96,6 +103,10 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr } } + if err := dbService.CreateInstanceAccount(); err != nil { + return fmt.Errorf("error creating instance account: %s", err) + } + gts, err := New(dbService, &cache.MockCache{}, router, federation.New(dbService, log), c) if err != nil { return fmt.Errorf("error creating gotosocial service: %s", err) diff --git a/internal/mastotypes/converter.go b/internal/mastotypes/converter.go index 0e81f1a06..cec138058 100644 --- a/internal/mastotypes/converter.go +++ b/internal/mastotypes/converter.go @@ -52,9 +52,14 @@ type Converter interface { // fields sanitized so that it can be served to non-authorized accounts without revealing any private information. AppToMastoPublic(application *gtsmodel.Application) (*mastotypes.Application, error) + // AttachmentToMasto converts a gts model media attacahment into its mastodon representation for serialization on the API. AttachmentToMasto(attachment *gtsmodel.MediaAttachment) (mastotypes.Attachment, error) + // MentionToMasto converts a gts model mention into its mastodon (frontend) representation for serialization on the API. MentionToMasto(m *gtsmodel.Mention) (mastotypes.Mention, error) + + // EmojiToMasto converts a gts model emoji into its mastodon (frontend) representation for serialization on the API. + EmojiToMasto(e *gtsmodel.Emoji) (mastotypes.Emoji, error) } type converter struct { @@ -62,6 +67,7 @@ type converter struct { db db.DB } +// New returns a new Converter func New(config *config.Config, db db.DB) Converter { return &converter{ config: config, @@ -290,3 +296,13 @@ func (c *converter) MentionToMasto(m *gtsmodel.Mention) (mastotypes.Mention, err Acct: acct, }, nil } + +func (c *converter) EmojiToMasto(e *gtsmodel.Emoji) (mastotypes.Emoji, error) { + return mastotypes.Emoji{ + Shortcode: e.Shortcode, + URL: e.ImageURL, + StaticURL: e.ImageStaticURL, + VisibleInPicker: e.VisibleInPicker, + Category: e.CategoryID, + }, nil +} diff --git a/internal/mastotypes/mastomodel/emoji.go b/internal/mastotypes/mastomodel/emoji.go index e9ef95460..e9a0322bc 100644 --- a/internal/mastotypes/mastomodel/emoji.go +++ b/internal/mastotypes/mastomodel/emoji.go @@ -18,6 +18,8 @@ package mastotypes +import "mime/multipart" + // Emoji represents a custom emoji. See https://docs.joinmastodon.org/entities/emoji/ type Emoji struct { // REQUIRED @@ -36,3 +38,11 @@ type Emoji struct { // Used for sorting custom emoji in the picker. Category string `json:"category,omitempty"` } + +// EmojiCreateRequest represents a request to create a custom emoji made through the admin API. +type EmojiCreateRequest struct { + // Desired shortcode for the emoji, without surrounding colons. This must be unique for the domain. + Shortcode string `form:"shortcode" validation:"required"` + // Image file to use for the emoji. Must be png or gif and no larger than 50kb. + Image *multipart.FileHeader `form:"image" validation:"required"` +} diff --git a/internal/media/media.go b/internal/media/media.go index 28b58ba39..93a899e00 100644 --- a/internal/media/media.go +++ b/internal/media/media.go @@ -33,15 +33,23 @@ import ( ) const ( + // Key for small/thumbnail versions of media MediaSmall = "small" + // Key for original/fullsize versions of media and emoji MediaOriginal = "original" + // Key for static (non-animated) versions of emoji MediaStatic = "static" + // Key for media attachments MediaAttachment = "attachment" + // Key for profile header MediaHeader = "header" + // Key for profile avatar MediaAvatar = "avatar" + // Key for emoji type MediaEmoji = "emoji" - emojiMaxBytes = 51200 + // Maximum permitted bytes of an emoji upload (50kb) + EmojiMaxBytes = 51200 ) // MediaHandler provides an interface for parsing, storing, and retrieving media objects like photos, videos, and gifs. @@ -55,6 +63,11 @@ type MediaHandler interface { // puts it in whatever storage backend we're using, sets the relevant fields in the database for the new media, // and then returns information to the caller about the attachment. ProcessLocalAttachment(attachment []byte, accountID string) (*gtsmodel.MediaAttachment, error) + + // ProcessLocalEmoji takes a new emoji and a shortcode, cleans it up, puts it in storage, and creates a new + // *gts.Emoji for it, then returns it to the caller. It's the caller's responsibility to put the returned struct + // in the database. + ProcessLocalEmoji(emojiBytes []byte, shortcode string) (*gtsmodel.Emoji, error) } type mediaHandler struct { @@ -165,8 +178,8 @@ func (mh *mediaHandler) ProcessLocalEmoji(emojiBytes []byte, shortcode string) ( if len(emojiBytes) == 0 { return nil, errors.New("emoji was of size 0") } - if len(emojiBytes) > emojiMaxBytes { - return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), emojiMaxBytes) + if len(emojiBytes) > EmojiMaxBytes { + return nil, fmt.Errorf("emoji size %d bytes exceeded max emoji size of %d bytes", len(emojiBytes), EmojiMaxBytes) } // clean any exif data from image/png type but leave gifs alone @@ -227,7 +240,7 @@ func (mh *mediaHandler) ProcessLocalEmoji(emojiBytes []byte, shortcode string) ( } // store the static - if err := mh.storage.StoreFileAt(emojiPath, static.image); err != nil { + if err := mh.storage.StoreFileAt(emojiStaticPath, static.image); err != nil { return nil, fmt.Errorf("storage error: %s", err) } diff --git a/internal/media/media_test.go b/internal/media/media_test.go index 9022d2e20..75a544121 100644 --- a/internal/media/media_test.go +++ b/internal/media/media_test.go @@ -115,6 +115,11 @@ func (suite *MediaTestSuite) SetupTest() { logrus.Panicf("db connection error: %s", err) } } + + err := suite.db.CreateInstanceAccount() + if err != nil { + logrus.Panic(err) + } } // TearDownTest drops tables to make sure there's no data in the db @@ -151,6 +156,15 @@ func (suite *MediaTestSuite) TestSetHeaderOrAvatarForAccountID() { //TODO: add more checks here, cba right now! } +func (suite *MediaTestSuite) TestProcessLocalEmoji() { + f, err := ioutil.ReadFile("./test/rainbow-original.png") + assert.NoError(suite.T(), err) + + emoji, err := suite.mediaHandler.ProcessLocalEmoji(f, "rainbow") + assert.NoError(suite.T(), err) + suite.log.Debugf("%+v", emoji) +} + // TODO: add tests for sad path, gif, png.... func TestMediaTestSuite(t *testing.T) { diff --git a/internal/media/test/rainbow-original.png b/internal/media/test/rainbow-original.png new file mode 100644 index 000000000..fdbfaeec3 Binary files /dev/null and b/internal/media/test/rainbow-original.png differ diff --git a/internal/media/test/rainbow-static.png b/internal/media/test/rainbow-static.png new file mode 100644 index 000000000..d364b1171 Binary files /dev/null and b/internal/media/test/rainbow-static.png differ diff --git a/internal/media/util.go b/internal/media/util.go index 6501a34b8..64d1ee770 100644 --- a/internal/media/util.go +++ b/internal/media/util.go @@ -248,6 +248,7 @@ func deriveThumbnail(b []byte, contentType string, x uint, y uint) (*imageAndMet }, nil } +// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. func deriveStaticEmoji(b []byte, contentType string) (*imageAndMeta, error) { var i image.Image var err error diff --git a/internal/util/parse.go b/internal/util/parse.go index 9f3f7fad5..f0bcff5dc 100644 --- a/internal/util/parse.go +++ b/internal/util/parse.go @@ -25,6 +25,7 @@ import ( mastotypes "github.com/superseriousbusiness/gotosocial/internal/mastotypes/mastomodel" ) +// URIs contains a bunch of URIs and URLs for a user, host, account, etc. type URIs struct { HostURL string UserURL string @@ -38,6 +39,7 @@ type URIs struct { CollectionURI string } +// GenerateURIs throws together a bunch of URIs for the given username, with the given protocol and host. func GenerateURIs(username string, protocol string, host string) *URIs { hostURL := fmt.Sprintf("%s://%s", protocol, host) userURL := fmt.Sprintf("%s/@%s", hostURL, username) @@ -74,8 +76,6 @@ func ParseGTSVisFromMastoVis(m mastotypes.Visibility) gtsmodel.Visibility { return gtsmodel.VisibilityFollowersOnly case mastotypes.VisibilityDirect: return gtsmodel.VisibilityDirect - default: - break } return "" } @@ -91,8 +91,6 @@ func ParseMastoVisFromGTSVis(m gtsmodel.Visibility) mastotypes.Visibility { return mastotypes.VisibilityPrivate case gtsmodel.VisibilityDirect: return mastotypes.VisibilityDirect - default: - break } return "" } diff --git a/internal/util/regexes.go b/internal/util/regexes.go new file mode 100644 index 000000000..60b397d86 --- /dev/null +++ b/internal/util/regexes.go @@ -0,0 +1,36 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package util + +import "regexp" + +var ( + // mention regex can be played around with here: https://regex101.com/r/qwM9D3/1 + mentionRegexString = `(?: |^|\W)(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?: |\n)` + mentionRegex = regexp.MustCompile(mentionRegexString) + // hashtag regex can be played with here: https://regex101.com/r/Vhy8pg/1 + hashtagRegexString = `(?: |^|\W)?#([a-zA-Z0-9]{1,30})(?:\b|\r)` + hashtagRegex = regexp.MustCompile(hashtagRegexString) + // emoji regex can be played with here: https://regex101.com/r/478XGM/1 + emojiRegexString = `(?: |^|\W)?:([a-zA-Z0-9_]{2,30}):(?:\b|\r)?` + emojiRegex = regexp.MustCompile(emojiRegexString) + // emoji shortcode regex can be played with here: https://regex101.com/r/zMDRaG/1 + emojiShortcodeString = `^[a-z0-9_]{2,30}$` + emojiShortcodeRegex = regexp.MustCompile(emojiShortcodeString) +) diff --git a/internal/util/status.go b/internal/util/status.go index 7e4e669bf..e4b3ec6a5 100644 --- a/internal/util/status.go +++ b/internal/util/status.go @@ -19,22 +19,9 @@ package util import ( - "regexp" "strings" ) -var ( - // mention regex can be played around with here: https://regex101.com/r/qwM9D3/1 - mentionRegexString = `(?: |^|\W)(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?: |\n)` - mentionRegex = regexp.MustCompile(mentionRegexString) - // hashtag regex can be played with here: https://regex101.com/r/Vhy8pg/1 - hashtagRegexString = `(?: |^|\W)?#([a-zA-Z0-9]{1,30})(?:\b|\r)` - hashtagRegex = regexp.MustCompile(hashtagRegexString) - // emoji regex can be played with here: https://regex101.com/r/478XGM/1 - emojiRegexString = `(?: |^|\W)?:([a-zA-Z0-9_]{2,30}):(?:\b|\r)?` - emojiRegex = regexp.MustCompile(emojiRegexString) -) - // DeriveMentions takes a plaintext (ie., not html-formatted) status, // and applies a regex to it to return a deduplicated list of accounts // mentioned in that status. diff --git a/internal/util/validation.go b/internal/util/validation.go index 88a56875c..8102bc35d 100644 --- a/internal/util/validation.go +++ b/internal/util/validation.go @@ -142,3 +142,13 @@ func ValidatePrivacy(privacy string) error { // TODO: add some validation logic here -- length, characters, etc return nil } + +// ValidateEmojiShortcode just runs the given shortcode through the regular expression +// for emoji shortcodes, to figure out whether it's a valid shortcode, ie., 2-30 characters, +// lowercase a-z, numbers, and underscores. +func ValidateEmojiShortcode(shortcode string) error { + if !emojiShortcodeRegex.MatchString(shortcode) { + return fmt.Errorf("shortcode %s did not pass validation, must be between 2 and 30 characters, lowercase letters, numbers, and underscores only", shortcode) + } + return nil +} diff --git a/testrig/db.go b/testrig/db.go index 4d5bb6f18..260020c50 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -40,6 +40,7 @@ var testModels []interface{} = []interface{}{ >smodel.Status{}, >smodel.Tag{}, >smodel.User{}, + >smodel.Emoji{}, &oauth.Token{}, &oauth.Client{}, } @@ -106,6 +107,12 @@ func StandardDBSetup(db db.DB) { } } + for _, v := range NewTestEmojis() { + if err := db.Put(v); err != nil { + panic(err) + } + } + if err := db.CreateInstanceAccount(); err != nil { panic(err) } diff --git a/testrig/distributor.go b/testrig/distributor.go index c37f3dd26..e21321d53 100644 --- a/testrig/distributor.go +++ b/testrig/distributor.go @@ -21,5 +21,5 @@ package testrig import "github.com/superseriousbusiness/gotosocial/internal/distributor" func NewTestDistributor() distributor.Distributor { - return distributor.New(nil, NewTestLog()) + return distributor.New(NewTestLog()) } diff --git a/testrig/media/rainbow-original.png b/testrig/media/rainbow-original.png new file mode 100755 index 000000000..fdbfaeec3 Binary files /dev/null and b/testrig/media/rainbow-original.png differ diff --git a/testrig/media/rainbow-static.png b/testrig/media/rainbow-static.png new file mode 100755 index 000000000..79ed5c03a Binary files /dev/null and b/testrig/media/rainbow-static.png differ diff --git a/testrig/storage.go b/testrig/storage.go index c4ea9a951..3b520364b 100644 --- a/testrig/storage.go +++ b/testrig/storage.go @@ -36,9 +36,9 @@ func NewTestStorage() storage.Storage { // StandardStorageSetup populates the storage with standard test entries from the given directory. func StandardStorageSetup(s storage.Storage, relativePath string) { - stored := NewTestStored() + storedA := NewTestStoredAttachments() a := NewTestAttachments() - for k, paths := range stored { + for k, paths := range storedA { attachmentInfo, ok := a[k] if !ok { panic(fmt.Errorf("key %s not found in test attachments", k)) @@ -62,6 +62,33 @@ func StandardStorageSetup(s storage.Storage, relativePath string) { panic(err) } } + + storedE := NewTestStoredEmoji() + e := NewTestEmojis() + for k, paths := range storedE { + emojiInfo, ok := e[k] + if !ok { + panic(fmt.Errorf("key %s not found in test emojis", k)) + } + filenameOriginal := paths.original + filenameStatic := paths.static + pathOriginal := emojiInfo.ImagePath + pathStatic := emojiInfo.ImageStaticPath + bOriginal, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameOriginal)) + if err != nil { + panic(err) + } + if err := s.StoreFileAt(pathOriginal, bOriginal); err != nil { + panic(err) + } + bStatic, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameStatic)) + if err != nil { + panic(err) + } + if err := s.StoreFileAt(pathStatic, bStatic); err != nil { + panic(err) + } + } } // StandardStorageTeardown deletes everything in storage so that it's clean for the next test diff --git a/testrig/testmodels.go b/testrig/testmodels.go index 12c9f519a..dde38912c 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -206,6 +206,10 @@ func NewTestUsers() map[string]*gtsmodel.User { // NewTestAccounts returns a map of accounts keyed by what type of account they are. func NewTestAccounts() map[string]*gtsmodel.Account { accounts := map[string]*gtsmodel.Account{ + "instance_account": { + ID: "39b745a3-774d-4b65-8bb2-b63d9e20a343", + Username: "localhost:8080", + }, "unconfirmed_account": { ID: "59e197f5-87cd-4be8-ac7c-09082ccc4b4d", Username: "weed_lord420", @@ -610,14 +614,41 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { } } -type paths struct { - original string - small string +func NewTestEmojis() map[string]*gtsmodel.Emoji { + return map[string]*gtsmodel.Emoji{ + "rainbow": { + ID: "a96ec4f3-1cae-47e4-a508-f9d66a6b221b", + Shortcode: "rainbow", + Domain: "", + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + ImageRemoteURL: "", + ImageStaticRemoteURL: "", + ImageURL: "http://localhost:8080/fileserver/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/original/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png", + ImagePath: "/tmp/gotosocial/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/original/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png", + ImageStaticURL: "http://localhost:8080/fileserver/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/static/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png", + ImageStaticPath: "/tmp/gotosocial/39b745a3-774d-4b65-8bb2-b63d9e20a343/emoji/static/a96ec4f3-1cae-47e4-a508-f9d66a6b221b.png", + ImageContentType: "image/png", + ImageFileSize: 36702, + ImageStaticFileSize: 10413, + ImageUpdatedAt: time.Now(), + Disabled: false, + URI: "http://localhost:8080/emoji/a96ec4f3-1cae-47e4-a508-f9d66a6b221b", + VisibleInPicker: true, + CategoryID: "", + }, + } } -// NewTestStored returns a map of filenames, keyed according to which attachment they pertain to. -func NewTestStored() map[string]paths { - return map[string]paths{ +type filenames struct { + original string + small string + static string +} + +// NewTestStoredAttachments returns a map of filenames, keyed according to which attachment they pertain to. +func NewTestStoredAttachments() map[string]filenames { + return map[string]filenames{ "admin_account_status_1_attachment_1": { original: "welcome-original.jpeg", small: "welcome-small.jpeg", @@ -633,6 +664,15 @@ func NewTestStored() map[string]paths { } } +func NewTestStoredEmoji() map[string]filenames { + return map[string]filenames{ + "rainbow": { + original: "rainbow-original.png", + static: "rainbow-static.png", + }, + } +} + // NewTestStatuses returns a map of statuses keyed according to which account // and status they are. func NewTestStatuses() map[string]*gtsmodel.Status { diff --git a/testrig/util.go b/testrig/util.go index e6342d93d..96a979342 100644 --- a/testrig/util.go +++ b/testrig/util.go @@ -47,15 +47,13 @@ func CreateMultipartFormData(fieldName string, fileName string, extraFields map[ return b, nil, err } - if extraFields != nil { - for k, v := range extraFields { - f, err := w.CreateFormField(k) - if err != nil { - return b, nil, err - } - if _, err := io.Copy(f, bytes.NewBufferString(v)); err != nil { - return b, nil, err - } + for k, v := range extraFields { + f, err := w.CreateFormField(k) + if err != nil { + return b, nil, err + } + if _, err := io.Copy(f, bytes.NewBufferString(v)); err != nil { + return b, nil, err } }