mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-12-10 05:48:08 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Hashtag federation (in/out), hashtag client API endpoints (#2032)

Browse source 
* update go-fed

* do the things

* remove unused columns from tags

* update to latest lingo from main

* further tag shenanigans

* serve stub page at tag endpoint

* we did it lads

* tests, oh tests, ohhh tests, oh tests (doo doo doo doo)

* swagger docs

* document hashtag usage + federation

* instanceGet

* don't bother parsing tag href

* rename whereStartsWith -> whereStartsLike

* remove GetOrCreateTag

* dont cache status tag timelineability
This commit is contained in:
tobi 2023-07-31 15:47:35 +02:00 • committed by GitHub
commit 2796a2e82f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
69 changed files with 2536 additions and 482 deletions
Download patch file Download diff file Expand all files Collapse all files

91
internal/ap/ap_test.go
View file

@ -98,6 +98,97 @@ func noteWithMentions1() vocab.ActivityStreamsNote {
return note
}
func (suite *APTestSuite) noteWithHashtags1() ap.Statusable {
noteJson := []byte(`
{
"@context": [
"https://www.w3.org/ns/activitystreams",
{
"ostatus": "http://ostatus.org#",
"atomUri": "ostatus:atomUri",
"inReplyToAtomUri": "ostatus:inReplyToAtomUri",
"conversation": "ostatus:conversation",
"sensitive": "as:sensitive",
"toot": "http://joinmastodon.org/ns#",
"votersCount": "toot:votersCount",
"Hashtag": "as:Hashtag"
}
],
"id": "https://mastodon.social/users/pixelfed/statuses/110609702372389319",
"type": "Note",
"summary": null,
"inReplyTo": null,
"published": "2023-06-26T09:01:56Z",
"url": "https://mastodon.social/@pixelfed/110609702372389319",
"attributedTo": "https://mastodon.social/users/pixelfed",
"to": [
"https://www.w3.org/ns/activitystreams#Public"
],
"cc": [
"https://mastodon.social/users/pixelfed/followers",
"https://gts.superseriousbusiness.org/users/gotosocial"
],
"sensitive": false,
"atomUri": "https://mastodon.social/users/pixelfed/statuses/110609702372389319",
"inReplyToAtomUri": null,
"conversation": "tag:mastodon.social,2023-06-26:objectId=474977189:objectType=Conversation",
"content": "<p>⚡ Heard of <span class=\"h-card\" translate=\"no\"><a href=\"https://gts.superseriousbusiness.org/@gotosocial\" class=\"u-url mention\">@<span>gotosocial</span></a></span> ?</p><p>GoToSocial provides a lightweight, customizable, and safety-focused entryway into the <a href=\"https://mastodon.social/tags/fediverse\" class=\"mention hashtag\" rel=\"tag\">#<span>fediverse</span></a>, you can keep in touch with your friends, post, read, and share images and articles.</p><p>Consider <a href=\"https://mastodon.social/tags/GoToSocial\" class=\"mention hashtag\" rel=\"tag\">#<span>GoToSocial</span></a> instead of Pixelfed if you&#39;d like a safety-focused alternative with text-only post support that is maintained by a stellar developer community!</p><p>We ❤️ GtS, check them out!</p><p>🌍 <a href=\"https://gotosocial.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">gotosocial.org/</span><span class=\"invisible\"></span></a></p><p>🔍 <a href=\"https://fedidb.org/software/gotosocial\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">fedidb.org/software/gotosocial</span><span class=\"invisible\"></span></a></p>",
"contentMap": {
"en": "<p>⚡ Heard of <span class=\"h-card\" translate=\"no\"><a href=\"https://gts.superseriousbusiness.org/@gotosocial\" class=\"u-url mention\">@<span>gotosocial</span></a></span> ?</p><p>GoToSocial provides a lightweight, customizable, and safety-focused entryway into the <a href=\"https://mastodon.social/tags/fediverse\" class=\"mention hashtag\" rel=\"tag\">#<span>fediverse</span></a>, you can keep in touch with your friends, post, read, and share images and articles.</p><p>Consider <a href=\"https://mastodon.social/tags/GoToSocial\" class=\"mention hashtag\" rel=\"tag\">#<span>GoToSocial</span></a> instead of Pixelfed if you&#39;d like a safety-focused alternative with text-only post support that is maintained by a stellar developer community!</p><p>We ❤️ GtS, check them out!</p><p>🌍 <a href=\"https://gotosocial.org/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">gotosocial.org/</span><span class=\"invisible\"></span></a></p><p>🔍 <a href=\"https://fedidb.org/software/gotosocial\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">fedidb.org/software/gotosocial</span><span class=\"invisible\"></span></a></p>"
},
"attachment": [],
"tag": [
{
"type": "Mention",
"href": "https://gts.superseriousbusiness.org/users/gotosocial",
"name": "@gotosocial@superseriousbusiness.org"
},
{
"type": "Hashtag",
"href": "https://mastodon.social/tags/fediverse",
"name": "#fediverse"
},
{
"type": "Hashtag",
"href": "https://mastodon.social/tags/gotosocial",
"name": "#gotosocial"
},
{
"type": "Hashtag",
"href": "https://mastodon.social/tags/this_hashtag_will_be_ignored_since_it_cant_be_normalized",
"name": "#b̴̛͇̒̌͑̓̐̑͗̏̐̇͗̎̕͝O̵̧̧͎̟̰̭̊͌͒́̊̑̄̐͐͗Ọ̷̧̡̰̟̪̫̹͖͇̱͕̺̦̲̀̐̽̓̇̚͠b̶̨̖͍͙͈̹͉̯͕̯̯̯̞̼̞̏͊͂̐̔͛s̴̢̞̺͈͇̘͚͉͔̥̔͛͆͑͑̍̄̌̚͜͜ͅ"
},
{
"type": "Hashtag",
"href": "https://mastodon.social/tags/this_hashtag_will_be_included_correctly",
"name": "#Grüvy"
},
{
"type": "Hashtag",
"href": "https://mastodon.social/tags/this_hashtag_will_be_squashed_into_a_single_character",
"name": "#` + `ᄀ` + `ᅡ` + `ᆨ` + `"
}
],
"replies": {
"id": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies",
"type": "Collection",
"first": {
"type": "CollectionPage",
"next": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies?only_other_accounts=true&page=true",
"partOf": "https://mastodon.social/users/pixelfed/statuses/110609702372389319/replies",
"items": []
}
}
}`)
statusable, err := ap.ResolveStatusable(context.Background(), noteJson)
if err != nil {
suite.FailNow(err.Error())
}
return statusable
}
func addressable1() ap.Addressable {
// make a note addressed to public with followers in cc
note := streams.NewActivityStreamsNote()

43
internal/ap/extract.go
View file

@ -30,6 +30,7 @@ import (
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@ -529,7 +530,7 @@ func ExtractBlurhash(i WithBlurhash) string {
// ExtractHashtags extracts a slice of minimal gtsmodel.Tags
// from a WithTag. If an entry in the WithTag is not a hashtag,
// it will be quietly ignored.
// or has a name that cannot be normalized, it will be ignored.
//
// TODO: find a better heuristic for determining if something
// is a hashtag or not, since looking for type name "Hashtag"
@ -562,18 +563,29 @@ func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) {
continue
}
tag, err := ExtractHashtag(hashtaggable)
tag, err := extractHashtag(hashtaggable)
if err != nil {
continue
}
// "Normalize" this tag by combining diacritics +
// unicode chars. If this returns false, it means
// we couldn't normalize it well enough to make it
// valid on our instance, so just ignore it.
normalized, ok := text.NormalizeHashtag(tag.Name)
if !ok {
continue
}
// We store tag names lowercased, might
// as well change case here already.
tag.Name = strings.ToLower(normalized)
// Only append this tag if we haven't
// seen it already, to avoid duplicates
// in the slice.
if _, set := keys[tag.URL]; !set {
keys[tag.URL] = nil // Value doesn't matter.
tags = append(tags, tag)
tags = append(tags, tag)
if _, set := keys[tag.Name]; !set {
keys[tag.Name] = nil // Value doesn't matter.
tags = append(tags, tag)
}
}
@ -581,16 +593,9 @@ func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) {
return tags, nil
}
// ExtractEmoji extracts a minimal gtsmodel.Tag
// from the given Hashtaggable.
func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
// Extract href/link for this tag.
hrefProp := i.GetActivityStreamsHref()
if hrefProp == nil || !hrefProp.IsIRI() {
return nil, gtserror.New("no href prop")
}
tagURL := hrefProp.GetIRI().String()
// extractHashtag extracts a minimal gtsmodel.Tag from the given
// Hashtaggable, without yet doing any normalization on it.
func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
// Extract name for the tag; trim leading hash
// character, so '#example' becomes 'example'.
name := ExtractName(i)
@ -599,9 +604,11 @@ func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
}
tagName := strings.TrimPrefix(name, "#")
yeah := func() *bool { t := true; return &t }
return &gtsmodel.Tag{
URL: tagURL,
Name: tagName,
Name: tagName,
Useable: yeah(), // Assume true by default.
Listable: yeah(), // Assume true by default.
}, nil
}

66
internal/ap/extracthashtags_test.go Normal file
View file

@ -0,0 +1,66 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ap_test
import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/ap"
)
type ExtractHashtagsTestSuite struct {
APTestSuite
}
func (suite *ExtractHashtagsTestSuite) TestExtractHashtags1() {
note := suite.noteWithHashtags1()
hashtags, err := ap.ExtractHashtags(note)
if err != nil {
suite.FailNow(err.Error())
}
if l := len(hashtags); l != 4 {
suite.FailNow("", "expected 4 hashtags, got %d", l)
}
hashtagFediverse := hashtags[0]
suite.Equal("fediverse", hashtagFediverse.Name)
suite.Equal(true, *hashtagFediverse.Useable)
suite.Equal(true, *hashtagFediverse.Listable)
hashtagGoToSocial := hashtags[1]
suite.Equal("gotosocial", hashtagGoToSocial.Name)
suite.Equal(true, *hashtagGoToSocial.Useable)
suite.Equal(true, *hashtagGoToSocial.Listable)
hashtagGrüvy := hashtags[2]
suite.Equal("grüvy", hashtagGrüvy.Name)
suite.Equal(true, *hashtagGrüvy.Useable)
suite.Equal(true, *hashtagGrüvy.Listable)
hashtagAngle := hashtags[3]
suite.Equal("각", hashtagAngle.Name)
suite.Equal(true, *hashtagAngle.Useable)
suite.Equal(true, *hashtagAngle.Listable)
}
func TestExtractHashtagsTestSuite(t *testing.T) {
suite.Run(t, &ExtractHashtagsTestSuite{})
}

10
internal/api/client/media/media.go
View file

@ -21,16 +21,14 @@ import (
"net/http"
"github.com/gin-gonic/gin"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/processing"
)
const (
IDKey = "id" // IDKey is the key for media attachment IDs
APIVersionKey = "api_version" // APIVersionKey is the key for which version of the API to use (v1 or v2)
APIv1 = "v1" // APIV1 corresponds to version 1 of the api
APIv2 = "v2" // APIV2 corresponds to version 2 of the api
BasePath = "/:" + APIVersionKey + "/media" // BasePath is the base API path for making media requests through v1 or v2 of the api (for mastodon API compatibility)
AttachmentWithID = BasePath + "/:" + IDKey // BasePathWithID corresponds to a media attachment with the given ID
IDKey = "id" // IDKey is the key for media attachment IDs
BasePath = "/:" + apiutil.APIVersionKey + "/media" // BasePath is the base API path for making media requests through v1 or v2 of the api (for mastodon API compatibility)
AttachmentWithID = BasePath + "/:" + IDKey // BasePathWithID corresponds to a media attachment with the given ID
)
type Module struct {

12
internal/api/client/media/mediacreate.go
View file

@ -93,10 +93,12 @@ import (
// '500':
// description: internal server error
func (m *Module) MediaCreatePOSTHandler(c *gin.Context) {
apiVersion := c.Param(APIVersionKey)
if apiVersion != APIv1 && apiVersion != APIv2 {
err := errors.New("api version must be one of v1 or v2 for this path")
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGetV1)
apiVersion, errWithCode := apiutil.ParseAPIVersion(
c.Param(apiutil.APIVersionKey),
[]string{apiutil.APIv1, apiutil.APIv2}...,
)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
@ -128,7 +130,7 @@ func (m *Module) MediaCreatePOSTHandler(c *gin.Context) {
return
}
if apiVersion == APIv2 {
if apiVersion == apiutil.APIv2 {
// the mastodon v2 media API specifies that the URL should be null
// and that the client should call /api/v1/media/:id to get the URL
//

9
internal/api/client/media/mediacreate_test.go
View file

@ -32,6 +32,7 @@ import (
"github.com/stretchr/testify/suite"
mediamodule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -169,7 +170,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessful() {
ctx.Request = httptest.NewRequest(http.MethodPost, "http://localhost:8080/api/v1/media", bytes.NewReader(buf.Bytes())) // the endpoint we're hitting
ctx.Request.Header.Set("Content-Type", w.FormDataContentType())
ctx.Request.Header.Set("accept", "application/json")
ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1)
ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1)
// do the actual request
suite.mediaModule.MediaCreatePOSTHandler(ctx)
@ -254,7 +255,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateSuccessfulV2() {
ctx.Request = httptest.NewRequest(http.MethodPost, "http://localhost:8080/api/v2/media", bytes.NewReader(buf.Bytes())) // the endpoint we're hitting
ctx.Request.Header.Set("Content-Type", w.FormDataContentType())
ctx.Request.Header.Set("accept", "application/json")
ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv2)
ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv2)
// do the actual request
suite.mediaModule.MediaCreatePOSTHandler(ctx)
@ -337,7 +338,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateLongDescription() {
ctx.Request = httptest.NewRequest(http.MethodPost, "http://localhost:8080/api/v1/media", bytes.NewReader(buf.Bytes())) // the endpoint we're hitting
ctx.Request.Header.Set("Content-Type", w.FormDataContentType())
ctx.Request.Header.Set("accept", "application/json")
ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1)
ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1)
// do the actual request
suite.mediaModule.MediaCreatePOSTHandler(ctx)
@ -378,7 +379,7 @@ func (suite *MediaCreateTestSuite) TestMediaCreateTooShortDescription() {
ctx.Request = httptest.NewRequest(http.MethodPost, "http://localhost:8080/api/v1/media", bytes.NewReader(buf.Bytes())) // the endpoint we're hitting
ctx.Request.Header.Set("Content-Type", w.FormDataContentType())
ctx.Request.Header.Set("accept", "application/json")
ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1)
ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1)
// do the actual request
suite.mediaModule.MediaCreatePOSTHandler(ctx)

8
internal/api/client/media/mediaget.go
View file

@ -66,9 +66,11 @@ import (
// '500':
// description: internal server error
func (m *Module) MediaGETHandler(c *gin.Context) {
if apiVersion := c.Param(APIVersionKey); apiVersion != APIv1 {
err := errors.New("api version must be one v1 for this path")
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGetV1)
if _, errWithCode := apiutil.ParseAPIVersion(
c.Param(apiutil.APIVersionKey),
[]string{apiutil.APIv1, apiutil.APIv2}...,
); errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}

8
internal/api/client/media/mediaupdate.go
View file

@ -98,9 +98,11 @@ import (
// '500':
// description: internal server error
func (m *Module) MediaPUTHandler(c *gin.Context) {
if apiVersion := c.Param(APIVersionKey); apiVersion != APIv1 {
err := errors.New("api version must be one v1 for this path")
apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGetV1)
if _, errWithCode := apiutil.ParseAPIVersion(
c.Param(apiutil.APIVersionKey),
[]string{apiutil.APIv1, apiutil.APIv2}...,
); errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}

5
internal/api/client/media/mediaupdate_test.go
View file

@ -30,6 +30,7 @@ import (
"github.com/stretchr/testify/suite"
mediamodule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -160,7 +161,7 @@ func (suite *MediaUpdateTestSuite) TestUpdateImage() {
ctx.Request = httptest.NewRequest(http.MethodPut, fmt.Sprintf("http://localhost:8080/api/v1/media/%s", toUpdate.ID), bytes.NewReader(buf.Bytes())) // the endpoint we're hitting
ctx.Request.Header.Set("Content-Type", w.FormDataContentType())
ctx.Request.Header.Set("accept", "application/json")
ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1)
ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1)
ctx.AddParam(mediamodule.IDKey, toUpdate.ID)
// do the actual request
@ -221,7 +222,7 @@ func (suite *MediaUpdateTestSuite) TestUpdateImageShortDescription() {
ctx.Request = httptest.NewRequest(http.MethodPut, fmt.Sprintf("http://localhost:8080/api/v1/media/%s", toUpdate.ID), bytes.NewReader(buf.Bytes())) // the endpoint we're hitting
ctx.Request.Header.Set("Content-Type", w.FormDataContentType())
ctx.Request.Header.Set("accept", "application/json")
ctx.AddParam(mediamodule.APIVersionKey, mediamodule.APIv1)
ctx.AddParam(apiutil.APIVersionKey, apiutil.APIv1)
ctx.AddParam(mediamodule.IDKey, toUpdate.ID)
// do the actual request

7
internal/api/client/search/search.go
View file

@ -21,12 +21,12 @@ import (
"net/http"
"github.com/gin-gonic/gin"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/processing"
)
const (
BasePathV1 = "/v1/search" // Base path for serving v1 of the search API, minus the 'api' prefix.
BasePathV2 = "/v2/search" // Base path for serving v2 of the search API, minus the 'api' prefix.
BasePath = "/:" + apiutil.APIVersionKey + "/search"
)
type Module struct {
@ -40,6 +40,5 @@ func New(processor *processing.Processor) *Module {
}
func (m *Module) Route(attachHandler func(method string, path string, f ...gin.HandlerFunc) gin.IRoutes) {
attachHandler(http.MethodGet, BasePathV1, m.SearchGETHandler)
attachHandler(http.MethodGet, BasePathV2, m.SearchGETHandler)
attachHandler(http.MethodGet, BasePath, m.SearchGETHandler)
}

32
internal/api/client/search/searchget.go
View file

@ -27,7 +27,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oauth"
)
// SearchGETHandler swagger:operation GET /api/v1/search searchGet
// SearchGETHandler swagger:operation GET /api/{api_version}/search searchGet
//
// Search for statuses, accounts, or hashtags, on this instance or elsewhere.
//
@ -42,6 +42,15 @@ import (
//
// parameters:
// -
// name: api_version
// type: string
// in: path
// description: >-
// Version of the API to use. Must be either `v1` or `v2`.
// If v1 is used, Hashtag results will be a slice of strings.
// If v2 is used, Hashtag results will be a slice of apimodel tags.
// required: true
// -
// name: max_id
// type: string
// description: >-
@ -88,6 +97,7 @@ import (
// - `@[username]` -- search for an account with the given username on any domain. Can return multiple results.
// - @[username]@[domain]` -- search for a remote account with exact username and domain. Will only ever return 1 result at most.
// - `https://example.org/some/arbitrary/url` -- search for an account OR a status with the given URL. Will only ever return 1 result at most.
// - `#[hashtag_name]` -- search for a hashtag with the given hashtag name, or starting with the given hashtag name. Case insensitive. Can return multiple results.
// - any arbitrary string -- search for accounts or statuses containing the given string. Can return multiple results.
// in: query
// required: true
@ -97,9 +107,9 @@ import (
// description: |-
// Type of item to return. One of:
// - `` -- empty string; return any/all results.
// - `accounts` -- return account(s).
// - `statuses` -- return status(es).
// - `hashtags` -- return hashtag(s).
// - `accounts` -- return only account(s).
// - `statuses` -- return only status(es).
// - `hashtags` -- return only hashtag(s).
// If `type` is specified, paging can be performed using max_id and min_id parameters.
// If `type` is not specified, see the `offset` parameter for paging.
// in: query
@ -138,9 +148,7 @@ import (
// name: search results
// description: Results of the search.
// schema:
// type: array
// items:
// "$ref": "#/definitions/searchResult"
// "$ref": "#/definitions/searchResult"
// '400':
// description: bad request
// '401':
@ -152,6 +160,15 @@ import (
// '500':
// description: internal server error
func (m *Module) SearchGETHandler(c *gin.Context) {
apiVersion, errWithCode := apiutil.ParseAPIVersion(
c.Param(apiutil.APIVersionKey),
[]string{apiutil.APIv1, apiutil.APIv2}...,
)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
authed, err := oauth.Authed(c, true, true, true, true)
if err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
@ -209,6 +226,7 @@ func (m *Module) SearchGETHandler(c *gin.Context) {
Resolve: resolve,
Following: following,
ExcludeUnreviewed: excludeUnreviewed,
APIv1: apiVersion == apiutil.APIv1,
}
results, errWithCode := m.processor.Search().Get(c.Request.Context(), authed.Account, searchRequest)

195
internal/api/client/search/searchget_test.go
View file

@ -47,6 +47,7 @@ type SearchGetTestSuite struct {
func (suite *SearchGetTestSuite) getSearch(
requestingAccount *gtsmodel.Account,
token *gtsmodel.Token,
apiVersion string,
user *gtsmodel.User,
maxID *string,
minID *string,
@ -62,11 +63,13 @@ func (suite *SearchGetTestSuite) getSearch(
var (
recorder = httptest.NewRecorder()
ctx, _ = testrig.CreateGinTestContext(recorder, nil)
requestURL = testrig.URLMustParse("/api" + search.BasePathV1)
requestURL = testrig.URLMustParse("/api" + search.BasePath)
queryParts []string
)
// Put the request together.
ctx.AddParam(apiutil.APIVersionKey, apiVersion)
if maxID != nil {
queryParts = append(queryParts, apiutil.MaxIDKey+"="+url.QueryEscape(*maxID))
}
@ -175,6 +178,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByURI() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -218,6 +222,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestring() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -261,6 +266,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringUppercase()
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -304,6 +310,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringNoLeadingAt(
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -347,6 +354,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringNoResolve()
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -385,6 +393,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringSpecialChars
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -426,6 +435,7 @@ func (suite *SearchGetTestSuite) TestSearchRemoteAccountByNamestringSpecialChars
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -467,6 +477,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalAccountByNamestring() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -510,6 +521,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalAccountByNamestringWithDomain()
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -553,6 +565,7 @@ func (suite *SearchGetTestSuite) TestSearchNonexistingLocalAccountByNamestringRe
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -591,6 +604,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalAccountByURI() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -634,6 +648,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalAccountByURL() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -677,6 +692,7 @@ func (suite *SearchGetTestSuite) TestSearchNonexistingLocalAccountByURL() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -715,6 +731,7 @@ func (suite *SearchGetTestSuite) TestSearchStatusByURL() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -758,6 +775,7 @@ func (suite *SearchGetTestSuite) TestSearchBlockedDomainURL() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -798,6 +816,7 @@ func (suite *SearchGetTestSuite) TestSearchBlockedDomainNamestring() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -838,6 +857,7 @@ func (suite *SearchGetTestSuite) TestSearchAAny() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -878,6 +898,7 @@ func (suite *SearchGetTestSuite) TestSearchAAnyFollowingOnly() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -918,6 +939,7 @@ func (suite *SearchGetTestSuite) TestSearchAStatuses() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -958,6 +980,7 @@ func (suite *SearchGetTestSuite) TestSearchAAccounts() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -998,6 +1021,7 @@ func (suite *SearchGetTestSuite) TestSearchAAccountsLimit1() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -1038,6 +1062,7 @@ func (suite *SearchGetTestSuite) TestSearchLocalInstanceAccountByURI() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -1084,6 +1109,7 @@ func (suite *SearchGetTestSuite) TestSearchInstanceAccountFull() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -1130,6 +1156,7 @@ func (suite *SearchGetTestSuite) TestSearchInstanceAccountPartial() {
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -1170,6 +1197,7 @@ func (suite *SearchGetTestSuite) TestSearchBadQueryType() {
_, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -1206,6 +1234,7 @@ func (suite *SearchGetTestSuite) TestSearchEmptyQuery() {
_, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
@ -1222,6 +1251,170 @@ func (suite *SearchGetTestSuite) TestSearchEmptyQuery() {
}
}
func (suite *SearchGetTestSuite) TestSearchHashtagV1() {
var (
requestingAccount = suite.testAccounts["local_account_1"]
token = suite.testTokens["local_account_1"]
user = suite.testUsers["local_account_1"]
maxID *string = nil
minID *string = nil
limit *int = nil
offset *int = nil
resolve *bool = nil
query = "#welcome"
queryType *string = func() *string { i := "hashtags"; return &i }()
following *bool = nil
expectedHTTPStatus = http.StatusOK
expectedBody = `{"accounts":[],"statuses":[],"hashtags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome","history":[]}]}`
)
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
limit,
offset,
query,
queryType,
resolve,
following,
expectedHTTPStatus,
expectedBody)
if err != nil {
suite.FailNow(err.Error())
}
suite.Len(searchResult.Accounts, 0)
suite.Len(searchResult.Statuses, 0)
suite.Len(searchResult.Hashtags, 1)
}
func (suite *SearchGetTestSuite) TestSearchHashtagV2() {
var (
requestingAccount = suite.testAccounts["local_account_1"]
token = suite.testTokens["local_account_1"]
user = suite.testUsers["local_account_1"]
maxID *string = nil
minID *string = nil
limit *int = nil
offset *int = nil
resolve *bool = nil
query = "#welcome"
queryType *string = func() *string { i := "hashtags"; return &i }()
following *bool = nil
expectedHTTPStatus = http.StatusOK
expectedBody = `{"accounts":[],"statuses":[],"hashtags":["welcome"]}`
)
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv1,
user,
maxID,
minID,
limit,
offset,
query,
queryType,
resolve,
following,
expectedHTTPStatus,
expectedBody)
if err != nil {
suite.FailNow(err.Error())
}
suite.Len(searchResult.Accounts, 0)
suite.Len(searchResult.Statuses, 0)
suite.Len(searchResult.Hashtags, 1)
}
func (suite *SearchGetTestSuite) TestSearchHashtagButWithAccountSearch() {
var (
requestingAccount = suite.testAccounts["local_account_1"]
token = suite.testTokens["local_account_1"]
user = suite.testUsers["local_account_1"]
maxID *string = nil
minID *string = nil
limit *int = nil
offset *int = nil
resolve *bool = nil
query = "#welcome"
queryType *string = func() *string { i := "accounts"; return &i }()
following *bool = nil
expectedHTTPStatus = http.StatusOK
expectedBody = ``
)
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
limit,
offset,
query,
queryType,
resolve,
following,
expectedHTTPStatus,
expectedBody)
if err != nil {
suite.FailNow(err.Error())
}
suite.Len(searchResult.Accounts, 0)
suite.Len(searchResult.Statuses, 0)
suite.Len(searchResult.Hashtags, 0)
}
func (suite *SearchGetTestSuite) TestSearchNotHashtagButWithTypeHashtag() {
var (
requestingAccount = suite.testAccounts["local_account_1"]
token = suite.testTokens["local_account_1"]
user = suite.testUsers["local_account_1"]
maxID *string = nil
minID *string = nil
limit *int = nil
offset *int = nil
resolve *bool = nil
query = "welco"
queryType *string = func() *string { i := "hashtags"; return &i }()
following *bool = nil
expectedHTTPStatus = http.StatusOK
expectedBody = ``
)
searchResult, err := suite.getSearch(
requestingAccount,
token,
apiutil.APIv2,
user,
maxID,
minID,
limit,
offset,
query,
queryType,
resolve,
following,
expectedHTTPStatus,
expectedBody)
if err != nil {
suite.FailNow(err.Error())
}
suite.Len(searchResult.Accounts, 0)
suite.Len(searchResult.Statuses, 0)
suite.Len(searchResult.Hashtags, 1)
}
func TestSearchGetTestSuite(t *testing.T) {
suite.Run(t, &SearchGetTestSuite{})
}

1
internal/api/client/statuses/statuscreate_test.go
View file

@ -98,7 +98,6 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
gtsTag := &gtsmodel.Tag{}
err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "name", Value: "helloworld"}}, gtsTag)
suite.NoError(err)
suite.Equal(statusReply.Account.ID, gtsTag.FirstSeenFromAccountID)
}
func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() {

6
internal/api/client/timelines/home.go
View file

@ -133,9 +133,9 @@ func (m *Module) HomeTimelineGETHandler(c *gin.Context) {
resp, errWithCode := m.processor.Timeline().HomeTimelineGet(
c.Request.Context(),
authed,
c.Query(MaxIDKey),
c.Query(SinceIDKey),
c.Query(MinIDKey),
c.Query(apiutil.MaxIDKey),
c.Query(apiutil.SinceIDKey),
c.Query(apiutil.MinIDKey),
limit,
local,
)

15
internal/api/client/timelines/list.go
View file

@ -18,7 +18,6 @@
package timelines
import (
"errors"
"net/http"
"github.com/gin-gonic/gin"
@ -118,11 +117,9 @@ func (m *Module) ListTimelineGETHandler(c *gin.Context) {
return
}
targetListID := c.Param(IDKey)
if targetListID == "" {
err := errors.New("no list id specified")
apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1)
return
targetListID, errWithCode := apiutil.ParseID(c.Param(apiutil.IDKey))
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
}
limit, errWithCode := apiutil.ParseLimit(c.Query(apiutil.LimitKey), 20, 40, 1)
@ -135,9 +132,9 @@ func (m *Module) ListTimelineGETHandler(c *gin.Context) {
c.Request.Context(),
authed,
targetListID,
c.Query(MaxIDKey),
c.Query(SinceIDKey),
c.Query(MinIDKey),
c.Query(apiutil.MaxIDKey),
c.Query(apiutil.SinceIDKey),
c.Query(apiutil.MinIDKey),
limit,
)
if errWithCode != nil {

6
internal/api/client/timelines/public.go
View file

@ -144,9 +144,9 @@ func (m *Module) PublicTimelineGETHandler(c *gin.Context) {
resp, errWithCode := m.processor.Timeline().PublicTimelineGet(
c.Request.Context(),
authed,
c.Query(MaxIDKey),
c.Query(SinceIDKey),
c.Query(MinIDKey),
c.Query(apiutil.MaxIDKey),
c.Query(apiutil.SinceIDKey),
c.Query(apiutil.MinIDKey),
limit,
local,
)

146
internal/api/client/timelines/tag.go Normal file
View file

@ -0,0 +1,146 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timelines
import (
"net/http"
"github.com/gin-gonic/gin"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
)
// HomeTimelineGETHandler swagger:operation GET /api/v1/timelines/tag/{tag_name} tagTimeline
//
// See public statuses that use the given hashtag (case insensitive).
//
// The statuses will be returned in descending chronological order (newest first), with sequential IDs (bigger = newer).
//
// The returned Link header can be used to generate the previous and next queries when scrolling up or down a timeline.
//
// Example:
//
// ```
// <https://example.org/api/v1/timelines/tag/example?limit=20&max_id=01FC3GSQ8A3MMJ43BPZSGEG29M>; rel="next", <https://example.org/api/v1/timelines/tag/example?limit=20&min_id=01FC3KJW2GYXSDDRA6RWNDM46M>; rel="prev"
// ````
//
// ---
// tags:
// - timelines
//
// produces:
// - application/json
//
// parameters:
// -
// name: max_id
// type: string
// description: >-
// Return only statuses *OLDER* than the given max status ID.
// The status with the specified ID will not be included in the response.
// in: query
// required: false
// -
// name: since_id
// type: string
// description: >-
// Return only statuses *newer* than the given since status ID.
// The status with the specified ID will not be included in the response.
// in: query
// -
// name: min_id
// type: string
// description: >-
// Return only statuses *immediately newer* than the given since status ID.
// The status with the specified ID will not be included in the response.
// in: query
// required: false
// -
// name: limit
// type: integer
// description: Number of statuses to return.
// default: 20
// minimum: 1
// maximum: 40
// in: query
// required: false
//
// security:
// - OAuth2 Bearer:
// - read:statuses
//
// responses:
// '200':
// name: statuses
// description: Array of statuses.
// schema:
// type: array
// items:
// "$ref": "#/definitions/status"
// headers:
// Link:
// type: string
// description: Links to the next and previous queries.
// '401':
// description: unauthorized
// '400':
// description: bad request
func (m *Module) TagTimelineGETHandler(c *gin.Context) {
authed, err := oauth.Authed(c, true, true, true, true)
if err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorUnauthorized(err, err.Error()), m.processor.InstanceGetV1)
return
}
if _, err := apiutil.NegotiateAccept(c, apiutil.JSONAcceptHeaders...); err != nil {
apiutil.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1)
return
}
limit, errWithCode := apiutil.ParseLimit(c.Query(apiutil.LimitKey), 20, 40, 1)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
tagName, errWithCode := apiutil.ParseTagName(c.Param(apiutil.TagNameKey))
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
resp, errWithCode := m.processor.Timeline().TagTimelineGet(
c.Request.Context(),
authed.Account,
tagName,
c.Query(apiutil.MaxIDKey),
c.Query(apiutil.SinceIDKey),
c.Query(apiutil.MinIDKey),
limit,
)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
if resp.LinkHeader != "" {
c.Header("Link", resp.LinkHeader)
}
c.JSON(http.StatusOK, resp.Items)
}

23
internal/api/client/timelines/timeline.go
View file

@ -21,28 +21,16 @@ import (
"net/http"
"github.com/gin-gonic/gin"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/processing"
)
const (
// BasePath is the base URI path for serving timelines, minus the 'api' prefix.
BasePath = "/v1/timelines"
IDKey = "id"
// HomeTimeline is the path for the home timeline
HomeTimeline = BasePath + "/home"
// PublicTimeline is the path for the public (and public local) timeline
BasePath = "/v1/timelines"
HomeTimeline = BasePath + "/home"
PublicTimeline = BasePath + "/public"
ListTimeline = BasePath + "/list/:" + IDKey
// MaxIDKey is the url query for setting a max status ID to return
MaxIDKey = "max_id"
// SinceIDKey is the url query for returning results newer than the given ID
SinceIDKey = "since_id"
// MinIDKey is the url query for returning results immediately newer than the given ID
MinIDKey = "min_id"
// LimitKey is for specifying maximum number of results to return.
LimitKey = "limit"
// LocalKey is for specifying whether only local statuses should be returned
LocalKey = "local"
ListTimeline = BasePath + "/list/:" + apiutil.IDKey
TagTimeline = BasePath + "/tag/:" + apiutil.TagNameKey
)
type Module struct {
@ -59,4 +47,5 @@ func (m *Module) Route(attachHandler func(method string, path string, f ...gin.H
attachHandler(http.MethodGet, HomeTimeline, m.HomeTimelineGETHandler)
attachHandler(http.MethodGet, PublicTimeline, m.PublicTimelineGETHandler)
attachHandler(http.MethodGet, ListTimeline, m.ListTimelineGETHandler)
attachHandler(http.MethodGet, TagTimeline, m.TagTimelineGETHandler)
}

4
internal/api/model/search.go
View file

@ -28,6 +28,7 @@ type SearchRequest struct {
Resolve bool
Following bool
ExcludeUnreviewed bool
APIv1 bool // Set to 'true' if using version 1 of the search API.
}
// SearchResult models a search result.
@ -36,5 +37,6 @@ type SearchRequest struct {
type SearchResult struct {
Accounts []*Account `json:"accounts"`
Statuses []*Status `json:"statuses"`
Hashtags []*Tag `json:"hashtags"`
// Slice of strings if api v1, slice of tags if api v2.
Hashtags []any `json:"hashtags"`
}

4
internal/api/model/tag.go
View file

@ -27,4 +27,8 @@ type Tag struct {
// Web link to the hashtag.
// example: https://example.org/tags/helloworld
URL string `json:"url"`
// History of this hashtag's usage.
// Currently just a stub, if provided will always be an empty array.
// example: []
History *[]any `json:"history,omitempty"`
}

41
internal/api/util/parsequery.go
View file

@ -20,11 +20,18 @@ package util
import (
"fmt"
"strconv"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
const (
/* API version keys */
APIVersionKey = "api_version"
APIv1 = "v1"
APIv2 = "v2"
/* Common keys */
IDKey = "id"
@ -44,6 +51,10 @@ const (
SearchResolveKey = "resolve"
SearchTypeKey = "type"
/* Tag keys */
TagNameKey = "tag_name"
/* Web endpoint keys */
WebUsernameKey = "username"
@ -122,6 +133,26 @@ func ParseDomainBlockImport(value string, defaultValue bool) (bool, gtserror.Wit
Parse functions for *REQUIRED* parameters.
*/
func ParseAPIVersion(value string, availableVersion ...string) (string, gtserror.WithCode) {
key := APIVersionKey
if value == "" {
return "", requiredError(key)
}
for _, av := range availableVersion {
if value == av {
return value, nil
}
}
err := fmt.Errorf(
"invalid API version, valid versions for this path are [%s]",
strings.Join(availableVersion, ", "),
)
return "", gtserror.NewErrorBadRequest(err, err.Error())
}
func ParseID(value string) (string, gtserror.WithCode) {
key := IDKey
@ -152,6 +183,16 @@ func ParseSearchQuery(value string) (string, gtserror.WithCode) {
return value, nil
}
func ParseTagName(value string) (string, gtserror.WithCode) {
key := TagNameKey
if value == "" {
return "", requiredError(key)
}
return value, nil
}
func ParseWebUsername(value string) (string, gtserror.WithCode) {
key := WebUsernameKey

22
internal/cache/gts.go vendored
View file

@ -47,6 +47,7 @@ type GTSCaches struct {
report *result.Cache[*gtsmodel.Report]
status *result.Cache[*gtsmodel.Status]
statusFave *result.Cache[*gtsmodel.StatusFave]
tag *result.Cache[*gtsmodel.Tag]
tombstone *result.Cache[*gtsmodel.Tombstone]
user *result.Cache[*gtsmodel.User]
@ -78,6 +79,7 @@ func (c *GTSCaches) Init() {
c.initReport()
c.initStatus()
c.initStatusFave()
c.initTag()
c.initTombstone()
c.initUser()
c.initWebfinger()
@ -120,6 +122,7 @@ func (c *GTSCaches) Start() {
tryStart(c.report, config.GetCacheGTSReportSweepFreq())
tryStart(c.status, config.GetCacheGTSStatusSweepFreq())
tryStart(c.statusFave, config.GetCacheGTSStatusFaveSweepFreq())
tryStart(c.tag, config.GetCacheGTSTagSweepFreq())
tryStart(c.tombstone, config.GetCacheGTSTombstoneSweepFreq())
tryStart(c.user, config.GetCacheGTSUserSweepFreq())
tryUntil("starting *gtsmodel.Webfinger cache", 5, func() bool {
@ -167,6 +170,7 @@ func (c *GTSCaches) Stop() {
tryStop(c.report, config.GetCacheGTSReportSweepFreq())
tryStop(c.status, config.GetCacheGTSStatusSweepFreq())
tryStop(c.statusFave, config.GetCacheGTSStatusFaveSweepFreq())
tryStop(c.tag, config.GetCacheGTSTagSweepFreq())
tryStop(c.tombstone, config.GetCacheGTSTombstoneSweepFreq())
tryStop(c.user, config.GetCacheGTSUserSweepFreq())
tryUntil("stopping *gtsmodel.Webfinger cache", 5, func() bool {
@ -290,6 +294,11 @@ func (c *GTSCaches) StatusFave() *result.Cache[*gtsmodel.StatusFave] {
return c.statusFave
}
// Tag provides access to the gtsmodel Tag database cache.
func (c *GTSCaches) Tag() *result.Cache[*gtsmodel.Tag] {
return c.tag
}
// Tombstone provides access to the gtsmodel Tombstone database cache.
func (c *GTSCaches) Tombstone() *result.Cache[*gtsmodel.Tombstone] {
return c.tombstone
@ -568,6 +577,19 @@ func (c *GTSCaches) initStatusFave() {
c.status.IgnoreErrors(ignoreErrors)
}
func (c *GTSCaches) initTag() {
c.tag = result.New([]result.Lookup{
{Name: "ID"},
{Name: "Name"},
}, func(m1 *gtsmodel.Tag) *gtsmodel.Tag {
m2 := new(gtsmodel.Tag)
*m2 = *m1
return m2
}, config.GetCacheGTSTagMaxSize())
c.tag.SetTTL(config.GetCacheGTSTagTTL(), true)
c.tag.IgnoreErrors(ignoreErrors)
}
func (c *GTSCaches) initTombstone() {
c.tombstone = result.New([]result.Lookup{
{Name: "ID"},

4
internal/config/config.go
View file

@ -266,6 +266,10 @@ type GTSCacheConfiguration struct {
StatusFaveTTL time.Duration `name:"status-fave-ttl"`
StatusFaveSweepFreq time.Duration `name:"status-fave-sweep-freq"`
TagMaxSize int `name:"tag-max-size"`
TagTTL time.Duration `name:"tag-ttl"`
TagSweepFreq time.Duration `name:"tag-sweep-freq"`
TombstoneMaxSize int `name:"tombstone-max-size"`
TombstoneTTL time.Duration `name:"tombstone-ttl"`
TombstoneSweepFreq time.Duration `name:"tombstone-sweep-freq"`

4
internal/config/defaults.go
View file

@ -211,6 +211,10 @@ var Defaults = Configuration{
StatusFaveTTL: time.Minute * 30,
StatusFaveSweepFreq: time.Minute,
TagMaxSize: 2000,
TagTTL: time.Minute * 30,
TagSweepFreq: time.Minute,
TombstoneMaxSize: 500,
TombstoneTTL: time.Minute * 30,
TombstoneSweepFreq: time.Minute,

75
internal/config/helpers.gen.go
View file

@ -3984,6 +3984,81 @@ func GetCacheGTSStatusFaveSweepFreq() time.Duration { return global.GetCacheGTSS
// SetCacheGTSStatusFaveSweepFreq safely sets the value for global configuration 'Cache.GTS.StatusFaveSweepFreq' field
func SetCacheGTSStatusFaveSweepFreq(v time.Duration) { global.SetCacheGTSStatusFaveSweepFreq(v) }
// GetCacheGTSTagMaxSize safely fetches the Configuration value for state's 'Cache.GTS.TagMaxSize' field
func (st *ConfigState) GetCacheGTSTagMaxSize() (v int) {
st.mutex.RLock()
v = st.config.Cache.GTS.TagMaxSize
st.mutex.RUnlock()
return
}
// SetCacheGTSTagMaxSize safely sets the Configuration value for state's 'Cache.GTS.TagMaxSize' field
func (st *ConfigState) SetCacheGTSTagMaxSize(v int) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.TagMaxSize = v
st.reloadToViper()
}
// CacheGTSTagMaxSizeFlag returns the flag name for the 'Cache.GTS.TagMaxSize' field
func CacheGTSTagMaxSizeFlag() string { return "cache-gts-tag-max-size" }
// GetCacheGTSTagMaxSize safely fetches the value for global configuration 'Cache.GTS.TagMaxSize' field
func GetCacheGTSTagMaxSize() int { return global.GetCacheGTSTagMaxSize() }
// SetCacheGTSTagMaxSize safely sets the value for global configuration 'Cache.GTS.TagMaxSize' field
func SetCacheGTSTagMaxSize(v int) { global.SetCacheGTSTagMaxSize(v) }
// GetCacheGTSTagTTL safely fetches the Configuration value for state's 'Cache.GTS.TagTTL' field
func (st *ConfigState) GetCacheGTSTagTTL() (v time.Duration) {
st.mutex.RLock()
v = st.config.Cache.GTS.TagTTL
st.mutex.RUnlock()
return
}
// SetCacheGTSTagTTL safely sets the Configuration value for state's 'Cache.GTS.TagTTL' field
func (st *ConfigState) SetCacheGTSTagTTL(v time.Duration) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.TagTTL = v
st.reloadToViper()
}
// CacheGTSTagTTLFlag returns the flag name for the 'Cache.GTS.TagTTL' field
func CacheGTSTagTTLFlag() string { return "cache-gts-tag-ttl" }
// GetCacheGTSTagTTL safely fetches the value for global configuration 'Cache.GTS.TagTTL' field
func GetCacheGTSTagTTL() time.Duration { return global.GetCacheGTSTagTTL() }
// SetCacheGTSTagTTL safely sets the value for global configuration 'Cache.GTS.TagTTL' field
func SetCacheGTSTagTTL(v time.Duration) { global.SetCacheGTSTagTTL(v) }
// GetCacheGTSTagSweepFreq safely fetches the Configuration value for state's 'Cache.GTS.TagSweepFreq' field
func (st *ConfigState) GetCacheGTSTagSweepFreq() (v time.Duration) {
st.mutex.RLock()
v = st.config.Cache.GTS.TagSweepFreq
st.mutex.RUnlock()
return
}
// SetCacheGTSTagSweepFreq safely sets the Configuration value for state's 'Cache.GTS.TagSweepFreq' field
func (st *ConfigState) SetCacheGTSTagSweepFreq(v time.Duration) {
st.mutex.Lock()
defer st.mutex.Unlock()
st.config.Cache.GTS.TagSweepFreq = v
st.reloadToViper()
}
// CacheGTSTagSweepFreqFlag returns the flag name for the 'Cache.GTS.TagSweepFreq' field
func CacheGTSTagSweepFreqFlag() string { return "cache-gts-tag-sweep-freq" }
// GetCacheGTSTagSweepFreq safely fetches the value for global configuration 'Cache.GTS.TagSweepFreq' field
func GetCacheGTSTagSweepFreq() time.Duration { return global.GetCacheGTSTagSweepFreq() }
// SetCacheGTSTagSweepFreq safely sets the value for global configuration 'Cache.GTS.TagSweepFreq' field
func SetCacheGTSTagSweepFreq(v time.Duration) { global.SetCacheGTSTagSweepFreq(v) }
// GetCacheGTSTombstoneMaxSize safely fetches the Configuration value for state's 'Cache.GTS.TombstoneMaxSize' field
func (st *ConfigState) GetCacheGTSTombstoneMaxSize() (v int) {
st.mutex.RLock()

1
internal/db/bundb/basic.go
View file

@ -133,7 +133,6 @@ func (b *basicDB) CreateAllTables(ctx context.Context) error {
&gtsmodel.Mention{},
&gtsmodel.Status{},
&gtsmodel.StatusToEmoji{},
&gtsmodel.StatusToTag{},
&gtsmodel.StatusFave{},
&gtsmodel.StatusBookmark{},
&gtsmodel.StatusMute{},

48
internal/db/bundb/bundb.go
View file

@ -39,7 +39,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/tracing"
@ -77,6 +76,7 @@ type DBService struct {
db.Status
db.StatusBookmark
db.StatusFave
db.Tag
db.Timeline
db.User
db.Tombstone
@ -230,6 +230,10 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) {
db: db,
state: state,
},
Tag: &tagDB{
conn: db,
state: state,
},
Timeline: &timelineDB{
db: db,
state: state,
@ -494,45 +498,3 @@ func sqlitePragmas(ctx context.Context, db *WrappedDB) error {
return nil
}
/*
CONVERSION FUNCTIONS
*/
func (dbService *DBService) TagStringToTag(ctx context.Context, t string, originAccountID string) (*gtsmodel.Tag, error) {
protocol := config.GetProtocol()
host := config.GetHost()
now := time.Now()
tag := &gtsmodel.Tag{}
// we can use selectorinsert here to create the new tag if it doesn't exist already
// inserted will be true if this is a new tag we just created
if err := dbService.db.NewSelect().Model(tag).Where("LOWER(?) = LOWER(?)", bun.Ident("name"), t).Scan(ctx); err != nil && err != sql.ErrNoRows {
return nil, fmt.Errorf("error getting tag with name %s: %s", t, err)
}
if tag.ID == "" {
// tag doesn't exist yet so populate it
newID, err := id.NewRandomULID()
if err != nil {
return nil, err
}
tag.ID = newID
tag.URL = protocol + "://" + host + "/tags/" + t
tag.Name = t
tag.FirstSeenFromAccountID = originAccountID
tag.CreatedAt = now
tag.UpdatedAt = now
useable := true
tag.Useable = &useable
listable := true
tag.Listable = &listable
}
// bail already if the tag isn't useable
if !*tag.Useable {
return nil, fmt.Errorf("tag %s is not useable", t)
}
tag.LastStatusAt = now
return tag, nil
}

4
internal/db/bundb/bundb_test.go
View file

@ -84,5 +84,7 @@ func (suite *BunDBStandardTestSuite) SetupTest() {
}
func (suite *BunDBStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
if suite.db != nil {
testrig.StandardDBTeardown(suite.db)
}
}

76
internal/db/bundb/migrations/20230718161520_hashtaggery.go Normal file
View file

@ -0,0 +1,76 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package migrations
import (
"context"
"github.com/uptrace/bun"
)
func init() {
up := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
// Drop now unused columns from tags table.
for _, column := range []string{
"url",
"first_seen_from_account_id",
"last_status_at",
} {
if _, err := tx.
NewDropColumn().
Table("tags").
Column(column).
Exec(ctx); err != nil {
return err
}
}
// Index status_to_tags table properly.
for index, columns := range map[string][]string{
// Index for tag timeline paging.
"status_to_tags_tag_timeline_idx": {"tag_id", "status_id"},
// These indexes were only implicit
// before, make them explicit now.
"status_to_tags_tag_id_idx": {"tag_id"},
"status_to_tags_status_id_idx": {"status_id"},
} {
if _, err := tx.
NewCreateIndex().
Table("status_to_tags").
Index(index).
Column(columns...).
Exec(ctx); err != nil {
return err
}
}
return nil
})
}
down := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
return nil
})
}
if err := Migrations.Register(up, down); err != nil {
panic(err)
}
}

99
internal/db/bundb/search.go
View file

@ -19,6 +19,7 @@ package bundb
import (
"context"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
@ -385,3 +386,101 @@ func (s *searchDB) statusText() *bun.SelectQuery {
return statusText
}
// Query example (SQLite):
//
// SELECT "tag"."id" FROM "tags" AS "tag"
// WHERE ("tag"."id" < 'ZZZZZZZZZZZZZZZZZZZZZZZZZZ')
// AND (("tag"."name") LIKE 'welcome%' ESCAPE '\')
// ORDER BY "tag"."id" DESC LIMIT 10
func (s *searchDB) SearchForTags(
ctx context.Context,
query string,
maxID string,
minID string,
limit int,
offset int,
) ([]*gtsmodel.Tag, error) {
// Ensure reasonable
if limit < 0 {
limit = 0
}
// Make educated guess for slice size
var (
tagIDs = make([]string, 0, limit)
frontToBack = true
)
q := s.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("tags"), bun.Ident("tag")).
// Select only IDs from table
Column("tag.id")
// Return only items with a LOWER id than maxID.
if maxID == "" {
maxID = id.Highest
}
q = q.Where("? < ?", bun.Ident("tag.id"), maxID)
if minID != "" {
// return only tags HIGHER (ie., newer) than minID
q = q.Where("? > ?", bun.Ident("tag.id"), minID)
// page up
frontToBack = false
}
// Normalize tag 'name' string.
name := strings.TrimSpace(query)
name = strings.ToLower(name)
// Search using LIKE for tags that start with `name`.
q = whereStartsLike(q, bun.Ident("tag.name"), name)
if limit > 0 {
// Limit amount of tags returned.
q = q.Limit(limit)
}
if frontToBack {
// Page down.
q = q.Order("tag.id DESC")
} else {
// Page up.
q = q.Order("tag.id ASC")
}
if err := q.Scan(ctx, &tagIDs); err != nil {
return nil, s.db.ProcessError(err)
}
if len(tagIDs) == 0 {
return nil, nil
}
// If we're paging up, we still want tags
// to be sorted by ID desc, so reverse slice.
// https://zchee.github.io/golang-wiki/SliceTricks/#reversing
if !frontToBack {
for l, r := 0, len(tagIDs)-1; l < r; l, r = l+1, r-1 {
tagIDs[l], tagIDs[r] = tagIDs[r], tagIDs[l]
}
}
tags := make([]*gtsmodel.Tag, 0, len(tagIDs))
for _, id := range tagIDs {
// Fetch tag from db for ID
tag, err := s.state.DB.GetTag(ctx, id)
if err != nil {
log.Errorf(ctx, "error fetching tag %q: %v", id, err)
continue
}
// Append status to slice
tags = append(tags, tag)
}
return tags, nil
}

17
internal/db/bundb/search_test.go
View file

@ -77,6 +77,23 @@ func (suite *SearchTestSuite) TestSearchStatuses() {
suite.Len(statuses, 1)
}
func (suite *SearchTestSuite) TestSearchTags() {
// Search with full tag string.
tags, err := suite.db.SearchForTags(context.Background(), "welcome", "", "", 10, 0)
suite.NoError(err)
suite.Len(tags, 1)
// Search with partial tag string.
tags, err = suite.db.SearchForTags(context.Background(), "wel", "", "", 10, 0)
suite.NoError(err)
suite.Len(tags, 1)
// Search with end of tag string.
tags, err = suite.db.SearchForTags(context.Background(), "come", "", "", 10, 0)
suite.NoError(err)
suite.Len(tags, 0)
}
func TestSearchTestSuite(t *testing.T) {
suite.Run(t, new(SearchTestSuite))
}

13
internal/db/bundb/status.go
View file

@ -214,9 +214,16 @@ func (s *statusDB) PopulateStatus(ctx context.Context, status *gtsmodel.Status)
}
}
// TODO: once we don't fetch using relations.
// if !status.TagsPopulated() {
// }
if !status.TagsPopulated() {
// Status tags are out-of-date with IDs, repopulate.
status.Tags, err = s.state.DB.GetTags(
ctx,
status.TagIDs,
)
if err != nil {
errs.Append(fmt.Errorf("error populating status tags: %w", err))
}
}
if !status.MentionsPopulated() {
// Status mentions are out-of-date with IDs, repopulate.

119
internal/db/bundb/tag.go Normal file
View file

@ -0,0 +1,119 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bundb
import (
"context"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/uptrace/bun"
)
type tagDB struct {
conn *WrappedDB
state *state.State
}
func (m *tagDB) GetTag(ctx context.Context, id string) (*gtsmodel.Tag, error) {
return m.state.Caches.GTS.Tag().Load("ID", func() (*gtsmodel.Tag, error) {
var tag gtsmodel.Tag
q := m.conn.
NewSelect().
Model(&tag).
Where("? = ?", bun.Ident("tag.id"), id)
if err := q.Scan(ctx); err != nil {
return nil, m.conn.ProcessError(err)
}
return &tag, nil
}, id)
}
func (m *tagDB) GetTagByName(ctx context.Context, name string) (*gtsmodel.Tag, error) {
// Normalize 'name' string.
name = strings.TrimSpace(name)
name = strings.ToLower(name)
return m.state.Caches.GTS.Tag().Load("Name", func() (*gtsmodel.Tag, error) {
var tag gtsmodel.Tag
q := m.conn.
NewSelect().
Model(&tag).
Where("? = ?", bun.Ident("tag.name"), name)
if err := q.Scan(ctx); err != nil {
return nil, m.conn.ProcessError(err)
}
return &tag, nil
}, name)
}
func (m *tagDB) GetTags(ctx context.Context, ids []string) ([]*gtsmodel.Tag, error) {
tags := make([]*gtsmodel.Tag, 0, len(ids))
for _, id := range ids {
// Attempt fetch from DB
tag, err := m.GetTag(ctx, id)
if err != nil {
log.Errorf(ctx, "error getting tag %q: %v", id, err)
continue
}
// Append tag
tags = append(tags, tag)
}
return tags, nil
}
func (m *tagDB) PutTag(ctx context.Context, tag *gtsmodel.Tag) error {
// Normalize 'name' string before it enters
// the db, without changing tag we were given.
//
// First copy tag to new pointer.
t2 := new(gtsmodel.Tag)
*t2 = *tag
// Normalize name on new pointer.
t2.Name = strings.TrimSpace(t2.Name)
t2.Name = strings.ToLower(t2.Name)
// Insert the copy.
if err := m.state.Caches.GTS.Tag().Store(t2, func() error {
_, err := m.conn.NewInsert().Model(t2).Exec(ctx)
return m.conn.ProcessError(err)
}); err != nil {
return err // err already processed
}
// Update original tag with
// field values populated by db.
tag.CreatedAt = t2.CreatedAt
tag.UpdatedAt = t2.UpdatedAt
tag.Useable = t2.Useable
tag.Listable = t2.Listable
return nil
}

91
internal/db/bundb/tag_test.go Normal file
View file

@ -0,0 +1,91 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package bundb_test
import (
"context"
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
)
type TagTestSuite struct {
BunDBStandardTestSuite
}
func (suite *TagTestSuite) TestGetTag() {
testTag := suite.testTags["welcome"]
dbTag, err := suite.db.GetTag(context.Background(), testTag.ID)
suite.NoError(err)
suite.NotNil(dbTag)
suite.Equal(testTag.ID, dbTag.ID)
}
func (suite *TagTestSuite) TestGetTagByName() {
testTag := suite.testTags["welcome"]
// Name is normalized when doing
// selects from the db, so these
// should all yield the same result.
for _, name := range []string{
"WELCOME",
"welcome",
"Welcome",
"WELCoME ",
} {
dbTag, err := suite.db.GetTagByName(context.Background(), name)
suite.NoError(err)
suite.NotNil(dbTag)
suite.Equal(testTag.ID, dbTag.ID)
}
}
func (suite *TagTestSuite) TestPutTag() {
// Name is normalized when doing
// inserts to the db, so these
// should all yield the same result.
for i, name := range []string{
"NewTag",
"newtag",
"NEWtag",
"NEWTAG ",
} {
err := suite.db.PutTag(context.Background(), &gtsmodel.Tag{
ID: id.NewULID(),
Name: name,
})
if i == 0 {
// This is the first one, so it
// should have just been created.
suite.NoError(err)
continue
}
// Subsequent inserts should fail
// since all these tags are equivalent.
suite.ErrorIs(err, db.ErrAlreadyExists)
}
}
func TestTagTestSuite(t *testing.T) {
suite.Run(t, new(TagTestSuite))
}

108
internal/db/bundb/timeline.go
View file

@ -410,3 +410,111 @@ func (t *timelineDB) GetListTimeline(
return statuses, nil
}
func (t *timelineDB) GetTagTimeline(
ctx context.Context,
tagID string,
maxID string,
sinceID string,
minID string,
limit int,
) ([]*gtsmodel.Status, error) {
// Ensure reasonable
if limit < 0 {
limit = 0
}
// Make educated guess for slice size
var (
statusIDs = make([]string, 0, limit)
frontToBack = true
)
q := t.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("status_to_tags"), bun.Ident("status_to_tag")).
Column("status_to_tag.status_id").
// Join with statuses for filtering.
Join(
"INNER JOIN ? AS ? ON ? = ?",
bun.Ident("statuses"), bun.Ident("status"),
bun.Ident("status.id"), bun.Ident("status_to_tag.status_id"),
).
// Public only.
Where("? = ?", bun.Ident("status.visibility"), gtsmodel.VisibilityPublic).
// This tag only.
Where("? = ?", bun.Ident("status_to_tag.tag_id"), tagID)
if maxID == "" || maxID >= id.Highest {
const future = 24 * time.Hour
var err error
// don't return statuses more than 24hr in the future
maxID, err = id.NewULIDFromTime(time.Now().Add(future))
if err != nil {
return nil, err
}
}
// return only statuses LOWER (ie., older) than maxID
q = q.Where("? < ?", bun.Ident("status_to_tag.status_id"), maxID)
if sinceID != "" {
// return only statuses HIGHER (ie., newer) than sinceID
q = q.Where("? > ?", bun.Ident("status_to_tag.status_id"), sinceID)
}
if minID != "" {
// return only statuses HIGHER (ie., newer) than minID
q = q.Where("? > ?", bun.Ident("status_to_tag.status_id"), minID)
// page up
frontToBack = false
}
if limit > 0 {
// limit amount of statuses returned
q = q.Limit(limit)
}
if frontToBack {
// Page down.
q = q.Order("status_to_tag.status_id DESC")
} else {
// Page up.
q = q.Order("status_to_tag.status_id ASC")
}
if err := q.Scan(ctx, &statusIDs); err != nil {
return nil, t.db.ProcessError(err)
}
if len(statusIDs) == 0 {
return nil, nil
}
// If we're paging up, we still want statuses
// to be sorted by ID desc, so reverse ids slice.
// https://zchee.github.io/golang-wiki/SliceTricks/#reversing
if !frontToBack {
for l, r := 0, len(statusIDs)-1; l < r; l, r = l+1, r-1 {
statusIDs[l], statusIDs[r] = statusIDs[r], statusIDs[l]
}
}
statuses := make([]*gtsmodel.Status, 0, len(statusIDs))
for _, id := range statusIDs {
// Fetch status from db for ID
status, err := t.state.DB.GetStatusByID(ctx, id)
if err != nil {
log.Errorf(ctx, "error fetching status %q: %v", id, err)
continue
}
// Append status to slice
statuses = append(statuses, status)
}
return statuses, nil
}

15
internal/db/bundb/timeline_test.go
View file

@ -272,6 +272,21 @@ func (suite *TimelineTestSuite) TestGetListTimelineMinIDPagingUp() {
suite.Equal("01F8MHCP5P2NWYQ416SBA0XSEV", s[len(s)-1].ID)
}
func (suite *TimelineTestSuite) TestGetTagTimelineNoParams() {
var (
ctx = context.Background()
tag = suite.testTags["welcome"]
)
s, err := suite.db.GetTagTimeline(ctx, tag.ID, "", "", "", 1)
if err != nil {
suite.FailNow(err.Error())
}
suite.checkStatuses(s, id.Highest, id.Lowest, 1)
suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", s[0].ID)
}
func TestTimelineTestSuite(t *testing.T) {
suite.Run(t, new(TimelineTestSuite))
}

31
internal/db/bundb/util.go
View file

@ -34,9 +34,10 @@ var likeEscaper = strings.NewReplacer(
`_`, `\_`, // Exactly one char.
)
// whereSubqueryLike appends a WHERE clause to the
// given SelectQuery, which searches for matches
// of `search` in the given subQuery using LIKE.
// whereLike appends a WHERE clause to the
// given SelectQuery, which searches for
// matches of `search` in the given subQuery
// using LIKE.
func whereLike(
query *bun.SelectQuery,
subject interface{},
@ -58,6 +59,30 @@ func whereLike(
)
}
// whereStartsLike is like whereLike,
// but only searches for strings that
// START WITH `search`.
func whereStartsLike(
query *bun.SelectQuery,
subject interface{},
search string,
) *bun.SelectQuery {
// Escape existing wildcard + escape
// chars in the search query string.
search = likeEscaper.Replace(search)
// Add our own wildcards back in; search
// zero or more chars after the query.
search += `%`
// Append resulting WHERE
// clause to the main query.
return query.Where(
"(?) LIKE ? ESCAPE ?",
subject, search, `\`,
)
}
// updateWhere parses []db.Where and adds it to the given update query.
func updateWhere(q *bun.UpdateQuery, where []db.Where) {
for _, w := range where {

20
internal/db/db.go
View file

@ -17,12 +17,6 @@
package db
import (
"context"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
const (
// DBTypePostgres represents an underlying POSTGRES database type.
DBTypePostgres string = "POSTGRES"
@ -48,20 +42,8 @@ type DB interface {
Status
StatusBookmark
StatusFave
Tag
Timeline
User
Tombstone
/*
USEFUL CONVERSION FUNCTIONS
*/
// TagStringToTag takes a lowercase tag in the form "somehashtag", which has been
// used in a status. It takes the id of the account that wrote the status, and the id of the status itself, and then
// returns an *apimodel.Tag corresponding to the given tags. If the tag already exists in database, that tag
// will be returned. Otherwise a pointer to a new tag struct will be created and returned.
//
// Note: this func doesn't/shouldn't do any manipulation of tags in the DB, it's just for checking
// if they exist in the db already, and conveniently returning them, or creating new tag structs.
TagStringToTag(ctx context.Context, tag string, originAccountID string) (*gtsmodel.Tag, error)
}

3
internal/db/search.go
View file

@ -29,4 +29,7 @@ type Search interface {
// SearchForStatuses uses the given query text to search for statuses created by accountID, or in reply to accountID.
SearchForStatuses(ctx context.Context, accountID string, query string, maxID string, minID string, limit int, offset int) ([]*gtsmodel.Status, error)
// SearchForTags searches for tags that start with the given query text (case insensitive).
SearchForTags(ctx context.Context, query string, maxID string, minID string, limit int, offset int) ([]*gtsmodel.Tag, error)
}

39
internal/db/tag.go Normal file
View file

@ -0,0 +1,39 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package db
import (
"context"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// Tag contains functions for getting/creating tags in the database.
type Tag interface {
// GetTag gets a single tag by ID
GetTag(ctx context.Context, id string) (*gtsmodel.Tag, error)
// GetTagByName gets a single tag using the given name.
GetTagByName(ctx context.Context, name string) (*gtsmodel.Tag, error)
// PutTag inserts the given tag in the database.
PutTag(ctx context.Context, tag *gtsmodel.Tag) error
// GetTags gets multiple tags.
GetTags(ctx context.Context, ids []string) ([]*gtsmodel.Tag, error)
}

4
internal/db/timeline.go
View file

@ -48,4 +48,8 @@ type Timeline interface {
// GetListTimeline returns a slice of statuses from followed accounts collected within the list with the given listID.
// Statuses should be returned in descending order of when they were created (newest first).
GetListTimeline(ctx context.Context, listID string, maxID string, sinceID string, minID string, limit int) ([]*gtsmodel.Status, error)
// GetTagTimeline returns a slice of public-visibility statuses that use the given tagID.
// Statuses should be returned in descending order of when they were created (newest first).
GetTagTimeline(ctx context.Context, tagID string, maxID string, sinceID string, minID string, limit int) ([]*gtsmodel.Status, error)
}

54
internal/federation/dereferencing/status.go
View file

@ -288,7 +288,10 @@ func (d *deref) enrichStatus(
return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err)
}
// TODO: populateStatusTags()
// Ensure the status' tags are populated.
if err := d.fetchStatusTags(ctx, requestUser, latestStatus); err != nil {
return nil, nil, gtserror.Newf("error populating tags for status %s: %w", uri, err)
}
// Ensure the status' media attachments are populated, passing in existing to check for changes.
if err := d.fetchStatusAttachments(ctx, tsport, status, latestStatus); err != nil {
@ -400,6 +403,55 @@ func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, exi
return nil
}
func (d *deref) fetchStatusTags(ctx context.Context, requestUser string, status *gtsmodel.Status) error {
// Allocate new slice to take the yet-to-be determined tag IDs.
status.TagIDs = make([]string, len(status.Tags))
for i := range status.Tags {
placeholder := status.Tags[i]
// Look for existing tag with this name first.
tag, err := d.state.DB.GetTagByName(ctx, placeholder.Name)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
log.Errorf(ctx, "db error getting tag %s: %v", tag.Name, err)
continue
}
// No tag with this name yet, create it.
if tag == nil {
tag = &gtsmodel.Tag{
ID: id.NewULID(),
Name: placeholder.Name,
}
if err := d.state.DB.PutTag(ctx, tag); err != nil {
log.Errorf(ctx, "db error putting tag %s: %v", tag.Name, err)
continue
}
}
// Set the *new* tag and ID.
status.Tags[i] = tag
status.TagIDs[i] = tag.ID
}
// Remove any tag we couldn't get or create.
for i := 0; i < len(status.TagIDs); {
if status.TagIDs[i] == "" {
// This is a failed tag population, likely due
// to some database peculiarity / race condition.
copy(status.Tags[i:], status.Tags[i+1:])
copy(status.TagIDs[i:], status.TagIDs[i+1:])
status.Tags = status.Tags[:len(status.Tags)-1]
status.TagIDs = status.TagIDs[:len(status.TagIDs)-1]
continue
}
i++
}
return nil
}
func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Transport, existing, status *gtsmodel.Status) error {
// Allocate new slice to take the yet-to-be fetched attachment IDs.
status.AttachmentIDs = make([]string, len(status.Attachments))

50
internal/federation/dereferencing/status_test.go
View file

@ -123,6 +123,56 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
suite.False(*m.Silent)
}
func (suite *StatusTestSuite) TestDereferenceStatusWithTag() {
fetchingAccount := suite.testAccounts["local_account_1"]
statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7")
status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL)
suite.NoError(err)
suite.NotNil(status)
// status values should be set
suite.Equal("https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7", status.URI)
suite.Equal("https://unknown-instance.com/users/@brand_new_person/01H641QSRS3TCXSVC10X4GPKW7", status.URL)
suite.Equal("<p>Babe are you okay, you've hardly touched your <a href=\"https://unknown-instance.com/tags/piss\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>piss</span></a></p>", status.Content)
suite.Equal("https://unknown-instance.com/users/brand_new_person", status.AccountURI)
suite.False(*status.Local)
suite.Empty(status.ContentWarning)
suite.Equal(gtsmodel.VisibilityPublic, status.Visibility)
suite.Equal(ap.ObjectNote, status.ActivityStreamsType)
// Ensure tags set + ID'd.
suite.Len(status.Tags, 1)
suite.Len(status.TagIDs, 1)
// status should be in the database
dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
suite.NoError(err)
suite.Equal(status.ID, dbStatus.ID)
suite.True(*dbStatus.Federated)
suite.True(*dbStatus.Boostable)
suite.True(*dbStatus.Replyable)
suite.True(*dbStatus.Likeable)
// account should be in the database now too
account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)
suite.NoError(err)
suite.NotNil(account)
suite.True(*account.Discoverable)
suite.Equal("https://unknown-instance.com/users/brand_new_person", account.URI)
suite.Equal("hey I'm a new person, your instance hasn't seen me yet uwu", account.Note)
suite.Equal("Geoff Brando New Personson", account.DisplayName)
suite.Equal("brand_new_person", account.Username)
suite.NotNil(account.PublicKey)
suite.Nil(account.PrivateKey)
// we should have a tag in the database
t := &gtsmodel.Tag{}
err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "name", Value: "piss"}}, t)
suite.NoError(err)
suite.NotNil(t)
}
func (suite *StatusTestSuite) TestDereferenceStatusWithImageAndNoContent() {
fetchingAccount := suite.testAccounts["local_account_1"]

2
internal/federation/federatingactor_test.go
View file

@ -50,6 +50,7 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() {
false,
nil,
nil,
nil,
)
testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), time.Now(), testNote)
@ -97,6 +98,7 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
false,
nil,
nil,
nil,
)
testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), testNote)

15
internal/gtsmodel/tag.go
View file

@ -21,13 +21,10 @@ import "time"
// Tag represents a hashtag for gathering public statuses together.
type Tag struct {
ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
URL string `validate:"required,url" bun:",nullzero,notnull"` // Href/web address of this tag, eg https://example.org/tags/somehashtag
Name string `validate:"required" bun:",unique,nullzero,notnull"` // name of this tag -- the tag without the hash part
FirstSeenFromAccountID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Which account ID is the first one we saw using this tag?
Useable *bool `validate:"-" bun:",nullzero,notnull,default:true"` // can our instance users use this tag?
Listable *bool `validate:"-" bun:",nullzero,notnull,default:true"` // can our instance users look up this tag?
LastStatusAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was this tag last used?
ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
Name string `validate:"required" bun:",unique,nullzero,notnull"` // (lowercase) name of the tag without the hash prefix
Useable *bool `validate:"-" bun:",nullzero,notnull,default:true"` // Tag is useable on this instance.
Listable *bool `validate:"-" bun:",nullzero,notnull,default:true"` // Tagged statuses can be listed on this instance.
}

129
internal/processing/search/get.go
View file

@ -33,6 +33,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@ -108,14 +109,16 @@ func (p *Processor) Get(
// supply an offset greater than 0, return nothing as
// though there were no additional results.
if req.Offset > 0 {
return p.packageSearchResult(ctx, account, nil, nil)
return p.packageSearchResult(ctx, account, nil, nil, nil, req.APIv1)
}
var (
foundStatuses = make([]*gtsmodel.Status, 0, limit)
foundAccounts = make([]*gtsmodel.Account, 0, limit)
appendStatus = func(foundStatus *gtsmodel.Status) { foundStatuses = append(foundStatuses, foundStatus) }
appendAccount = func(foundAccount *gtsmodel.Account) { foundAccounts = append(foundAccounts, foundAccount) }
foundTags = make([]*gtsmodel.Tag, 0, limit)
appendStatus = func(s *gtsmodel.Status) { foundStatuses = append(foundStatuses, s) }
appendAccount = func(a *gtsmodel.Account) { foundAccounts = append(foundAccounts, a) }
appendTag = func(t *gtsmodel.Tag) { foundTags = append(foundTags, t) }
keepLooking bool
err error
)
@ -162,6 +165,8 @@ func (p *Processor) Get(
account,
foundAccounts,
foundStatuses,
foundTags,
req.APIv1,
)
}
}
@ -189,6 +194,48 @@ func (p *Processor) Get(
account,
foundAccounts,
foundStatuses,
foundTags,
req.APIv1,
)
}
// If query looks like a hashtag (ie., starts
// with '#'), then search for tags.
//
// Since '#' is a very unique prefix and isn't
// shared among account or status searches, we
// can save a bit of time by searching for this
// now, and bailing quickly if we get no results,
// or we're not allowed to include hashtags in
// search results.
//
// We know that none of the subsequent searches
// would show any good results either, and those
// searches are *much* more expensive.
keepLooking, err = p.hashtag(
ctx,
maxID,
minID,
limit,
offset,
query,
queryType,
appendTag,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err = gtserror.Newf("error searching for hashtag: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
if !keepLooking {
// Return whatever we have.
return p.packageSearchResult(
ctx,
account,
foundAccounts,
foundStatuses,
foundTags,
req.APIv1,
)
}
@ -218,6 +265,8 @@ func (p *Processor) Get(
account,
foundAccounts,
foundStatuses,
foundTags,
req.APIv1,
)
}
@ -559,6 +608,80 @@ func (p *Processor) statusByURI(
return nil, gtserror.SetUnretrievable(err)
}
func (p *Processor) hashtag(
ctx context.Context,
maxID string,
minID string,
limit int,
offset int,
query string,
queryType string,
appendTag func(*gtsmodel.Tag),
) (bool, error) {
if query[0] != '#' {
// Query doesn't look like a hashtag,
// but if we're being instructed to
// look explicitly *only* for hashtags,
// let's be generous and assume caller
// just left out the hash prefix.
if queryType != queryTypeHashtags {
// Nope, search isn't explicitly
// for hashtags, keep looking.
return true, nil
}
// Search is explicitly for
// tags, let this one through.
} else if !includeHashtags(queryType) {
// Query looks like a hashtag,
// but we're not meant to include
// hashtags in the results.
//
// Indicate to caller they should
// stop looking, since they're not
// going to get results for this by
// looking in any other way.
return false, nil
}
// Query looks like a hashtag, and we're allowed
// to search for hashtags.
//
// Ensure this is a valid tag for our instance.
normalized, ok := text.NormalizeHashtag(query)
if !ok {
// Couldn't normalize/not a
// valid hashtag after all.
// Caller should stop looking.
return false, nil
}
// Search for tags starting with the normalized string.
tags, err := p.state.DB.SearchForTags(
ctx,
normalized,
maxID,
minID,
limit,
offset,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err := gtserror.Newf(
"error checking database for tags using text %s: %w",
normalized, err,
)
return false, err
}
// Return whatever we got.
for _, tag := range tags {
appendTag(tag)
}
return false, nil
}
// byText searches in the database for accounts and/or
// statuses containing the given query string, using
// the provided parameters.

57
internal/processing/search/util.go
View file

@ -36,6 +36,11 @@ func includeStatuses(queryType string) bool {
return queryType == queryTypeAny || queryType == queryTypeStatuses
}
// return true if given queryType should include hashtags.
func includeHashtags(queryType string) bool {
return queryType == queryTypeAny || queryType == queryTypeHashtags
}
// packageAccounts is a util function that just
// converts the given accounts into an apimodel
// account slice, or errors appropriately.
@ -111,14 +116,59 @@ func (p *Processor) packageStatuses(
return apiStatuses, nil
}
// packageHashtags is a util function that just
// converts the given hashtags into an apimodel
// hashtag slice, or errors appropriately.
func (p *Processor) packageHashtags(
ctx context.Context,
requestingAccount *gtsmodel.Account,
tags []*gtsmodel.Tag,
v1 bool,
) ([]any, gtserror.WithCode) {
apiTags := make([]any, 0, len(tags))
var rangeF func(*gtsmodel.Tag)
if v1 {
// If API version 1, just provide slice of tag names.
rangeF = func(tag *gtsmodel.Tag) {
apiTags = append(apiTags, tag.Name)
}
} else {
// If API not version 1, provide slice of full tags.
rangeF = func(tag *gtsmodel.Tag) {
apiTag, err := p.tc.TagToAPITag(ctx, tag, true)
if err != nil {
log.Debugf(
ctx,
"skipping tag %s because it couldn't be converted to its api representation: %s",
tag.Name, err,
)
return
}
apiTags = append(apiTags, &apiTag)
}
}
for _, tag := range tags {
rangeF(tag)
}
return apiTags, nil
}
// packageSearchResult wraps up the given accounts
// and statuses into an apimodel SearchResult that
// can be serialized to an API caller as JSON.
//
// Set v1 to 'true' if the search is using v1 of the API.
func (p *Processor) packageSearchResult(
ctx context.Context,
requestingAccount *gtsmodel.Account,
accounts []*gtsmodel.Account,
statuses []*gtsmodel.Status,
tags []*gtsmodel.Tag,
v1 bool,
) (*apimodel.SearchResult, gtserror.WithCode) {
apiAccounts, errWithCode := p.packageAccounts(ctx, requestingAccount, accounts)
if errWithCode != nil {
@ -130,9 +180,14 @@ func (p *Processor) packageSearchResult(
return nil, errWithCode
}
apiTags, errWithCode := p.packageHashtags(ctx, requestingAccount, tags, v1)
if errWithCode != nil {
return nil, errWithCode
}
return &apimodel.SearchResult{
Accounts: apiAccounts,
Statuses: apiStatuses,
Hashtags: make([]*apimodel.Tag, 0),
Hashtags: apiTags,
}, nil
}

141
internal/processing/timeline/tag.go Normal file
View file

@ -0,0 +1,141 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package timeline
import (
"context"
"errors"
"fmt"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// TagTimelineGet gets a pageable timeline for the given
// tagName and given paging parameters. It will ensure
// that each status in the timeline is actually visible
// to requestingAcct before returning it.
func (p *Processor) TagTimelineGet(
ctx context.Context,
requestingAcct *gtsmodel.Account,
tagName string,
maxID string,
sinceID string,
minID string,
limit int,
) (*apimodel.PageableResponse, gtserror.WithCode) {
tag, errWithCode := p.getTag(ctx, tagName)
if errWithCode != nil {
return nil, errWithCode
}
if tag == nil || !*tag.Useable || !*tag.Listable {
// Obey mastodon API by returning 404 for this.
err := fmt.Errorf("tag was not found, or not useable/listable on this instance")
return nil, gtserror.NewErrorNotFound(err, err.Error())
}
statuses, err := p.state.DB.GetTagTimeline(ctx, tag.ID, maxID, sinceID, minID, limit)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err = gtserror.Newf("db error getting statuses: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
return p.packageTagResponse(
ctx,
requestingAcct,
statuses,
limit,
// Use API URL for tag.
"/api/v1/timelines/tag/"+tagName,
)
}
func (p *Processor) getTag(ctx context.Context, tagName string) (*gtsmodel.Tag, gtserror.WithCode) {
// Normalize + validate tag name.
tagNameNormal, ok := text.NormalizeHashtag(tagName)
if !ok {
err := gtserror.Newf("string '%s' could not be normalized to a valid hashtag", tagName)
return nil, gtserror.NewErrorBadRequest(err, err.Error())
}
// Ensure we have tag with this name in the db.
tag, err := p.state.DB.GetTagByName(ctx, tagNameNormal)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
// Real db error.
err = gtserror.Newf("db error getting tag by name: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
return tag, nil
}
func (p *Processor) packageTagResponse(
ctx context.Context,
requestingAcct *gtsmodel.Account,
statuses []*gtsmodel.Status,
limit int,
requestPath string,
) (*apimodel.PageableResponse, gtserror.WithCode) {
count := len(statuses)
if count == 0 {
return util.EmptyPageableResponse(), nil
}
var (
items = make([]interface{}, 0, count)
// Set next + prev values before filtering and API
// converting, so caller can still page properly.
nextMaxIDValue = statuses[count-1].ID
prevMinIDValue = statuses[0].ID
)
for _, s := range statuses {
timelineable, err := p.filter.StatusTagTimelineable(ctx, requestingAcct, s)
if err != nil {
log.Errorf(ctx, "error checking status visibility: %v", err)
continue
}
if !timelineable {
continue
}
apiStatus, err := p.tc.StatusToAPIStatus(ctx, s, requestingAcct)
if err != nil {
log.Errorf(ctx, "error converting to api status: %v", err)
continue
}
items = append(items, apiStatus)
}
return util.PackagePageableResponse(util.PageableResponseParams{
Items: items,
Path: requestPath,
NextMaxIDValue: nextMaxIDValue,
PrevMinIDValue: prevMinIDValue,
Limit: limit,
})
}

10
internal/text/markdown_test.go
View file

@ -49,13 +49,13 @@ const (
withInlineCode2 = "`Nobody tells you about the </code><del>SECRET CODE</del><code>, do they?`"
withInlineCode2Expected = "<p><code>Nobody tells you about the &lt;/code>&lt;del>SECRET CODE&lt;/del>&lt;code>, do they?</code></p>"
withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!"
withHashtagExpected = "<h1>Title</h1><p>here's a simple status that uses hashtag <a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a>!</p>"
withHashtagExpected = "<h1>Title</h1><p>here's a simple status that uses hashtag <a href=\"http://localhost:8080/tags/hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a>!</p>"
mdWithHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a <a href=\"https://example.org\">link</a>.\n\nHere's an image: <img src=\"https://gts.superseriousbusiness.org/assets/logo.png\" alt=\"The GoToSocial sloth logo.\" width=\"500\" height=\"600\">"
mdWithHTMLExpected = "<h1>Title</h1><p>Here's a simple text in markdown.</p><p>Here's a <a href=\"https://example.org\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">link</a>.</p><p>Here's an image: <img src=\"https://gts.superseriousbusiness.org/assets/logo.png\" alt=\"The GoToSocial sloth logo.\" width=\"500\" height=\"600\" crossorigin=\"anonymous\"></p>"
mdWithCheekyHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a cheeky little script: <script>alert(ahhhh)</script>"
mdWithCheekyHTMLExpected = "<h1>Title</h1><p>Here's a simple text in markdown.</p><p>Here's a cheeky little script:</p>"
mdWithHashtagInitial = "#welcome #Hashtag"
mdWithHashtagInitialExpected = "<p><a href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>welcome</span></a> <a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a></p>"
mdWithHashtagInitialExpected = "<p><a href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>welcome</span></a> <a href=\"http://localhost:8080/tags/hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a></p>"
mdCodeBlockWithNewlines = "some code coming up\n\n```\n\n\n\n```\nthat was some code"
mdCodeBlockWithNewlinesExpected = "<p>some code coming up</p><pre><code>\n\n\n</code></pre><p>that was some code</p>"
mdWithFootnote = "fox mulder,fbi.[^1]\n\n[^1]: federated bureau of investigation"
@ -63,7 +63,7 @@ const (
mdWithBlockQuote = "get ready, there's a block quote coming:\n\n>line1\n>line2\n>\n>line3\n\n"
mdWithBlockQuoteExpected = "<p>get ready, there's a block quote coming:</p><blockquote><p>line1<br>line2</p><p>line3</p></blockquote>"
mdHashtagAndCodeBlock = "#Hashtag\n\n```\n#Hashtag\n```"
mdHashtagAndCodeBlockExpected = "<p><a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a></p><pre><code>#Hashtag\n</code></pre>"
mdHashtagAndCodeBlockExpected = "<p><a href=\"http://localhost:8080/tags/hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a></p><pre><code>#Hashtag\n</code></pre>"
mdMentionAndCodeBlock = "@the_mighty_zork\n\n```\n@the_mighty_zork\n```"
mdMentionAndCodeBlockExpected = "<p><span class=\"h-card\"><a href=\"http://localhost:8080/@the_mighty_zork\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>the_mighty_zork</span></a></span></p><pre><code>@the_mighty_zork\n</code></pre>"
mdWithSmartypants = "\"you have to quargle the bleepflorp\" they said with 1/2 of nominal speed and 1/3 of the usual glumping"
@ -77,9 +77,9 @@ const (
mdObjectInCodeBlock = "@foss_satan@fossbros-anonymous.io this is how to mention a user\n```\n@the_mighty_zork hey bud! nice #ObjectOrientedProgramming software you've been writing lately! :rainbow:\n```\nhope that helps"
mdObjectInCodeBlockExpected = "<p><span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span> this is how to mention a user</p><pre><code>@the_mighty_zork hey bud! nice #ObjectOrientedProgramming software you&#39;ve been writing lately! :rainbow:\n</code></pre><p>hope that helps</p>"
mdItalicHashtag = "_#hashtag_"
mdItalicHashtagExpected = "<p><em><a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashtag</span></a></em></p>"
mdItalicHashtagExpected = "<p><em><a href=\"http://localhost:8080/tags/hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashtag</span></a></em></p>"
mdItalicHashtags = "_#hashtag #hashtag #hashtag_"
mdItalicHashtagsExpected = "<p><em><a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashtag</span></a> <a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashtag</span></a> <a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashtag</span></a></em></p>"
mdItalicHashtagsExpected = "<p><em><a href=\"http://localhost:8080/tags/hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashtag</span></a> <a href=\"http://localhost:8080/tags/hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashtag</span></a> <a href=\"http://localhost:8080/tags/hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>hashtag</span></a></em></p>"
// BEWARE: sneaky unicode business going on.
// the first ö is one rune, the second ö is an o with a combining diacritic.
mdUnnormalizedHashtag = "#hellöthere #hellöthere"

60
internal/text/normalize.go Normal file
View file

@ -0,0 +1,60 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package text
import (
"strings"
"github.com/superseriousbusiness/gotosocial/internal/util"
"golang.org/x/text/unicode/norm"
)
const (
maximumHashtagLength = 100
)
// NormalizeHashtag normalizes the given hashtag text by
// removing the initial '#' symbol, and then decomposing
// and canonically recomposing chars + combining diacritics
// in the text to single unicode characters, following
// Normalization Form C (https://unicode.org/reports/tr15/).
//
// Finally, it will do a check on the normalized string to
// ensure that it's below maximumHashtagLength chars, and
// contains only unicode letters and numbers. If this passes,
// returned bool will be true.
func NormalizeHashtag(text string) (string, bool) {
// This normalization is specifically to avoid cases
// where visually-identical hashtags are stored with
// different unicode representations (e.g. with combining
// diacritics). It allows a tasteful number of combining
// diacritics to be used, as long as they can be combined
// with parent characters to form regular letter symbols.
normalized := norm.NFC.String(strings.TrimPrefix(text, "#"))
// Validate normalized.
ok := true
for i, r := range normalized {
if i >= maximumHashtagLength || !util.IsPermittedInHashtag(r) {
ok = false
break
}
}
return normalized, ok
}

6
internal/text/plain_test.go
View file

@ -34,7 +34,7 @@ const (
withHTML = "<div>blah this should just be html escaped blah</div>"
withHTMLExpected = "<p>&lt;div>blah this should just be html escaped blah&lt;/div></p>"
moreComplex = "Another test @foss_satan@fossbros-anonymous.io\n\n#Hashtag\n\nText\n\n:rainbow:"
moreComplexExpected = "<p>Another test <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span><br><br><a href=\"http://localhost:8080/tags/Hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a><br><br>Text<br><br>:rainbow:</p>"
moreComplexExpected = "<p>Another test <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span><br><br><a href=\"http://localhost:8080/tags/hashtag\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>Hashtag</span></a><br><br>Text<br><br>:rainbow:</p>"
)
type PlainTestSuite struct {
@ -103,7 +103,7 @@ func (suite *PlainTestSuite) TestDeriveHashtagsOK() {
#111111 thisalsoshouldn'twork#### ##
#alimentación, #saúde, #lävistää, #ö, #네
#ThisOneIsThirtyOneCharactersLon... ...ng
#ThisOneIsOneHundredAndOneCharactersLongWhichIsReallyJustWayWayTooLongDefinitelyLongerThanYouWouldNeed...
#ThisOneIsThirteyCharactersLong
`
@ -141,7 +141,7 @@ func (suite *PlainTestSuite) TestDeriveMultiple() {
assert.Equal(suite.T(), "@foss_satan@fossbros-anonymous.io", f.Mentions[0].NameString)
assert.Len(suite.T(), f.Tags, 1)
assert.Equal(suite.T(), "Hashtag", f.Tags[0].Name)
assert.Equal(suite.T(), "hashtag", f.Tags[0].Name)
assert.Len(suite.T(), f.Emojis, 0)
}

103
internal/text/replace.go
View file

@ -23,19 +23,13 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/util"
"golang.org/x/text/unicode/norm"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
const (
maximumHashtagLength = 30
)
// given a mention or a hashtag string, the methods in this file will attempt to parse it,
// add it to the database, and render it as HTML. If any of these steps fails, the method
// will just return the original string and log an error.
// replaceMention takes a string in the form @username@domain.com or @localusername
func (r *customRenderer) replaceMention(text string) string {
mention, err := r.parseMention(r.ctx, text, r.accountID, r.statusID)
@ -90,55 +84,78 @@ func (r *customRenderer) replaceMention(text string) string {
return b.String()
}
// replaceMention takes a string in the form #HashedTag, and will normalize it before
// adding it to the db and turning it into HTML.
// replaceHashtag takes a string in the form #SomeHashtag, and will normalize
// it before adding it to the db (or just getting it from the db if it already
// exists) and turning it into HTML.
func (r *customRenderer) replaceHashtag(text string) string {
// this normalization is specifically to avoid cases where visually-identical
// hashtags are stored with different unicode representations (e.g. with combining
// diacritics). It allows a tasteful number of combining diacritics to be used,
// as long as they can be combined with parent characters to form regular letter
// symbols.
normalized := norm.NFC.String(text[1:])
for i, r := range normalized {
if i >= maximumHashtagLength || !util.IsPermittedInHashtag(r) {
return text
}
normalized, ok := NormalizeHashtag(text)
if !ok {
// Not a valid hashtag.
return text
}
tag, err := r.f.db.TagStringToTag(r.ctx, normalized, r.accountID)
tag, err := r.getOrCreateHashtag(normalized)
if err != nil {
log.Errorf(r.ctx, "error generating hashtags from status: %s", err)
return text
}
// only append if it's not been listed yet
listed := false
for _, t := range r.result.Tags {
if tag.ID == t.ID {
listed = true
break
}
}
if !listed {
err = r.f.db.Put(r.ctx, tag)
if err != nil {
if !errors.Is(err, db.ErrAlreadyExists) {
log.Errorf(r.ctx, "error putting tags in db: %s", err)
return text
// Append tag to result if not done already.
//
// This prevents multiple uses of a tag in
// the same status generating multiple
// entries for the same tag in result.
func() {
for _, t := range r.result.Tags {
if tag.ID == t.ID {
// Already appended.
return
}
}
r.result.Tags = append(r.result.Tags, tag)
}
// Not appended yet.
r.result.Tags = append(r.result.Tags, tag)
}()
// Replace tag with the formatted tag content, eg. `#SomeHashtag` becomes:
// `<a href="https://example.org/tags/somehashtag" class="mention hashtag" rel="tag">#<span>SomeHashtag</span></a>`
var b strings.Builder
// replace the #tag with the formatted tag content
// `<a href="tag.URL" class="mention hashtag" rel="tag">#<span>tagAsEntered</span></a>
b.WriteString(`<a href="`)
b.WriteString(tag.URL)
b.WriteString(uris.GenerateURIForTag(normalized))
b.WriteString(`" class="mention hashtag" rel="tag">#<span>`)
b.WriteString(normalized)
b.WriteString(`</span></a>`)
return b.String()
}
func (r *customRenderer) getOrCreateHashtag(name string) (*gtsmodel.Tag, error) {
var (
tag *gtsmodel.Tag
err error
)
// Check if we have a tag with this name already.
tag, err = r.f.db.GetTagByName(r.ctx, name)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, gtserror.Newf("db error getting tag %s: %w", name, err)
}
if tag != nil {
// We had it!
return tag, nil
}
// We didn't have a tag with
// this name, create one.
tag = &gtsmodel.Tag{
ID: id.NewULID(),
Name: name,
}
if err = r.f.db.PutTag(r.ctx, tag); err != nil {
return nil, gtserror.Newf("db error putting new tag %s: %w", name, err)
}
return tag, nil
}

5
internal/typeutils/converter.go
View file

@ -71,7 +71,8 @@ type TypeConverter interface {
// EmojiCategoryToAPIEmojiCategory converts a gts model emoji category into its api (frontend) representation.
EmojiCategoryToAPIEmojiCategory(ctx context.Context, category *gtsmodel.EmojiCategory) (*apimodel.EmojiCategory, error)
// TagToAPITag converts a gts model tag into its api (frontend) representation for serialization on the API.
TagToAPITag(ctx context.Context, t *gtsmodel.Tag) (apimodel.Tag, error)
// If stubHistory is set to 'true', then the 'history' field of the tag will be populated with a pointer to an empty slice, for API compatibility reasons.
TagToAPITag(ctx context.Context, t *gtsmodel.Tag, stubHistory bool) (apimodel.Tag, error)
// StatusToAPIStatus converts a gts model status into its api (frontend) representation for serialization on the API.
//
// Requesting account can be nil.
@ -160,6 +161,8 @@ type TypeConverter interface {
MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab.ActivityStreamsMention, error)
// EmojiToAS converts a gts emoji into a mastodon ns Emoji, suitable for federation
EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.TootEmoji, error)
// TagToAS converts a gts model tag into a toot Hashtag, suitable for federation.
TagToAS(ctx context.Context, t *gtsmodel.Tag) (vocab.TootHashtag, error)
// AttachmentToAS converts a gts model media attachment into an activity streams Attachment, suitable for federation
AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachment) (vocab.ActivityStreamsDocument, error)
// FaveToAS converts a gts model status fave into an activityStreams LIKE, suitable for federation.

79
internal/typeutils/internaltoas.go
View file

@ -24,6 +24,7 @@ import (
"errors"
"fmt"
"net/url"
"strings"
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/activity/streams"
@ -33,6 +34,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
// Converts a gts model account into an Activity Streams person type.
@ -407,7 +409,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
if s.Account == nil {
a, err := c.db.GetAccountByID(ctx, s.AccountID)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error retrieving author account from db: %s", err)
return nil, gtserror.Newf("error retrieving author account from db: %w", err)
}
s.Account = a
}
@ -418,7 +420,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
// id
statusURI, err := url.Parse(s.URI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.URI, err)
return nil, gtserror.Newf("error parsing url %s: %w", s.URI, err)
}
statusIDProp := streams.NewJSONLDIdProperty()
statusIDProp.SetIRI(statusURI)
@ -436,7 +438,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
if s.InReplyToURI != "" {
rURI, err := url.Parse(s.InReplyToURI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.InReplyToURI, err)
return nil, gtserror.Newf("error parsing url %s: %w", s.InReplyToURI, err)
}
inReplyToProp := streams.NewActivityStreamsInReplyToProperty()
@ -453,7 +455,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
if s.URL != "" {
sURL, err := url.Parse(s.URL)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.URL, err)
return nil, gtserror.Newf("error parsing url %s: %w", s.URL, err)
}
urlProp := streams.NewActivityStreamsUrlProperty()
@ -464,7 +466,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
// attributedTo
authorAccountURI, err := url.Parse(s.Account.URI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.Account.URI, err)
return nil, gtserror.Newf("error parsing url %s: %w", s.Account.URI, err)
}
attributedToProp := streams.NewActivityStreamsAttributedToProperty()
attributedToProp.AppendIRI(authorAccountURI)
@ -478,13 +480,13 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
if len(s.MentionIDs) > len(mentions) {
mentions, err = c.db.GetMentions(ctx, s.MentionIDs)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error getting mentions: %w", err)
return nil, gtserror.Newf("error getting mentions: %w", err)
}
}
for _, m := range mentions {
asMention, err := c.MentionToAS(ctx, m)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error converting mention to AS mention: %s", err)
return nil, gtserror.Newf("error converting mention to AS mention: %w", err)
}
tagProp.AppendActivityStreamsMention(asMention)
}
@ -496,7 +498,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
for _, emojiID := range s.EmojiIDs {
emoji, err := c.db.GetEmojiByID(ctx, emojiID)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error getting emoji %s from database: %s", emojiID, err)
return nil, gtserror.Newf("error getting emoji %s from database: %w", emojiID, err)
}
emojis = append(emojis, emoji)
}
@ -504,25 +506,38 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
for _, emoji := range emojis {
asEmoji, err := c.EmojiToAS(ctx, emoji)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error converting emoji to AS emoji: %s", err)
return nil, gtserror.Newf("error converting emoji to AS emoji: %w", err)
}
tagProp.AppendTootEmoji(asEmoji)
}
// tag -- hashtags
// TODO
hashtags := s.Tags
if len(s.TagIDs) > len(hashtags) {
hashtags, err = c.db.GetTags(ctx, s.TagIDs)
if err != nil {
return nil, gtserror.Newf("error getting tags: %w", err)
}
}
for _, ht := range hashtags {
asHashtag, err := c.TagToAS(ctx, ht)
if err != nil {
return nil, gtserror.Newf("error converting tag to AS tag: %w", err)
}
tagProp.AppendTootHashtag(asHashtag)
}
status.SetActivityStreamsTag(tagProp)
// parse out some URIs we need here
authorFollowersURI, err := url.Parse(s.Account.FollowersURI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", s.Account.FollowersURI, err)
return nil, gtserror.Newf("error parsing url %s: %w", s.Account.FollowersURI, err)
}
publicURI, err := url.Parse(pub.PublicActivityPubIRI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing url %s: %s", pub.PublicActivityPubIRI, err)
return nil, gtserror.Newf("error parsing url %s: %w", pub.PublicActivityPubIRI, err)
}
// to and cc
@ -534,7 +549,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
for _, m := range mentions {
iri, err := url.Parse(m.TargetAccount.URI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err)
}
toProp.AppendIRI(iri)
}
@ -546,7 +561,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
for _, m := range mentions {
iri, err := url.Parse(m.TargetAccount.URI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err)
}
ccProp.AppendIRI(iri)
}
@ -557,7 +572,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
for _, m := range mentions {
iri, err := url.Parse(m.TargetAccount.URI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err)
}
ccProp.AppendIRI(iri)
}
@ -568,7 +583,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
for _, m := range mentions {
iri, err := url.Parse(m.TargetAccount.URI)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error parsing uri %s: %s", m.TargetAccount.URI, err)
return nil, gtserror.Newf("error parsing uri %s: %w", m.TargetAccount.URI, err)
}
ccProp.AppendIRI(iri)
}
@ -592,7 +607,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
for _, attachmentID := range s.AttachmentIDs {
attachment, err := c.db.GetAttachmentByID(ctx, attachmentID)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error getting attachment %s from database: %s", attachmentID, err)
return nil, gtserror.Newf("error getting attachment %s from database: %w", attachmentID, err)
}
attachments = append(attachments, attachment)
}
@ -600,7 +615,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
for _, a := range attachments {
doc, err := c.AttachmentToAS(ctx, a)
if err != nil {
return nil, fmt.Errorf("StatusToAS: error converting attachment: %s", err)
return nil, gtserror.Newf("error converting attachment: %w", err)
}
attachmentProp.AppendActivityStreamsDocument(doc)
}
@ -609,7 +624,7 @@ func (c *converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (vocab.A
// replies
repliesCollection, err := c.StatusToASRepliesCollection(ctx, s, false)
if err != nil {
return nil, fmt.Errorf("error creating repliesCollection: %s", err)
return nil, fmt.Errorf("error creating repliesCollection: %w", err)
}
repliesProp := streams.NewActivityStreamsRepliesProperty()
@ -846,6 +861,32 @@ func (c *converter) MentionToAS(ctx context.Context, m *gtsmodel.Mention) (vocab
return mention, nil
}
func (c *converter) TagToAS(ctx context.Context, t *gtsmodel.Tag) (vocab.TootHashtag, error) {
// This is probably already lowercase,
// but let's err on the safe side.
nameLower := strings.ToLower(t.Name)
tagURLString := uris.GenerateURIForTag(nameLower)
// Create the tag.
tag := streams.NewTootHashtag()
// `href` should be the URL of the tag.
hrefProp := streams.NewActivityStreamsHrefProperty()
tagURL, err := url.Parse(tagURLString)
if err != nil {
return nil, gtserror.Newf("error parsing url %s: %w", tagURLString, err)
}
hrefProp.SetIRI(tagURL)
tag.SetActivityStreamsHref(hrefProp)
// `name` should be the name of the tag with the # prefix.
nameProp := streams.NewActivityStreamsNameProperty()
nameProp.AppendXMLSchemaString("#" + nameLower)
tag.SetActivityStreamsName(nameProp)
return tag, nil
}
/*
we're making something like this:
{

54
internal/typeutils/internaltoas_test.go
View file

@ -403,17 +403,24 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
},
"sensitive": false,
"summary": "",
"tag": {
"icon": {
"mediaType": "image/png",
"type": "Image",
"url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"
"tag": [
{
"icon": {
"mediaType": "image/png",
"type": "Image",
"url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"
},
"id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
"name": ":rainbow:",
"type": "Emoji",
"updated": "2021-09-20T10:40:37Z"
},
"id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
"name": ":rainbow:",
"type": "Emoji",
"updated": "2021-09-20T10:40:37Z"
},
{
"href": "http://localhost:8080/tags/welcome",
"name": "#welcome",
"type": "Hashtag"
}
],
"to": "https://www.w3.org/ns/activitystreams#Public",
"type": "Note",
"url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"
@ -463,17 +470,24 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
},
"sensitive": false,
"summary": "",
"tag": {
"icon": {
"mediaType": "image/png",
"type": "Image",
"url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"
"tag": [
{
"icon": {
"mediaType": "image/png",
"type": "Image",
"url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"
},
"id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
"name": ":rainbow:",
"type": "Emoji",
"updated": "2021-09-20T10:40:37Z"
},
"id": "http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ",
"name": ":rainbow:",
"type": "Emoji",
"updated": "2021-09-20T10:40:37Z"
},
{
"href": "http://localhost:8080/tags/welcome",
"name": "#welcome",
"type": "Hashtag"
}
],
"to": "https://www.w3.org/ns/activitystreams#Public",
"type": "Note",
"url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"

33
internal/typeutils/internaltofrontend.go
View file

@ -32,6 +32,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/uris"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@ -568,10 +569,18 @@ func (c *converter) EmojiCategoryToAPIEmojiCategory(ctx context.Context, categor
}, nil
}
func (c *converter) TagToAPITag(ctx context.Context, t *gtsmodel.Tag) (apimodel.Tag, error) {
func (c *converter) TagToAPITag(ctx context.Context, t *gtsmodel.Tag, stubHistory bool) (apimodel.Tag, error) {
return apimodel.Tag{
Name: t.Name,
URL: t.URL,
Name: strings.ToLower(t.Name),
URL: uris.GenerateURIForTag(t.Name),
History: func() *[]any {
if !stubHistory {
return nil
}
h := make([]any, 0)
return &h
}(),
}, nil
}
@ -1297,19 +1306,11 @@ func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T
var errs gtserror.MultiError
if len(tags) == 0 {
// GTS model tags were not populated
var err error
// Preallocate expected GTS slice
tags = make([]*gtsmodel.Tag, 0, len(tagIDs))
// Fetch GTS models for tag IDs
for _, id := range tagIDs {
tag := new(gtsmodel.Tag)
if err := c.db.GetByID(ctx, id, tag); err != nil {
errs.Appendf("error fetching tag %s from database: %v", id, err)
continue
}
tags = append(tags, tag)
tags, err = c.db.GetTags(ctx, tagIDs)
if err != nil {
errs.Appendf("error fetching tags from database: %v", err)
}
}
@ -1318,7 +1319,7 @@ func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T
// Convert GTS models to frontend models
for _, tag := range tags {
apiTag, err := c.TagToAPITag(ctx, tag)
apiTag, err := c.TagToAPITag(ctx, tag, false)
if err != nil {
errs.Appendf("error converting tag %s to api tag: %v", tag.ID, err)
continue

9
internal/uris/uri.go
View file

@ -20,6 +20,7 @@ package uris
import (
"fmt"
"net/url"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/regexes"
@ -43,6 +44,7 @@ const (
ConfirmEmailPath = "confirm_email" // ConfirmEmailPath is used to generate the URI for an email confirmation link
FileserverPath = "fileserver" // FileserverPath is a path component for serving attachments + media
EmojiPath = "emoji" // EmojiPath represents the activitypub emoji location
TagsPath = "tags" // TagsPath represents the activitypub tags location
)
// UserURIs contains a bunch of UserURIs and URLs for a user, host, account, etc.
@ -178,6 +180,13 @@ func GenerateURIForEmoji(emojiID string) string {
return fmt.Sprintf("%s://%s/%s/%s", protocol, host, EmojiPath, emojiID)
}
// GenerateURIForTag generates an activitypub uri for a tag.
func GenerateURIForTag(name string) string {
protocol := config.GetProtocol()
host := config.GetHost()
return fmt.Sprintf("%s://%s/%s/%s", protocol, host, TagsPath, strings.ToLower(name))
}
// IsUserPath returns true if the given URL path corresponds to eg /users/example_username
func IsUserPath(id *url.URL) bool {
return regexes.UserPath.MatchString(id.Path)

93
internal/validate/tag_test.go
View file

@ -1,93 +0,0 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validate_test
import (
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/validate"
"github.com/superseriousbusiness/gotosocial/testrig"
)
func happyTag() *gtsmodel.Tag {
return &gtsmodel.Tag{
ID: "01FE91RJR88PSEEE30EV35QR8N",
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
URL: "https://example.org/tags/some_tag",
Name: "some_tag",
FirstSeenFromAccountID: "01FE91SR5P2GW06K3AJ98P72MT",
Useable: testrig.TrueBool(),
Listable: testrig.TrueBool(),
LastStatusAt: time.Now(),
}
}
type TagValidateTestSuite struct {
suite.Suite
}
func (suite *TagValidateTestSuite) TestValidateTagHappyPath() {
// no problem here
t := happyTag()
err := validate.Struct(t)
suite.NoError(err)
}
func (suite *TagValidateTestSuite) TestValidateTagNoName() {
t := happyTag()
t.Name = ""
err := validate.Struct(t)
suite.EqualError(err, "Key: 'Tag.Name' Error:Field validation for 'Name' failed on the 'required' tag")
}
func (suite *TagValidateTestSuite) TestValidateTagBadURL() {
t := happyTag()
t.URL = ""
err := validate.Struct(t)
suite.EqualError(err, "Key: 'Tag.URL' Error:Field validation for 'URL' failed on the 'required' tag")
t.URL = "no-schema.com"
err = validate.Struct(t)
suite.EqualError(err, "Key: 'Tag.URL' Error:Field validation for 'URL' failed on the 'url' tag")
t.URL = "justastring"
err = validate.Struct(t)
suite.EqualError(err, "Key: 'Tag.URL' Error:Field validation for 'URL' failed on the 'url' tag")
t.URL = "https://aaa\n\n\naaaaaaaa"
err = validate.Struct(t)
suite.EqualError(err, "Key: 'Tag.URL' Error:Field validation for 'URL' failed on the 'url' tag")
}
func (suite *TagValidateTestSuite) TestValidateTagNoFirstSeenFromAccountID() {
t := happyTag()
t.FirstSeenFromAccountID = ""
err := validate.Struct(t)
suite.NoError(err)
}
func TestTagValidateTestSuite(t *testing.T) {
suite.Run(t, new(TagValidateTestSuite))
}

60
internal/visibility/tag_timeline.go Normal file
View file

@ -0,0 +1,60 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package visibility
import (
"context"
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
)
// StatusHomeTimelineable checks if given status should be included
// on requester's tag timeline, primarily relying on status visibility
// to requester and the AP visibility setting.
func (f *Filter) StatusTagTimelineable(
ctx context.Context,
requester *gtsmodel.Account,
status *gtsmodel.Status,
) (bool, error) {
if status.CreatedAt.After(time.Now().Add(24 * time.Hour)) {
// Statuses made over 1 day in the future we don't show...
log.Warnf(ctx, "status >24hrs in the future: %+v", status)
return false, nil
}
// Don't show boosts on tag timeline.
if status.BoostOfID != "" {
return false, nil
}
// Check whether status is visible to requesting account.
visible, err := f.StatusVisible(ctx, requester, status)
if err != nil {
return false, err
}
if !visible {
log.Trace(ctx, "status not visible to timeline requester")
return false, nil
}
// Looks good!
return true, nil
}

71
internal/web/tag.go Normal file
View file

@ -0,0 +1,71 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package web
import (
"context"
"net/http"
"github.com/gin-gonic/gin"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
func (m *Module) tagGETHandler(c *gin.Context) {
ctx := c.Request.Context()
// We'll need the instance later, and we can also use it
// before then to make it easier to return a web error.
instance, errWithCode := m.processor.InstanceGetV1(ctx)
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
// Return instance we already got from the db,
// don't try to fetch it again when erroring.
instanceGet := func(ctx context.Context) (*apimodel.InstanceV1, gtserror.WithCode) {
return instance, nil
}
// We only serve text/html at this endpoint.
if _, err := apiutil.NegotiateAccept(c, []apiutil.MIME{apiutil.TextHTML}...); err != nil {
apiutil.WebErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), instanceGet)
return
}
tagName, errWithCode := apiutil.ParseTagName(c.Param(apiutil.TagNameKey))
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, instanceGet)
return
}
stylesheets := []string{
assetsPathPrefix + "/Fork-Awesome/css/fork-awesome.min.css",
distPathPrefix + "/status.css",
distPathPrefix + "/tag.css",
}
c.HTML(http.StatusOK, "tag.tmpl", gin.H{
"instance": instance,
"ogMeta": ogBase(instance),
"tagName": tagName,
"stylesheets": stylesheets,
})
}

6
internal/web/web.go
View file

@ -25,6 +25,7 @@ import (
"codeberg.org/gruf/go-cache/v3"
"github.com/gin-gonic/gin"
apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/log"
@ -37,7 +38,8 @@ import (
const (
confirmEmailPath = "/" + uris.ConfirmEmailPath
profileGroupPath = "/@:" + usernameKey
statusPath = "/statuses/:" + statusIDKey // leave out the '/@:username' prefix as this will be served within the profile group
statusPath = "/statuses/:" + apiutil.WebStatusIDKey // leave out the '/@:username' prefix as this will be served within the profile group
tagsPath = "/tags/:" + apiutil.TagNameKey
customCSSPath = profileGroupPath + "/custom.css"
rssFeedPath = profileGroupPath + "/feed.rss"
assetsPathPrefix = "/assets"
@ -49,7 +51,6 @@ const (
tokenParam = "token"
usernameKey = "username"
statusIDKey = "status"
cacheControlHeader = "Cache-Control" // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
cacheControlNoCache = "no-cache" // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#response_directives
@ -107,6 +108,7 @@ func (m *Module) Route(r router.Router, mi ...gin.HandlerFunc) {
r.AttachHandler(http.MethodGet, robotsPath, m.robotsGETHandler)
r.AttachHandler(http.MethodGet, aboutPath, m.aboutGETHandler)
r.AttachHandler(http.MethodGet, domainBlockListPath, m.domainBlockListGETHandler)
r.AttachHandler(http.MethodGet, tagsPath, m.tagGETHandler)
// Attach redirects from old endpoints to current ones for backwards compatibility
r.AttachHandler(http.MethodGet, "/auth/edit", func(c *gin.Context) { c.Redirect(http.StatusMovedPermanently, userPanelPath) })