mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-12-15 07:43:01 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Parse content warning to HTML, serialize via client API as plaintext (#3876)

Browse source 
* [feature] Parse content warning as HTML, serialize via API to plaintext

* tidy up some cruft

* whoops

* oops

* i'm da joker baybee

* clemency muy lorde

* rename some of the text functions for clarity

* jiggle the opts

* fiddle de deee

* hopefully the last test fix i ever have to do in my beautiful life
This commit is contained in:
tobi 2025-03-07 15:04:34 +01:00 • committed by GitHub
commit d8113c11e4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
48 changed files with 985 additions and 635 deletions
Download patch file Download diff file Expand all files Collapse all files

16
internal/typeutils/internaltoas_test.go
View file

@ -531,9 +531,9 @@ func (suite *InternalToASTestSuite) TestStatusToAS() {
"attachment": [],
"attributedTo": "http://localhost:8080/users/the_mighty_zork",
"cc": "http://localhost:8080/users/the_mighty_zork/followers",
"content": "hello everyone!",
"content": "\u003cp\u003ehello everyone!\u003c/p\u003e",
"contentMap": {
"en": "hello everyone!"
"en": "\u003cp\u003ehello everyone!\u003c/p\u003e"
},
"id": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
"interactionPolicy": {
@ -613,9 +613,9 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
],
"attributedTo": "http://localhost:8080/users/admin",
"cc": "http://localhost:8080/users/admin/followers",
"content": "hello world! #welcome ! first post on the instance :rainbow: !",
"content": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e",
"contentMap": {
"en": "hello world! #welcome ! first post on the instance :rainbow: !"
"en": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e"
},
"id": "http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
"interactionPolicy": {
@ -713,9 +713,9 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
],
"attributedTo": "http://localhost:8080/users/admin",
"cc": "http://localhost:8080/users/admin/followers",
"content": "hello world! #welcome ! first post on the instance :rainbow: !",
"content": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e",
"contentMap": {
"en": "hello world! #welcome ! first post on the instance :rainbow: !"
"en": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e"
},
"id": "http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
"interactionPolicy": {
@ -805,9 +805,9 @@ func (suite *InternalToASTestSuite) TestStatusToASWithMentions() {
"http://localhost:8080/users/admin/followers",
"http://localhost:8080/users/the_mighty_zork"
],
"content": "hi @the_mighty_zork welcome to the instance!",
"content": "\u003cp\u003ehi \u003cspan class=\"h-card\"\u003e\u003ca href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e@\u003cspan\u003ethe_mighty_zork\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e welcome to the instance!\u003c/p\u003e",
"contentMap": {
"en": "hi @the_mighty_zork welcome to the instance!"
"en": "\u003cp\u003ehi \u003cspan class=\"h-card\"\u003e\u003ca href=\"http://localhost:8080/@1happyturtle\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\"\u003e@\u003cspan\u003ethe_mighty_zork\u003c/span\u003e\u003c/a\u003e\u003c/span\u003e welcome to the instance!\u003c/p\u003e"
},
"id": "http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0",
"inReplyTo": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",

14
internal/typeutils/internaltofrontend.go
View file

@ -40,6 +40,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/language"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/uris"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@ -1125,13 +1126,16 @@ func (c *Converter) StatusToWebStatus(
}
webStatus := &apimodel.WebStatus{
Status: apiStatus,
Account: acct,
Status: apiStatus,
SpoilerContent: s.ContentWarning,
Account: acct,
}
// Whack a newline before and after each "pre" to make it easier to outdent it.
webStatus.Content = strings.ReplaceAll(webStatus.Content, "<pre>", "\n<pre>")
webStatus.Content = strings.ReplaceAll(webStatus.Content, "</pre>", "</pre>\n")
webStatus.SpoilerContent = strings.ReplaceAll(webStatus.SpoilerContent, "<pre>", "\n<pre>")
webStatus.SpoilerContent = strings.ReplaceAll(webStatus.SpoilerContent, "</pre>", "</pre>\n")
// Add additional information for template.
// Assume empty langs, hope for not empty language.
@ -1372,7 +1376,6 @@ func (c *Converter) baseStatusToFrontend(
InReplyToID: nil, // Set below.
InReplyToAccountID: nil, // Set below.
Sensitive: *s.Sensitive,
SpoilerText: s.ContentWarning,
Visibility: c.VisToAPIVis(ctx, s.Visibility),
LocalOnly: s.IsLocalOnly(),
Language: nil, // Set below.
@ -1393,6 +1396,11 @@ func (c *Converter) baseStatusToFrontend(
Text: s.Text,
ContentType: ContentTypeToAPIContentType(s.ContentType),
InteractionPolicy: *apiInteractionPolicy,
// Mastodon API says spoiler_text should be *text*, not HTML, so
// parse any HTML back to plaintext when serializing via the API,
// attempting to preserve semantic intent to keep it readable.
SpoilerText: text.ParseHTMLToPlain(s.ContentWarning),
}
if at := s.EditedAt; !at.IsZero() {

166
internal/typeutils/internaltofrontend_test.go
View file

@ -490,7 +490,155 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
"muted": false,
"bookmarked": true,
"pinned": false,
"content": "hello world! #welcome ! first post on the instance :rainbow: !",
"content": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e",
"reblog": null,
"application": {
"name": "superseriousbusiness",
"website": "https://superserious.business"
},
"account": {
"id": "01F8MH17FWEB39HZJ76B6VXSKF",
"username": "admin",
"acct": "admin",
"display_name": "",
"locked": false,
"discoverable": true,
"bot": false,
"created_at": "2022-05-17T13:10:59.000Z",
"note": "",
"url": "http://localhost:8080/@admin",
"avatar": "",
"avatar_static": "",
"header": "http://localhost:8080/assets/default_header.webp",
"header_static": "http://localhost:8080/assets/default_header.webp",
"header_description": "Flat gray background (default header).",
"followers_count": 1,
"following_count": 1,
"statuses_count": 4,
"last_status_at": "2021-10-20",
"emojis": [],
"fields": [],
"enable_rss": true,
"roles": [
{
"id": "admin",
"name": "admin",
"color": ""
}
],
"group": false
},
"media_attachments": [
{
"id": "01F8MH6NEM8D7527KZAECTCR76",
"type": "image",
"url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
"text_url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
"preview_url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.webp",
"remote_url": null,
"preview_remote_url": null,
"meta": {
"original": {
"width": 1200,
"height": 630,
"size": "1200x630",
"aspect": 1.9047619
},
"small": {
"width": 512,
"height": 268,
"size": "512x268",
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
"blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj"
}
],
"mentions": [],
"tags": [
{
"name": "welcome",
"url": "http://localhost:8080/tags/welcome"
}
],
"emojis": [
{
"shortcode": "rainbow",
"url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png",
"static_url": "http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png",
"visible_in_picker": true,
"category": "reactions"
}
],
"card": null,
"poll": null,
"text": "hello world! #welcome ! first post on the instance :rainbow: !",
"content_type": "text/plain",
"interaction_policy": {
"can_favourite": {
"always": [
"public",
"me"
],
"with_approval": []
},
"can_reply": {
"always": [
"public",
"me"
],
"with_approval": []
},
"can_reblog": {
"always": [
"public",
"me"
],
"with_approval": []
}
}
}`, string(b))
}
func (suite *InternalToFrontendTestSuite) TestStatusToFrontendHTMLContentWarning() {
// Change status content warning.
testStatus := new(gtsmodel.Status)
*testStatus = *suite.testStatuses["admin_account_status_1"]
testStatus.ContentWarning = `<p>First paragraph of content warning</p><h4>Here's the title!</h4><p></p><p>Big boobs<br>Tee hee!<br><br>Some more text<br>And a bunch more<br><br>Hasta la victoria siempre!</p>`
requestingAccount := suite.testAccounts["local_account_1"]
apiStatus, err := suite.typeconverter.StatusToAPIStatus(context.Background(), testStatus, requestingAccount, statusfilter.FilterContextNone, nil, nil)
suite.NoError(err)
b, err := json.MarshalIndent(apiStatus, "", " ")
suite.NoError(err)
suite.Equal(`{
"id": "01F8MH75CBF9JFX4ZAD54N0W0R",
"created_at": "2021-10-20T11:36:45.000Z",
"edited_at": null,
"in_reply_to_id": null,
"in_reply_to_account_id": null,
"sensitive": false,
"spoiler_text": "First paragraph of content warning\n\nHere's the title!\n\nBig boobs\nTee hee!\n\nSome more text\nAnd a bunch more\n\nHasta la victoria siempre!",
"visibility": "public",
"language": "en",
"uri": "http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
"url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
"replies_count": 1,
"reblogs_count": 0,
"favourites_count": 1,
"favourited": true,
"reblogged": false,
"muted": false,
"bookmarked": true,
"pinned": false,
"content": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e",
"reblog": null,
"application": {
"name": "superseriousbusiness",
@ -671,7 +819,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() {
"muted": false,
"bookmarked": true,
"pinned": false,
"content": "hello world! #welcome ! first post on the instance :rainbow: ! fnord",
"content": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e fnord",
"reblog": null,
"application": {
"name": "superseriousbusiness",
@ -861,7 +1009,7 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() {
"muted": false,
"bookmarked": true,
"pinned": false,
"content": "hello world! #welcome ! first post on the instance :rainbow: ! fnord",
"content": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e fnord",
"reblog": null,
"application": {
"name": "superseriousbusiness",
@ -1591,7 +1739,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
"muted": false,
"bookmarked": true,
"pinned": false,
"content": "hello world! #welcome ! first post on the instance :rainbow: !",
"content": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance :rainbow: !\u003c/p\u003e",
"reblog": null,
"application": {
"name": "superseriousbusiness",
@ -1737,7 +1885,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendPartialInteraction
"muted": false,
"bookmarked": false,
"pinned": false,
"content": "this is a very personal post that I don't want anyone to interact with at all, and i only want mutuals to see it",
"content": "\u003cp\u003ethis is a very personal post that I don't want anyone to interact with at all, and i only want mutuals to see it\u003c/p\u003e",
"reblog": null,
"application": {
"name": "really cool gts application",
@ -2818,7 +2966,7 @@ func (suite *InternalToFrontendTestSuite) TestAdminReportToFrontend2() {
"muted": false,
"bookmarked": false,
"pinned": false,
"content": "dark souls status bot: \"thoughts of dog\"",
"content": "\u003cp\u003edark souls status bot: \"thoughts of dog\"\u003c/p\u003e",
"reblog": null,
"account": {
"id": "01F8MH5ZK5VRH73AKHQM6Y9VNX",
@ -3332,7 +3480,7 @@ func (suite *InternalToFrontendTestSuite) TestIntReqToAPI() {
"muted": false,
"bookmarked": false,
"pinned": false,
"content": "🐢 i don't mind people sharing and liking this one but I want to moderate replies to it 🐢",
"content": "\u003cp\u003e🐢 i don't mind people sharing and liking this one but I want to moderate replies to it 🐢\u003c/p\u003e",
"reblog": null,
"application": {
"name": "kindaweird",
@ -3599,7 +3747,7 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPISelfConvo() {
"muted": false,
"bookmarked": false,
"pinned": false,
"content": "hello everyone!",
"content": "\u003cp\u003ehello everyone!\u003c/p\u003e",
"reblog": null,
"application": {
"name": "really cool gts application",
@ -3769,7 +3917,7 @@ func (suite *InternalToFrontendTestSuite) TestConversationToAPI() {
"muted": false,
"bookmarked": false,
"pinned": false,
"content": "hello everyone!",
"content": "\u003cp\u003ehello everyone!\u003c/p\u003e",
"reblog": null,
"application": {
"name": "really cool gts application",

4
internal/typeutils/internaltorss_test.go
View file

@ -54,7 +54,7 @@ func (suite *InternalToRSSTestSuite) TestStatusToRSSItem1() {
suite.Equal("", item.Enclosure.Length)
suite.Equal("", item.Enclosure.Type)
suite.Equal("", item.Enclosure.Url)
suite.Equal("hello everyone!", item.Content)
suite.Equal("<p>hello everyone!</p>", item.Content)
}
func (suite *InternalToRSSTestSuite) TestStatusToRSSItem2() {
@ -79,7 +79,7 @@ func (suite *InternalToRSSTestSuite) TestStatusToRSSItem2() {
suite.Equal("62529", item.Enclosure.Length)
suite.Equal("image/jpeg", item.Enclosure.Type)
suite.Equal("http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", item.Enclosure.Url)
suite.Equal("hello world! #welcome ! first post on the instance <img src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" width=\"25\" height=\"25\" /> !", item.Content)
suite.Equal("<p>hello world! <a href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>welcome</span></a> ! first post on the instance <img src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" width=\"25\" height=\"25\" /> !</p>", item.Content)
}
func (suite *InternalToRSSTestSuite) TestStatusToRSSItem3() {

11
internal/typeutils/util.go
View file

@ -28,7 +28,6 @@ import (
"strconv"
"strings"
"github.com/k3a/html2text"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -247,7 +246,7 @@ func systemMessage(
wrappedNote.WriteString(`<div class="gts-system-message `)
wrappedNote.WriteString(messageClass)
wrappedNote.WriteString(`">`)
wrappedNote.WriteString(text.SanitizeToHTML(unsanitizedNoteHTML))
wrappedNote.WriteString(text.SanitizeHTML(unsanitizedNoteHTML))
wrappedNote.WriteString(`</div>`)
return wrappedNote.String()
@ -380,15 +379,11 @@ func filterableFields(s *gtsmodel.Status) []string {
// Status content. Though we have raw text
// available for statuses created on our
// instance, use the html2text version to
// instance, use the plaintext version to
// remove markdown-formatting characters
// and ensure more consistent filtering.
if s.Content != "" {
text := html2text.HTML2TextWithOptions(
s.Content,
html2text.WithLinksInnerText(),
html2text.WithUnixLineBreaks(),
)
text := text.ParseHTMLToPlain(s.Content)
if text != "" {
fields = append(fields, text)
}

4
internal/typeutils/wrap_test.go
View file

@ -90,9 +90,9 @@ func (suite *WrapTestSuite) TestWrapNoteInCreate() {
"attachment": [],
"attributedTo": "http://localhost:8080/users/the_mighty_zork",
"cc": "http://localhost:8080/users/the_mighty_zork/followers",
"content": "hello everyone!",
"content": "\u003cp\u003ehello everyone!\u003c/p\u003e",
"contentMap": {
"en": "hello everyone!"
"en": "\u003cp\u003ehello everyone!\u003c/p\u003e"
},
"id": "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
"interactionPolicy": {