diff --git a/internal/ap/normalize.go b/internal/ap/normalize.go index 23035419f..ffe90733c 100644 --- a/internal/ap/normalize.go +++ b/internal/ap/normalize.go @@ -113,7 +113,7 @@ func normalizeContent(rawContent interface{}) string { // // TODO: sanitize differently based on mediaType. // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-mediatype - content = text.SanitizeToHTML(content) + content = text.SanitizeHTML(content) content = text.MinifyHTML(content) return content } @@ -248,7 +248,7 @@ func NormalizeIncomingSummary(item WithSummary, rawJSON map[string]interface{}) // Summary should be HTML encoded: // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-summary - summary = text.SanitizeToHTML(summary) + summary = text.SanitizeHTML(summary) summary = text.MinifyHTML(summary) // Set normalized summary property from the raw string; this @@ -339,7 +339,7 @@ func NormalizeIncomingName(item WithName, rawJSON map[string]interface{}) { // // todo: We probably want to update this to allow // *escaped* HTML markup, but for now just nuke it. - name = text.SanitizeToPlaintext(name) + name = text.StripHTMLFromText(name) // Set normalized name property from the raw string; this // will replace any existing name property on the item. @@ -369,7 +369,7 @@ func NormalizeIncomingValue(item WithValue, rawJSON map[string]interface{}) { // Value often contains links or // mentions or other little snippets. // Sanitize to HTML to allow these. - value = text.SanitizeToHTML(value) + value = text.SanitizeHTML(value) // Set normalized name property from the raw string; this // will replace any existing value property on the item. diff --git a/internal/api/client/admin/reportsget_test.go b/internal/api/client/admin/reportsget_test.go index 8639e0c6e..ec15b05d3 100644 --- a/internal/api/client/admin/reportsget_test.go +++ b/internal/api/client/admin/reportsget_test.go @@ -508,7 +508,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetAll() { "muted": false, "bookmarked": false, "pinned": false, - "content": "dark souls status bot: \"thoughts of dog\"", + "content": "\u003cp\u003edark souls status bot: \"thoughts of dog\"\u003c/p\u003e", "reblog": null, "account": { "id": "01F8MH5ZK5VRH73AKHQM6Y9VNX", @@ -765,7 +765,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetCreatedByAccount() { "muted": false, "bookmarked": false, "pinned": false, - "content": "dark souls status bot: \"thoughts of dog\"", + "content": "\u003cp\u003edark souls status bot: \"thoughts of dog\"\u003c/p\u003e", "reblog": null, "account": { "id": "01F8MH5ZK5VRH73AKHQM6Y9VNX", @@ -1022,7 +1022,7 @@ func (suite *ReportsGetTestSuite) TestReportsGetTargetAccount() { "muted": false, "bookmarked": false, "pinned": false, - "content": "dark souls status bot: \"thoughts of dog\"", + "content": "\u003cp\u003edark souls status bot: \"thoughts of dog\"\u003c/p\u003e", "reblog": null, "account": { "id": "01F8MH5ZK5VRH73AKHQM6Y9VNX", diff --git a/internal/api/client/search/searchget_test.go b/internal/api/client/search/searchget_test.go index 2c4efd19c..318010387 100644 --- a/internal/api/client/search/searchget_test.go +++ b/internal/api/client/search/searchget_test.go @@ -916,7 +916,7 @@ func (suite *SearchGetTestSuite) TestSearchAAny() { } suite.Len(searchResult.Accounts, 5) - suite.Len(searchResult.Statuses, 8) + suite.Len(searchResult.Statuses, 9) suite.Len(searchResult.Hashtags, 0) } @@ -959,7 +959,7 @@ func (suite *SearchGetTestSuite) TestSearchAAnyFollowingOnly() { } suite.Len(searchResult.Accounts, 2) - suite.Len(searchResult.Statuses, 8) + suite.Len(searchResult.Statuses, 9) suite.Len(searchResult.Hashtags, 0) } @@ -1002,7 +1002,7 @@ func (suite *SearchGetTestSuite) TestSearchAStatuses() { } suite.Len(searchResult.Accounts, 0) - suite.Len(searchResult.Statuses, 8) + suite.Len(searchResult.Statuses, 9) suite.Len(searchResult.Hashtags, 0) } diff --git a/internal/api/client/statuses/statusboost_test.go b/internal/api/client/statuses/statusboost_test.go index fc750ca38..a9fee34f7 100644 --- a/internal/api/client/statuses/statusboost_test.go +++ b/internal/api/client/statuses/statusboost_test.go @@ -144,7 +144,7 @@ func (suite *StatusBoostTestSuite) TestPostBoost() { }, "bookmarked": true, "card": null, - "content": "hello world! #welcome ! first post on the instance :rainbow: !", + "content": "
hello world! #welcome ! first post on the instance :rainbow: !
", "content_type": "text/plain", "created_at": "right the hell just now babyee", "edited_at": null, @@ -331,7 +331,7 @@ func (suite *StatusBoostTestSuite) TestPostBoostOwnFollowersOnly() { }, "bookmarked": false, "card": null, - "content": "hi!", + "content": "hi!
", "content_type": "text/plain", "created_at": "right the hell just now babyee", "edited_at": null, diff --git a/internal/api/client/statuses/statusfave_test.go b/internal/api/client/statuses/statusfave_test.go index 42b812fdf..983935184 100644 --- a/internal/api/client/statuses/statusfave_test.go +++ b/internal/api/client/statuses/statusfave_test.go @@ -103,7 +103,7 @@ func (suite *StatusFaveTestSuite) TestPostFave() { }, "bookmarked": false, "card": null, - "content": "๐๐๐๐๐", + "content": "๐๐๐๐๐
", "content_type": "text/plain", "created_at": "right the hell just now babyee", "edited_at": null, diff --git a/internal/api/client/statuses/statushistory_test.go b/internal/api/client/statuses/statushistory_test.go index 61c15b58a..fe650402f 100644 --- a/internal/api/client/statuses/statushistory_test.go +++ b/internal/api/client/statuses/statushistory_test.go @@ -91,7 +91,7 @@ func (suite *StatusHistoryTestSuite) TestGetHistory() { suite.Equal(`[ { - "content": "hello everyone!", + "content": "\u003cp\u003ehello everyone!\u003c/p\u003e", "spoiler_text": "introduction post", "sensitive": true, "created_at": "2021-10-20T10:40:37.000Z", diff --git a/internal/api/client/statuses/statusmute_test.go b/internal/api/client/statuses/statusmute_test.go index 174ac14dc..a98eff78a 100644 --- a/internal/api/client/statuses/statusmute_test.go +++ b/internal/api/client/statuses/statusmute_test.go @@ -108,7 +108,7 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { "muted": true, "bookmarked": false, "pinned": false, - "content": "hello everyone!", + "content": "\u003cp\u003ehello everyone!\u003c/p\u003e", "reblog": null, "application": { "name": "really cool gts application", @@ -198,7 +198,7 @@ func (suite *StatusMuteTestSuite) TestMuteUnmuteStatus() { "muted": false, "bookmarked": false, "pinned": false, - "content": "hello everyone!", + "content": "\u003cp\u003ehello everyone!\u003c/p\u003e", "reblog": null, "application": { "name": "really cool gts application", diff --git a/internal/api/client/statuses/statusunfave_test.go b/internal/api/client/statuses/statusunfave_test.go index 4ef28b3b7..d02de47a5 100644 --- a/internal/api/client/statuses/statusunfave_test.go +++ b/internal/api/client/statuses/statusunfave_test.go @@ -129,7 +129,6 @@ func (suite *StatusUnfaveTestSuite) TestPostAlreadyNotFaved() { err = json.Unmarshal(b, statusReply) assert.NoError(suite.T(), err) - assert.Equal(suite.T(), targetStatus.ContentWarning, statusReply.SpoilerText) assert.Equal(suite.T(), targetStatus.Content, statusReply.Content) assert.True(suite.T(), statusReply.Sensitive) assert.Equal(suite.T(), apimodel.VisibilityPublic, statusReply.Visibility) diff --git a/internal/api/model/status.go b/internal/api/model/status.go index fe82f09e3..ec09f702d 100644 --- a/internal/api/model/status.go +++ b/internal/api/model/status.go @@ -127,6 +127,10 @@ type Status struct { type WebStatus struct { *Status + // HTML version of spoiler content + // (ie., not converted to plaintext). + SpoilerContent string `json:"-"` + // Override API account with web account. Account *WebAccount `json:"account"` diff --git a/internal/api/util/opengraph.go b/internal/api/util/opengraph.go index 094c80021..770bada83 100644 --- a/internal/api/util/opengraph.go +++ b/internal/api/util/opengraph.go @@ -67,7 +67,7 @@ func OGBase(instance *apimodel.InstanceV1) *OGMeta { } og := &OGMeta{ - Title: text.SanitizeToPlaintext(instance.Title) + " - GoToSocial", + Title: text.StripHTMLFromText(instance.Title) + " - GoToSocial", Type: "website", Locale: locale, URL: instance.URI, @@ -161,7 +161,7 @@ func AccountTitle(account *apimodel.WebAccount, accountDomain string) string { // ParseDescription returns a string description which is // safe to use as a template.HTMLAttr inside templates. func ParseDescription(in string) string { - i := text.SanitizeToPlaintext(in) + i := text.StripHTMLFromText(in) i = strings.ReplaceAll(i, "\n", " ") i = strings.Join(strings.Fields(i), " ") i = html.EscapeString(i) diff --git a/internal/cache/size.go b/internal/cache/size.go index 3920a6f19..cdaf3a03b 100644 --- a/internal/cache/size.go +++ b/internal/cache/size.go @@ -666,6 +666,7 @@ func sizeofStatus() uintptr { BoostOfID: exampleID, BoostOfAccountID: exampleID, ContentWarning: exampleUsername, // similar length + ContentWarningText: exampleUsername, // similar length Visibility: gtsmodel.VisibilityPublic, Sensitive: func() *bool { ok := false; return &ok }(), Language: "en", diff --git a/internal/db/bundb/migrations/20250305205820_content_warning_fixes.go b/internal/db/bundb/migrations/20250305205820_content_warning_fixes.go new file mode 100644 index 000000000..cf4de834c --- /dev/null +++ b/internal/db/bundb/migrations/20250305205820_content_warning_fixes.go @@ -0,0 +1,61 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, seetags. +func (f *Formatter) FromMarkdownBasic( + ctx context.Context, + parseMention gtsmodel.ParseMentionFunc, + authorID string, + statusID string, + input string, +) *FormatResult { + res := f.fromMarkdown( + ctx, + true, // basic = true + parseMention, + authorID, + statusID, + input, + ) + + res.HTML = unwrapParagraph(res.HTML) + return res +} + +// fromMarkdown parses the given input text either +// with or without emojis, and returns the result. +func (f *Formatter) fromMarkdown( + ctx context.Context, + basic bool, + parseMention gtsmodel.ParseMentionFunc, + authorID string, + statusID string, + input string, +) *FormatResult { + var ( + result = new(FormatResult) + opts []renderer.Option + ) + + if basic { + // Don't allow raw HTML tags, + // markdown syntax only. + opts = []renderer.Option{ + html.WithXHTML(), + html.WithHardWraps(), + } + } else { + opts = []renderer.Option{ + html.WithXHTML(), + html.WithHardWraps(), + + // Allow raw HTML tags, we + // sanitize at the end anyway. + html.WithUnsafe(), + } + } // Instantiate goldmark parser for // markdown, using custom renderer // to add hashtag/mention links. md := goldmark.New( goldmark.WithRendererOptions( - html.WithXHTML(), - html.WithHardWraps(), - // Allows raw HTML. We sanitize - // at the end so this is OK. - html.WithUnsafe(), + opts..., ), goldmark.WithExtensions( &customRenderer{ @@ -59,7 +132,9 @@ func (f *Formatter) FromMarkdown( parseMention, authorID, statusID, - false, // emojiOnly = false. + // If basic, pass + // emojiOnly = true. + basic, result, }, // Turns URLs into links. @@ -85,8 +160,36 @@ func (f *Formatter) FromMarkdown( // Clean and shrink HTML. result.HTML = byteutil.B2S(htmlBytes.Bytes()) - result.HTML = SanitizeToHTML(result.HTML) + result.HTML = SanitizeHTML(result.HTML) result.HTML = MinifyHTML(result.HTML) return result } + +var parasRegexp = regexp.MustCompile(`?p>`) + +// unwrapParagraph removes opening and closing paragraph tags +// of input HTML, if input html is a single paragraph only. +func unwrapParagraph(html string) string { + if !strings.HasPrefix(html, "
") { + return html + } + + if !strings.HasSuffix(html, "
") { + return html + } + + // Make a substring excluding the + // opening and closing paragraph tags. + sub := html[3 : len(html)-4] + + // If there are still other paragraph tags left + // inside the substring, return html unchanged. + containsOtherParas := parasRegexp.MatchString(sub) + if containsOtherParas { + return html + } + + // Return the substring. + return sub +} diff --git a/internal/text/markdown_test.go b/internal/text/markdown_test.go index 153673415..923487978 100644 --- a/internal/text/markdown_test.go +++ b/internal/text/markdown_test.go @@ -41,43 +41,45 @@ that was some JSON :) ` const ( - simpleMarkdown = "# Title\n\nHere's a simple text in markdown.\n\nHere's a [link](https://example.org)." - simpleMarkdownExpected = "Here's a simple text in markdown.
Here's a link.
" - withCodeBlockExpected = "Below is some JSON.
{\n "key": "value",\n "another_key": [\n "value1",\n "value2"\n ]\n}\nthat was some JSON :)
" - withInlineCode = "`Nobody tells you about theSECRET CODE, do they?`"
- withInlineCodeExpected = "Nobody tells you about the <code><del>SECRET CODE</del></code>, do they?
, do they?`"
- withInlineCode2Expected = "Nobody tells you about the </code><del>SECRET CODE</del><code>, do they?
"
- withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!"
- withHashtagExpected = "Title
here's a simple status that uses hashtag #Hashtag!
"
- withTamilHashtag = "here's a simple status that uses a hashtag in Tamil #เฎคเฎฎเฎฟเฎดเฏ"
- withTamilHashtagExpected = "here's a simple status that uses a hashtag in Tamil #เฎคเฎฎเฎฟเฎดเฏ
"
- mdWithHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a link.\n\nHere's an image:
"
- mdWithHTMLExpected = "Title
Here's a simple text in markdown.
Here's a link.
Here's an image:
"
- mdWithCheekyHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a cheeky little script: "
- mdWithCheekyHTMLExpected = "Title
Here's a simple text in markdown.
Here's a cheeky little script:
"
- mdWithHashtagInitial = "#welcome #Hashtag"
- mdWithHashtagInitialExpected = ""
- mdCodeBlockWithNewlines = "some code coming up\n\n```\n\n\n\n```\nthat was some code"
- mdCodeBlockWithNewlinesExpected = "some code coming up
\n\n\n
that was some code
"
- mdWithFootnote = "fox mulder,fbi.[^1]\n\n[^1]: federated bureau of investigation"
- mdWithFootnoteExpected = "fox mulder,fbi.[^1]
[^1]: federated bureau of investigation
"
- mdWithBlockQuote = "get ready, there's a block quote coming:\n\n>line1\n>line2\n>\n>line3\n\n"
- mdWithBlockQuoteExpected = "get ready, there's a block quote coming:
line1
line2
line3
"
- mdHashtagAndCodeBlock = "#Hashtag\n\n```\n#Hashtag\n```"
- mdHashtagAndCodeBlockExpected = "#Hashtag\n
"
- mdMentionAndCodeBlock = "@the_mighty_zork\n\n```\n@the_mighty_zork\n```"
- mdMentionAndCodeBlockExpected = "@the_mighty_zork\n
"
- mdWithSmartypants = "\"you have to quargle the bleepflorp\" they said with 1/2 of nominal speed and 1/3 of the usual glumping"
- mdWithSmartypantsExpected = "\"you have to quargle the bleepflorp\" they said with 1/2 of nominal speed and 1/3 of the usual glumping
"
- mdWithAsciiHeart = "hello <3 old friend <3 i loved u 3 :(( you stole my heart"
- mdWithAsciiHeartExpected = "hello <3 old friend <3 i loved u </3 :(( you stole my heart
"
- mdWithStrikethrough = "I have ~~mdae~~ made an error"
- mdWithStrikethroughExpected = "I have mdae made an error
"
- mdWithLink = "Check out this code, i heard it was written by a sloth https://github.com/superseriousbusiness/gotosocial"
- mdWithLinkExpected = "Check out this code, i heard it was written by a sloth https://github.com/superseriousbusiness/gotosocial
"
- mdObjectInCodeBlock = "@foss_satan@fossbros-anonymous.io this is how to mention a user\n```\n@the_mighty_zork hey bud! nice #ObjectOrientedProgramming software you've been writing lately! :rainbow:\n```\nhope that helps"
- mdObjectInCodeBlockExpected = "@foss_satan this is how to mention a user
@the_mighty_zork hey bud! nice #ObjectOrientedProgramming software you've been writing lately! :rainbow:\n
hope that helps
"
+ simpleMarkdown = "# Title\n\nHere's a simple text in markdown.\n\nHere's a [link](https://example.org)."
+ simpleMarkdownExpected = "Title
Here's a simple text in markdown.
Here's a link.
"
+ withCodeBlockExpected = "Title
Below is some JSON.
{\n "key": "value",\n "another_key": [\n "value1",\n "value2"\n ]\n}\n
that was some JSON :)
"
+ withInlineCode = "`Nobody tells you about the SECRET CODE, do they?`"
+ withInlineCodeExpected = "Nobody tells you about the <code><del>SECRET CODE</del></code>, do they?
"
+ withInlineCode2 = "`Nobody tells you about the , do they?`"
+ withInlineCode2Expected = "Nobody tells you about the </code><del>SECRET CODE</del><code>, do they?
"
+ withHashtag = "# Title\n\nhere's a simple status that uses hashtag #Hashtag!"
+ withHashtagExpected = "Title
here's a simple status that uses hashtag #Hashtag!
"
+ withTamilHashtag = "here's a simple status that uses a hashtag in Tamil #เฎคเฎฎเฎฟเฎดเฏ"
+ withTamilHashtagExpected = "here's a simple status that uses a hashtag in Tamil #เฎคเฎฎเฎฟเฎดเฏ
"
+ mdWithHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a link.\n\nHere's an image:
"
+ mdWithHTMLExpected = "Title
Here's a simple text in markdown.
Here's a link.
Here's an image:
"
+ mdWithCheekyHTML = "# Title\n\nHere's a simple text in markdown.\n\nHere's a cheeky little script: "
+ mdWithCheekyHTMLExpected = "Title
Here's a simple text in markdown.
Here's a cheeky little script:
"
+ mdWithHashtagInitial = "#welcome #Hashtag"
+ mdWithHashtagInitialExpected = ""
+ mdCodeBlockWithNewlines = "some code coming up\n\n```\n\n\n\n```\nthat was some code"
+ mdCodeBlockWithNewlinesExpected = "some code coming up
\n\n\n
that was some code
"
+ mdWithFootnote = "fox mulder,fbi.[^1]\n\n[^1]: federated bureau of investigation"
+ mdWithFootnoteExpected = "fox mulder,fbi.[^1]
[^1]: federated bureau of investigation
"
+ mdWithBlockQuote = "get ready, there's a block quote coming:\n\n>line1\n>line2\n>\n>line3\n\n"
+ mdWithBlockQuoteExpected = "get ready, there's a block quote coming:
line1
line2
line3
"
+ mdHashtagAndCodeBlock = "#Hashtag\n\n```\n#Hashtag\n```"
+ mdHashtagAndCodeBlockExpected = "#Hashtag\n
"
+ mdMentionAndCodeBlock = "@the_mighty_zork\n\n```\n@the_mighty_zork\n```"
+ mdMentionAndCodeBlockExpected = "@the_mighty_zork\n
"
+ mdMentionAndCodeBlockBasicExpected = "@the_mighty_zork
@the_mighty_zork\n
"
+ mdWithSmartypants = "\"you have to quargle the bleepflorp\" they said with 1/2 of nominal speed and 1/3 of the usual glumping"
+ mdWithSmartypantsExpected = "\"you have to quargle the bleepflorp\" they said with 1/2 of nominal speed and 1/3 of the usual glumping
"
+ mdWithAsciiHeart = "hello <3 old friend <3 i loved u 3 :(( you stole my heart"
+ mdWithAsciiHeartExpected = "hello <3 old friend <3 i loved u </3 :(( you stole my heart
"
+ mdWithStrikethrough = "I have ~~mdae~~ made an error"
+ mdWithStrikethroughExpected = "I have mdae made an error
"
+ mdWithLink = "Check out this code, i heard it was written by a sloth https://github.com/superseriousbusiness/gotosocial"
+ mdWithLinkExpected = "Check out this code, i heard it was written by a sloth https://github.com/superseriousbusiness/gotosocial
"
+ mdWithLinkBasicExpected = "Check out this code, i heard it was written by a sloth https://github.com/superseriousbusiness/gotosocial"
+ mdObjectInCodeBlock = "@foss_satan@fossbros-anonymous.io this is how to mention a user\n```\n@the_mighty_zork hey bud! nice #ObjectOrientedProgramming software you've been writing lately! :rainbow:\n```\nhope that helps"
+ mdObjectInCodeBlockExpected = "@foss_satan this is how to mention a user
@the_mighty_zork hey bud! nice #ObjectOrientedProgramming software you've been writing lately! :rainbow:\n
hope that helps
"
// Hashtags can be italicized but only with *, not _.
mdItalicHashtag = "*#hashtag*"
mdItalicHashtagExpected = ""
@@ -169,6 +171,11 @@ func (suite *MarkdownTestSuite) TestParseMentionWithCodeBlock() {
suite.Equal(mdMentionAndCodeBlockExpected, formatted.HTML)
}
+func (suite *MarkdownTestSuite) TestParseMentionWithCodeBlockBasic() {
+ formatted := suite.FromMarkdownBasic(mdMentionAndCodeBlock)
+ suite.Equal(mdMentionAndCodeBlockBasicExpected, formatted.HTML)
+}
+
func (suite *MarkdownTestSuite) TestParseSmartypants() {
formatted := suite.FromMarkdown(mdWithSmartypants)
suite.Equal(mdWithSmartypantsExpected, formatted.HTML)
@@ -189,6 +196,11 @@ func (suite *MarkdownTestSuite) TestParseLink() {
suite.Equal(mdWithLinkExpected, formatted.HTML)
}
+func (suite *MarkdownTestSuite) TestParseLinkBasic() {
+ formatted := suite.FromMarkdownBasic(mdWithLink)
+ suite.Equal(mdWithLinkBasicExpected, formatted.HTML)
+}
+
func (suite *MarkdownTestSuite) TestParseObjectInCodeBlock() {
formatted := suite.FromMarkdown(mdObjectInCodeBlock)
suite.Equal(mdObjectInCodeBlockExpected, formatted.HTML)
diff --git a/internal/text/plain.go b/internal/text/plain.go
index 362941773..ee4947bf7 100644
--- a/internal/text/plain.go
+++ b/internal/text/plain.go
@@ -20,8 +20,11 @@ package text
import (
"bytes"
"context"
+ gohtml "html"
+ "strings"
"codeberg.org/gruf/go-byteutil"
+ "github.com/k3a/html2text"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/regexes"
@@ -52,7 +55,7 @@ func (f *Formatter) FromPlain(
return f.fromPlain(
ctx,
plainTextParser,
- false, // emojiOnly = false
+ false, // basic = false
parseMention,
authorID,
statusID,
@@ -85,7 +88,7 @@ func (f *Formatter) FromPlainNoParagraph(
return f.fromPlain(
ctx,
plainTextParser,
- false, // emojiOnly = false
+ false, // basic = false
parseMention,
authorID,
statusID,
@@ -93,12 +96,14 @@ func (f *Formatter) FromPlainNoParagraph(
)
}
-// FromPlainEmojiOnly fulfils FormatFunc by parsing
+// FromPlainBasic fulfils FormatFunc by parsing
// the given plaintext input into a FormatResult.
//
// Unlike FromPlain, it will only parse emojis with
// the custom renderer, leaving aside mentions and tags.
-func (f *Formatter) FromPlainEmojiOnly(
+//
+// Resulting HTML will also NOT be wrapped in tags.
+func (f *Formatter) FromPlainBasic(
ctx context.Context,
parseMention gtsmodel.ParseMentionFunc,
authorID string,
@@ -116,7 +121,7 @@ func (f *Formatter) FromPlainEmojiOnly(
return f.fromPlain(
ctx,
plainTextParser,
- true, // emojiOnly = true
+ true, // basic = true
parseMention,
authorID,
statusID,
@@ -130,7 +135,7 @@ func (f *Formatter) FromPlainEmojiOnly(
func (f *Formatter) fromPlain(
ctx context.Context,
plainTextParser parser.Parser,
- emojiOnly bool,
+ basic bool,
parseMention gtsmodel.ParseMentionFunc,
authorID string,
statusID string,
@@ -156,7 +161,9 @@ func (f *Formatter) fromPlain(
parseMention,
authorID,
statusID,
- emojiOnly,
+ // If basic, pass
+ // emojiOnly = true.
+ basic,
result,
},
// Turns URLs into links.
@@ -181,8 +188,51 @@ func (f *Formatter) fromPlain(
// Clean and shrink HTML.
result.HTML = byteutil.B2S(htmlBytes.Bytes())
- result.HTML = SanitizeToHTML(result.HTML)
+ result.HTML = SanitizeHTML(result.HTML)
result.HTML = MinifyHTML(result.HTML)
return result
}
+
+// ParseHTMLToPlain parses the given HTML string, then
+// outputs it to equivalent plaintext while trying to
+// keep as much of the smenantic intent of the input
+// HTML as possible, ie., titles are placed on separate
+// lines, `
`s are converted to newlines, text inside
+// `` and `` tags is retained, but without
+// emphasis, `` links are unnested and the URL they
+// link to is placed in angle brackets next to them,
+// lists are replaced with newline-separated indented
+// items, etc.
+//
+// This function is useful when you need to filter on
+// HTML and want to avoid catching tags in the filter,
+// or when you want to serve something in a plaintext
+// format that may contain HTML tags (eg., CWs).
+func ParseHTMLToPlain(html string) string {
+ plain := html2text.HTML2TextWithOptions(
+ html,
+ html2text.WithLinksInnerText(),
+ html2text.WithUnixLineBreaks(),
+ html2text.WithListSupport(),
+ )
+ return strings.TrimSpace(plain)
+}
+
+// StripHTMLFromText runs text through strict sanitization
+// to completely remove any HTML from the input without
+// trying to preserve the semantic intent of any HTML tags.
+//
+// This is useful in cases where the input was not allowed
+// to contain HTML at all, and the output isn't either.
+func StripHTMLFromText(text string) string {
+ // Unescape first to catch any tricky critters.
+ content := gohtml.UnescapeString(text)
+
+ // Remove all detected HTML.
+ content = strict.Sanitize(content)
+
+ // Unescape again to return plaintext.
+ content = gohtml.UnescapeString(content)
+ return strings.TrimSpace(content)
+}
diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go
index ffa64ce44..cb8f4677e 100644
--- a/internal/text/plain_test.go
+++ b/internal/text/plain_test.go
@@ -21,6 +21,7 @@ import (
"testing"
"github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/text"
)
const (
@@ -183,6 +184,150 @@ func (suite *PlainTestSuite) TestNumbersAreNotHashtags() {
suite.Len(f.Tags, 0)
}
+func (suite *PlainTestSuite) TestParseHTMLToPlain() {
+ for _, t := range []struct {
+ html string
+ expectedPlain string
+ }{
+ {
+ // Check newlines between paras preserved.
+ html: "butting into a serious discussion about programming languages*: \"elixir? I barely know 'er! honk honk!\"
*insofar as any discussion about programming languages can truly be considered \"serious\" since programmers are fucking clowns
",
+ expectedPlain: `butting into a serious discussion about programming languages*: "elixir? I barely know 'er! honk honk!"
+
+*insofar as any discussion about programming languages can truly be considered "serious" since programmers are fucking clowns`,
+ },
+ {
+ // This one looks a bit wacky but nobody should
+ // be putting definition lists in summaries *really*.
+ html: "- Published
- Replies
- 0
- Favourites
- 4
- Reblogs
- 0
- Language
- Englishen
",
+ expectedPlain: `PublishedJan 16, 2025, 00:49Replies0Favourites4Reblogs0LanguageEnglishen`,
+ },
+ {
+ // Check
converted to newlines and leading / trailing space removed.
+ html: " i'm a milf,
i'm a lover,
do your mom,
do your brother
i'm a sinner,
i'm a saint,
i will not be ashamed!
",
+ expectedPlain: `i'm a milf,
+i'm a lover,
+do your mom,
+do your brother
+
+i'm a sinner,
+i'm a saint,
+i will not be ashamed!`,
+ },
+ {
+ // Check newlines, links, lists still more or less readable as such.
+ html: "Hello everyone, after a week or two down the release candidate mines, we've emerged blinking into the light carrying with us #GoToSocial v0.18.0 Scroingly Sloth!
https://github.com/superseriousbusiness/gotosocial/releases/tag/v0.18.0
Please read the migration notes carefully for instructions on how to upgrade to this version. This version contains several very long migrations so you will need to be patient when upgrading, and backup your database first!!
Release highlights
- Status edit support: one of our most-requested features! You can now edit your own statuses, and see instance edit history from other accounts too (if your instance has them stored).
- Push notifications: probably the second most-requested feature! GoToSocial can now send push notifications to clients via their configured push providers.
You may need to uninstall / reinstall client applications, or log out and back in again, for this feature to work. (And if you're using Tusky, make sure you've got ntfy installed). - Global instance css customization: admins can now apply custom CSS across their entire instance via the settings panel.
- Domain permission subscriptions: it's now possible to configure your instance to subscribe to CSV, JSON, or plaintext lists of domain permissions.
Each night, your instance will fetch and automatically create domain permissions (or permission drafts) based on what it finds in a subscribed list.
See the domain permission subscription documentation for more information. - Trusted-proxies helper: instances with improperly configured trusted-proxies settings will now show a warning on the homepage, so admins can make sure their instance is configured correctly. Check your own instance homepage after updating to see if you need to do anything.
- Better outbox sorting: messages from GoToSocial are now delivered more quickly to people you mention, so conversations across instances should feel a bit snappier.
- Log in button: there's now a login button in the top right of the instance homepage, which leads to a helpful page about clients, with a link to the settings panel. Should make things less confusing for new users!
- Granular stats controls: with the
instance-stats-mode setting, admins can now choose if and how their instance serves stats via the nodeinfo endpoints. Existing behavior from v0.17.0 is the default. - Post backdating: via the API you can now backdate posts (if enabled in config.yaml). This is our first step towards making it possible to import your post history from elsewhere into your GoToSocial instance. While there's no way to do this in the settings panel yet, you can already use third-party tools like Slurp to import posts from a Mastodon export (see Slurp).
- Configurable sign-up limits: you can now configure your sign-up backlog length and sign-up throttling (defaults remain the same).
- NetBSD and FreeBSD builds: yep!
- Respect users
prefers-color-scheme preference: there's now a light mode default theme to complement our trusty dark mode theme, and the theme will switch based on a visitor's prefers-color-scheme configuration. This applies to all page and profiles, with the exception of some custom themes. Works in the settings panel too!
Thanks for reading! And seriously back up your database.
",
+ expectedPlain: `Hello everyone, after a week or two down the release candidate mines, we've emerged blinking into the light carrying with us #GoToSocial v0.18.0 Scroingly Sloth!
+
+https://github.com/superseriousbusiness/gotosocial/releases/tag/v0.18.0
+
+Please read the migration notes carefully for instructions on how to upgrade to this version. This version contains several very long migrations so you will need to be patient when upgrading, and backup your database first!!
+
+Release highlights
+
+
+ - Status edit support: one of our most-requested features! You can now edit your own statuses, and see instance edit history from other accounts too (if your instance has them stored).
+ - Push notifications: probably the second most-requested feature! GoToSocial can now send push notifications to clients via their configured push providers.
+You may need to uninstall / reinstall client applications, or log out and back in again, for this feature to work. (And if you're using Tusky, make sure you've got ntfy installed ).
+ - Global instance css customization: admins can now apply custom CSS across their entire instance via the settings panel.
+ - Domain permission subscriptions: it's now possible to configure your instance to subscribe to CSV, JSON, or plaintext lists of domain permissions.
+Each night, your instance will fetch and automatically create domain permissions (or permission drafts) based on what it finds in a subscribed list.
+See the domain permission subscription documentation for more information.
+ - Trusted-proxies helper: instances with improperly configured trusted-proxies settings will now show a warning on the homepage, so admins can make sure their instance is configured correctly. Check your own instance homepage after updating to see if you need to do anything.
+ - Better outbox sorting: messages from GoToSocial are now delivered more quickly to people you mention, so conversations across instances should feel a bit snappier.
+ - Log in button: there's now a login button in the top right of the instance homepage, which leads to a helpful page about clients, with a link to the settings panel. Should make things less confusing for new users!
+ - Granular stats controls: with the instance-stats-mode setting, admins can now choose if and how their instance serves stats via the nodeinfo endpoints. Existing behavior from v0.17.0 is the default.
+ - Post backdating: via the API you can now backdate posts (if enabled in config.yaml). This is our first step towards making it possible to import your post history from elsewhere into your GoToSocial instance. While there's no way to do this in the settings panel yet, you can already use third-party tools like Slurp to import posts from a Mastodon export (see Slurp ).
+ - Configurable sign-up limits: you can now configure your sign-up backlog length and sign-up throttling (defaults remain the same).
+ - NetBSD and FreeBSD builds: yep!
+ - Respect users prefers-color-scheme preference: there's now a light mode default theme to complement our trusty dark mode theme, and the theme will switch based on a visitor's prefers-color-scheme configuration. This applies to all page and profiles, with the exception of some custom themes. Works in the settings panel too!
+
+
+Thanks for reading! And seriously back up your database.`,
+ },
+ } {
+ plain := text.ParseHTMLToPlain(t.html)
+ suite.Equal(t.expectedPlain, plain)
+ }
+}
+
+func (suite *PlainTestSuite) TestStripCaption1() {
+ dodgyCaption := "this is just a normal caption ;)"
+ stripped := text.StripHTMLFromText(dodgyCaption)
+ suite.Equal("this is just a normal caption ;)", stripped)
+}
+
+func (suite *PlainTestSuite) TestStripCaption2() {
+ dodgyCaption := "here's a LOUD caption"
+ stripped := text.StripHTMLFromText(dodgyCaption)
+ suite.Equal("here's a LOUD caption", stripped)
+}
+
+func (suite *PlainTestSuite) TestStripCaption3() {
+ dodgyCaption := ""
+ stripped := text.StripHTMLFromText(dodgyCaption)
+ suite.Equal("", stripped)
+}
+
+func (suite *PlainTestSuite) TestStripCaption4() {
+ dodgyCaption := `
+
+
+here is
+a multi line
+caption
+with some newlines
+
+
+
+`
+ stripped := text.StripHTMLFromText(dodgyCaption)
+ suite.Equal("here is\na multi line\ncaption\nwith some newlines", stripped)
+}
+
+func (suite *PlainTestSuite) TestStripCaption5() {
+ // html-escaped: " hello world"
+ dodgyCaption := `<script>console.log('aha!')</script> hello world`
+ stripped := text.StripHTMLFromText(dodgyCaption)
+ suite.Equal("hello world", stripped)
+}
+
+func (suite *PlainTestSuite) TestStripCaption6() {
+ // html-encoded: " hello world"
+ dodgyCaption := `<script>console.log('aha!')</script> hello world`
+ stripped := text.StripHTMLFromText(dodgyCaption)
+ suite.Equal("hello world", stripped)
+}
+
+func (suite *PlainTestSuite) TestStripCustomCSS() {
+ customCSS := `.toot .username {
+ color: var(--link_fg);
+ line-height: 2rem;
+ margin-top: -0.5rem;
+ align-self: start;
+
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+}`
+ stripped := text.StripHTMLFromText(customCSS)
+ suite.Equal(customCSS, stripped) // should be the same as it was before
+}
+
+func (suite *PlainTestSuite) TestStripNaughtyCustomCSS1() {
+ // try to break out of pee pee poo poo pee pee poo poo