[feature] Parse content warning to HTML, serialize via client API as plaintext (#3876)

* [feature] Parse content warning as HTML, serialize via API to plaintext

* tidy up some cruft

* whoops

* oops

* i'm da joker baybee

* clemency muy lorde

* rename some of the text functions for clarity

* jiggle the opts

* fiddle de deee

* hopefully the last test fix i ever have to do in my beautiful life

internal/processing/status/common.go

@@ -171,19 +171,25 @@ func (p *Processor) processContent(
 		)
 	}
 
-	// format is the currently set text formatting
-	// function, according to the provided content-type.
-	var format text.FormatFunc
+	var (
+		// format is the currently set text formatting
+		// function, according to the provided content-type.
+		format text.FormatFunc
+		// formatCW is like format, but for content warning.
+		formatCW text.FormatFunc
+	)
 
 	switch contentType {
 
 	// Format status according to text/plain.
 	case gtsmodel.StatusContentTypePlain:
 		format = p.formatter.FromPlain
+		formatCW = p.formatter.FromPlainBasic
 
 	// Format status according to text/markdown.
 	case gtsmodel.StatusContentTypeMarkdown:
 		format = p.formatter.FromMarkdown
+		formatCW = p.formatter.FromMarkdownBasic
 
 	// Unknown.
 	default:
@@ -215,26 +221,23 @@ func (p *Processor) processContent(
 	status.Emojis = contentRes.Emojis
 	status.Tags = contentRes.Tags
 
-	// From here-on-out just use emoji-only
-	// plain-text formatting as the FormatFunc.
-	format = p.formatter.FromPlainEmojiOnly
-
 	// Sanitize content warning and format.
-	warning := text.SanitizeToPlaintext(contentWarning)
-	warningRes := formatInput(format, warning)
+	cwRes := formatInput(formatCW, contentWarning)
 
 	// Gather results of the formatted.
-	status.ContentWarning = warningRes.HTML
-	status.Emojis = append(status.Emojis, warningRes.Emojis...)
+	status.ContentWarning = cwRes.HTML
+	status.Emojis = append(status.Emojis, cwRes.Emojis...)
 
 	if poll != nil {
 		// Pre-allocate slice of poll options of expected length.
 		status.PollOptions = make([]string, len(poll.Options))
 		for i, option := range poll.Options {
 
-			// Sanitize each poll option and format.
-			option = text.SanitizeToPlaintext(option)
-			optionRes := formatInput(format, option)
+			// Strip each poll option and format.
+			//
+			// For polls just use basic formatting.
+			option = text.StripHTMLFromText(option)
+			optionRes := formatInput(p.formatter.FromPlainBasic, option)
 
 			// Gather results of the formatted.
 			status.PollOptions[i] = optionRes.HTML

internal/processing/status/create.go

@@ -189,6 +189,13 @@ func (p *Processor) Create(
 		PendingApproval: util.Ptr(false),
 	}
 
+	// Only store ContentWarningText if the parsed
+	// result is different from the given SpoilerText,
+	// otherwise skip to avoid duplicating db columns.
+	if content.ContentWarning != form.SpoilerText {
+		status.ContentWarningText = form.SpoilerText
+	}
+
 	if backfill {
 		// Ensure backfilled status contains no
 		// mentions to anyone other than author.

internal/processing/status/create_test.go

@@ -60,33 +60,6 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithQuotationMarks(
 	suite.Equal("\"test\"", apiStatus.SpoilerText)
 }
 
-func (suite *StatusCreateTestSuite) TestProcessContentWarningWithHTMLEscapedQuotationMarks() {
-	ctx := context.Background()
-
-	creatingAccount := suite.testAccounts["local_account_1"]
-	creatingApplication := suite.testApplications["application_1"]
-
-	statusCreateForm := &apimodel.StatusCreateRequest{
-		Status:      "poopoo peepee",
-		MediaIDs:    []string{},
-		Poll:        nil,
-		InReplyToID: "",
-		Sensitive:   false,
-		SpoilerText: "&#34test&#34", // the html-escaped quotation marks should appear as normal quotation marks in the finished text
-		Visibility:  apimodel.VisibilityPublic,
-		LocalOnly:   util.Ptr(false),
-		ScheduledAt: nil,
-		Language:    "en",
-		ContentType: apimodel.StatusContentTypePlain,
-	}
-
-	apiStatus, err := suite.status.Create(ctx, creatingAccount, creatingApplication, statusCreateForm)
-	suite.NoError(err)
-	suite.NotNil(apiStatus)
-
-	suite.Equal("\"test\"", apiStatus.SpoilerText)
-}
-
 func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithUnderscoreEmoji() {
 	ctx := context.Background()
 

internal/processing/status/delete.go

@@ -50,6 +50,13 @@ func (p *Processor) Delete(ctx context.Context, requestingAccount *gtsmodel.Acco
 		return nil, errWithCode
 	}
 
+	// Replace content warning with raw
+	// version if it's available, to make
+	// delete + redraft work nicer.
+	if targetStatus.ContentWarningText != "" {
+		apiStatus.SpoilerText = targetStatus.ContentWarningText
+	}
+
 	// Process delete side effects.
 	p.state.Workers.Client.Queue.Push(&messages.FromClientAPI{
 		APObjectType:   ap.ObjectNote,

internal/processing/status/edit.go

@@ -301,7 +301,7 @@ func (p *Processor) Edit(
 	// update the other necessary status fields.
 	status.Content = content.Content
 	status.ContentWarning = content.ContentWarning
-	status.Text = form.Status
+	status.Text = form.Status // raw
 	status.ContentType = contentType
 	status.Language = content.Language
 	status.Sensitive = &form.Sensitive
@@ -309,6 +309,13 @@ func (p *Processor) Edit(
 	status.Attachments = media
 	status.EditedAt = now
 
+	// Only store ContentWarningText if the parsed
+	// result is different from the given SpoilerText,
+	// otherwise skip to avoid duplicating db columns.
+	if content.ContentWarning != form.SpoilerText {
+		status.ContentWarningText = form.SpoilerText
+	}
+
 	if poll != nil {
 		// Set relevent fields for latest with poll.
 		status.ActivityStreamsType = ap.ActivityQuestion

internal/processing/status/get.go

@@ -53,10 +53,21 @@ func (p *Processor) SourceGet(ctx context.Context, requester *gtsmodel.Account,
 			"target status not found",
 		)
 	}
+
+	// Try to use unparsed content
+	// warning text if available,
+	// fall back to parsed cw html.
+	var spoilerText string
+	if status.ContentWarningText != "" {
+		spoilerText = status.ContentWarningText
+	} else {
+		spoilerText = status.ContentWarning
+	}
+
 	return &apimodel.StatusSource{
 		ID:          status.ID,
 		Text:        status.Text,
-		SpoilerText: status.ContentWarning,
+		SpoilerText: spoilerText,
 		ContentType: typeutils.ContentTypeToAPIContentType(status.ContentType),
 	}, nil
 }