address review comments

.drone.yml

@@ -45,7 +45,7 @@ steps:
         go test
         -failfast
         -timeout=20m
-        -tags "netgo osusergo static_build kvformat timetzdata purego"
+        -tags "netgo osusergo static_build kvformat timetzdata"
         ./...
       - ./test/envparsing.sh
       - ./test/swagger.sh
@@ -207,6 +207,6 @@ steps:
 
 ---
 kind: signature
-hmac: 3f3a24557b67760dd0c4091eaaed4842b0545f5aa65f90ce70d5e45da23c5260
+hmac: f4008d87e4e5b67251eb89f255c1224e6ab5818828cab24fc319b8f829176058
 
 ...

.goreleaser.yml

@@ -27,7 +27,6 @@ builds:
       - static_build
       - kvformat
       - timetzdata
-      - purego
       - >-
         {{ if and (index .Env "DEBUG") (.Env.DEBUG) }}debugenv{{ end }}
       - >-

go.mod

@@ -28,7 +28,6 @@ require (
 	github.com/DmitriyVTitov/size v1.5.0
 	github.com/KimMachineGun/automemlimit v0.6.1
 	github.com/buckket/go-blurhash v1.1.0
-	github.com/cespare/xxhash v1.1.0
 	github.com/coreos/go-oidc/v3 v3.11.0
 	github.com/gin-contrib/cors v1.7.2
 	github.com/gin-contrib/gzip v1.0.1

go.sum

@@ -98,8 +98,6 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
 github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
@@ -120,8 +118,6 @@ github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -512,8 +508,6 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
 github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=

internal/typeutils/converter.go

@@ -29,26 +29,30 @@ import (
 )
 
 type Converter struct {
-	state                        *state.State
+	state          *state.State
-	defaultAvatars               []string
+	defaultAvatars []string
-	randAvatars                  sync.Map
+	randAvatars    sync.Map
-	visFilter                    *visibility.Filter
+	visFilter      *visibility.Filter
-	intFilter                    *interaction.Filter
+	intFilter      *interaction.Filter
-	statusHashesToFilterableText cache.TTLCache[string, string]
+
+	// TTL cache of statuses -> filterable text fields.
+	// To ensure up-to-date fields, cache is keyed as:
+	// [status.ID][status.UpdatedAt.Unix()]`
+	statusesFilterableFields cache.TTLCache[string, []string]
 }
 
 func NewConverter(state *state.State) *Converter {
-	statusHashesToFilterableText := cache.NewTTL[string, string](0, 512, 0)
+	statusHashesToFilterableText := cache.NewTTL[string, []string](0, 512, 0)
 	statusHashesToFilterableText.SetTTL(time.Hour, true)
 	if !statusHashesToFilterableText.Start(time.Minute) {
 		log.Panic(nil, "failed to start statusHashesToFilterableText cache")
 	}
 
 	return &Converter{
-		state:                        state,
+		state:                    state,
-		defaultAvatars:               populateDefaultAvatars(),
+		defaultAvatars:           populateDefaultAvatars(),
-		visFilter:                    visibility.NewFilter(state),
+		visFilter:                visibility.NewFilter(state),
-		intFilter:                    interaction.NewFilter(state),
+		intFilter:                interaction.NewFilter(state),
-		statusHashesToFilterableText: statusHashesToFilterableText,
+		statusesFilterableFields: statusHashesToFilterableText,
 	}
 }

internal/typeutils/internaltofrontend.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strings"
 	"time"
 
@@ -938,35 +939,47 @@ func (c *Converter) statusToAPIFilterResults(
 		return nil, nil
 	}
 
-	// Derive a hash of this status.
+	// Key this status based on ID + last updated time,
-	statusHash := StatusHash(s)
+	// to ensure we always filter on latest version.
+	statusKey := fmt.Sprintf("%s%d", s.ID, s.UpdatedAt.Unix())
 
-	// Check if we have the filterable
+	// Check if we have filterable fields cached for this status.
-	// text stored already for this hash.
+	fields, stored := c.statusesFilterableFields.Get(statusKey)
-	statusText, stored := c.statusHashesToFilterableText.Get(statusHash)
 	if !stored {
-		// We don't have this filterable text
+		// We don't have filterable fields
-		// cached, calculate + cache it now.
+		// cached, calculate + cache now.
-		statusText = filterableText(s)
+		fields = filterableFields(s)
-		c.statusHashesToFilterableText.Set(statusHash, statusText)
+		c.statusesFilterableFields.Set(statusKey, fields)
 	}
 
 	// Record all matching warn filters and the reasons they matched.
 	filterResults := make([]apimodel.FilterResult, 0, len(filters))
 	for _, filter := range filters {
 		if !filterAppliesInContext(filter, filterContext) {
-			// Filter doesn't apply to this context.
+			// Filter doesn't apply
-			continue
+			// to this context.
-		}
-		if filter.Expired(now) {
 			continue
 		}
 
-		// List all matching keywords.
+		if filter.Expired(now) {
+			// Filter doesn't
+			// apply anymore.
+			continue
+		}
+
+		// Assemble matching keywords (if any) from this filter.
 		keywordMatches := make([]string, 0, len(filter.Keywords))
-		for _, filterKeyword := range filter.Keywords {
+		for _, keyword := range filter.Keywords {
-			if filterKeyword.Regexp.MatchString(statusText) {
+			// Check if at least one filterable field
-				keywordMatches = append(keywordMatches, filterKeyword.Keyword)
+			// in the status matches on this filter.
+			if slices.ContainsFunc(
+				fields,
+				func(field string) bool {
+					return keyword.Regexp.MatchString(field)
+				},
+			) {
+				// At least one field matched on this filter.
+				keywordMatches = append(keywordMatches, keyword.Keyword)
 			}
 		}
 

internal/typeutils/internaltofrontend_test.go

@@ -1066,7 +1066,6 @@ func (suite *InternalToFrontendTestSuite) testHashtagFilteredStatusToFrontend(wh
 	testStatus := new(gtsmodel.Status)
 	*testStatus = *suite.testStatuses["admin_account_status_1"]
 	testStatus.Content = `<p>doggo doggin' it</p><p><a href="https://example.test/tags/dogsofmastodon" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>dogsofmastodon</span></a></p>`
-	testStatus.Text = "doggo doggin' it\n\n#dogsofmastodon"
 
 	if boost {
 		boost, err := suite.typeconverter.StatusToBoost(

internal/typeutils/util.go

@@ -19,7 +19,6 @@ package typeutils
 
 import (
 	"context"
-	"encoding/hex"
 	"fmt"
 	"math"
 	"net/url"
@@ -28,7 +27,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/cespare/xxhash/v2"
 	"github.com/k3a/html2text"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
@@ -288,74 +286,63 @@ func ContentToContentLanguage(
 	return contentStr, langTagStr
 }
 
-// StatusHash returns an xxhash of text
+// filterableFields returns text fields from
-// from a status, taking account of:
+// a status that we might want to filter on:
-//
-//   - content warning
-//   - content
-//   - media IDs + descriptions
-//   - poll options
-func StatusHash(s *gtsmodel.Status) string {
-	hash := xxhash.New()
-
-	// Content warning / title.
-	hash.WriteString(s.ContentWarning) // nolint:errcheck
-
-	// Status content.
-	hash.WriteString(s.Content) // nolint:errcheck
-
-	// Media IDs + descriptions.
-	for _, attachment := range s.Attachments {
-		hash.WriteString(attachment.ID)          // nolint:errcheck
-		hash.WriteString(attachment.Description) // nolint:errcheck
-	}
-
-	// Poll options.
-	if s.Poll != nil {
-		for _, option := range s.Poll.Options {
-			hash.WriteString(option) // nolint:errcheck
-		}
-	}
-
-	sum := hash.Sum(nil)
-	return hex.EncodeToString(sum)
-}
-
-// filterableText concatenates text from a
-// status that we might want to filter on:
 //
 //   - content warning
 //   - content (converted to plaintext from HTML)
 //   - media descriptions
 //   - poll options
-func filterableText(s *gtsmodel.Status) string {
+//
-	fields := []string{}
+// Each field should be filtered separately.
+// This avoids scenarios where false-positive
+// multiple-word matches can be made by matching
+// the last word of one field + the first word
+// of the next field together.
+func filterableFields(s *gtsmodel.Status) []string {
+	// Estimate length of fields.
+	fieldCount := 2 + len(s.Attachments)
+	if s.Poll != nil {
+		fieldCount += len(s.Poll.Options)
+	}
+	fields := make([]string, 0, fieldCount)
 
 	// Content warning / title.
-	fields = append(fields, s.ContentWarning)
+	if s.ContentWarning != "" {
+		fields = append(fields, s.ContentWarning)
+	}
 
-	// Status content; use raw text if available,
+	// Status content. Though we have raw text
-	// else use text parsed from content HTML.
+	// available for statuses created on our
-	if s.Text != "" {
+	// instance, use the html2text version to
-		fields = append(fields, s.Text)
+	// remove markdown-formatting characters
-	} else {
+	// and ensure more consistent filtering.
+	if s.Content != "" {
 		text := html2text.HTML2TextWithOptions(
 			s.Content,
 			html2text.WithLinksInnerText(),
 			html2text.WithUnixLineBreaks(),
 		)
-		fields = append(fields, text)
+		if text != "" {
+			fields = append(fields, text)
+		}
 	}
 
 	// Media descriptions.
 	for _, attachment := range s.Attachments {
-		fields = append(fields, attachment.Description)
+		if attachment.Description != "" {
+			fields = append(fields, attachment.Description)
+		}
 	}
 
 	// Poll options.
 	if s.Poll != nil {
-		fields = append(fields, s.Poll.Options...)
+		for _, opt := range s.Poll.Options {
+			if opt != "" {
+				fields = append(fields, opt)
+			}
+		}
 	}
 
-	return strings.Join(fields, " ")
+	return fields
 }

internal/typeutils/util_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/language"
@@ -161,87 +162,59 @@ func TestContentToContentLanguage(t *testing.T) {
 
 func TestFilterableText(t *testing.T) {
 	type testcase struct {
-		status       *gtsmodel.Status
+		status         *gtsmodel.Status
-		expectedText string
+		expectedFields []string
 	}
 
-	for i, testcase := range []testcase{
+	for _, testcase := range []testcase{
 		{
 			status: &gtsmodel.Status{
 				ContentWarning: "This is a test status",
 				Content:        `<p>Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> instance.</p>`,
 			},
-			expectedText: `This is a test status Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> instance.`,
+			expectedFields: []string{
+				"This is a test status",
+				"Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> instance.",
+			},
 		},
 		{
 			status: &gtsmodel.Status{
 				Content: `<p><span class="h-card"><a href="https://example.org/@zlatko" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>zlatko</span></a></span> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)</p><p><a href="https://github.com/superseriousbusiness/gotosocial/pull/2863" rel="nofollow noreferrer noopener" target="_blank">https://github.com/superseriousbusiness/gotosocial/pull/2863</a></p>`,
 			},
-			expectedText: ` @zlatko <https://example.org/@zlatko> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)
+			expectedFields: []string{
-
+				"@zlatko <https://example.org/@zlatko> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)\n\nhttps://github.com/superseriousbusiness/gotosocial/pull/2863 <https://github.com/superseriousbusiness/gotosocial/pull/2863>",
-https://github.com/superseriousbusiness/gotosocial/pull/2863 <https://github.com/superseriousbusiness/gotosocial/pull/2863>`,
+			},
 		},
 		{
 			status: &gtsmodel.Status{
-				Content: `<p>Latest graphs for <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> on <a href="https://github.com/ncruces/go-sqlite3" rel="nofollow noreferrer noopener" target="_blank">Wasm sqlite3</a> with <a href="https://codeberg.org/gruf/go-ffmpreg" rel="nofollow noreferrer noopener" target="_blank">embedded Wasm ffmpeg</a>, both running on <a href="https://wazero.io/" rel="nofollow noreferrer noopener" target="_blank">Wazero</a>, and configured with a <a href="https://github.com/superseriousbusiness/gotosocial/blob/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266" rel="nofollow noreferrer noopener" target="_blank">50MiB db cache target</a>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.</p>`,
+				ContentWarning: "Nerd stuff",
+				Content:        `<p>Latest graphs for <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> on <a href="https://github.com/ncruces/go-sqlite3" rel="nofollow noreferrer noopener" target="_blank">Wasm sqlite3</a> with <a href="https://codeberg.org/gruf/go-ffmpreg" rel="nofollow noreferrer noopener" target="_blank">embedded Wasm ffmpeg</a>, both running on <a href="https://wazero.io/" rel="nofollow noreferrer noopener" target="_blank">Wazero</a>, and configured with a <a href="https://github.com/superseriousbusiness/gotosocial/blob/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266" rel="nofollow noreferrer noopener" target="_blank">50MiB db cache target</a>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.</p>`,
 				Attachments: []*gtsmodel.MediaAttachment{
 					{
 						Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`,
 					},
-				},
-			},
-			expectedText: ` Latest graphs for #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> on Wasm sqlite3 <https://github.com/ncruces/go-sqlite3> with embedded Wasm ffmpeg <https://codeberg.org/gruf/go-ffmpreg>, both running on Wazero <https://wazero.io/>, and configured with a 50MiB db cache target <https://github.com/superseriousbusiness/gotosocial/blob/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything. Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`,
-		},
-	} {
-		text := filterableText(testcase.status)
-		if text != testcase.expectedText {
-			t.Errorf(
-				"test %d expected text '%s' got '%s'",
-				i, testcase.expectedText, text,
-			)
-		}
-	}
-}
-
-func TestStatusHash(t *testing.T) {
-	type testcase struct {
-		status       *gtsmodel.Status
-		expectedHash string
-	}
-
-	for i, testcase := range []testcase{
-		{
-			status: &gtsmodel.Status{
-				ContentWarning: "This is a test status",
-				Content:        `<p>Import / export of account data via CSV files will be coming in 0.17.0 :) No more having to run scripts + CLI tools to import a list of accounts you follow, after doing a migration to a <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> instance.</p>`,
-			},
-			expectedHash: `8bbb5439dbe62ae0`,
-		},
-		{
-			status: &gtsmodel.Status{
-				Content: `<p><span class="h-card"><a href="https://example.org/@zlatko" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>zlatko</span></a></span> currently we used modernc/sqlite3 for our sqlite driver, but we've been experimenting with wasm sqlite, and will likely move to that permanently in future; in the meantime, both options are available (the latter with a build tag)</p><p><a href="https://github.com/superseriousbusiness/gotosocial/pull/2863" rel="nofollow noreferrer noopener" target="_blank">https://github.com/superseriousbusiness/gotosocial/pull/2863</a></p>`,
-			},
-			expectedHash: `d039dfb4d04752d5`,
-		},
-		{
-			status: &gtsmodel.Status{
-				Content: `<p>Latest graphs for <a href="https://gts.superseriousbusiness.org/tags/gotosocial" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>GoToSocial</span></a> on <a href="https://github.com/ncruces/go-sqlite3" rel="nofollow noreferrer noopener" target="_blank">Wasm sqlite3</a> with <a href="https://codeberg.org/gruf/go-ffmpreg" rel="nofollow noreferrer noopener" target="_blank">embedded Wasm ffmpeg</a>, both running on <a href="https://wazero.io/" rel="nofollow noreferrer noopener" target="_blank">Wazero</a>, and configured with a <a href="https://github.com/superseriousbusiness/gotosocial/blob/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266" rel="nofollow noreferrer noopener" target="_blank">50MiB db cache target</a>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.</p>`,
-				Attachments: []*gtsmodel.MediaAttachment{
 					{
-						ID:          "01J7TYSH1V5V4DCTVPASH3K9PQ",
+						Description: `Another media attachment`,
-						Description: `Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.`,
+					},
+				},
+				Poll: &gtsmodel.Poll{
+					Options: []string{
+						"Poll option 1",
+						"Poll option 2",
 					},
 				},
 			},
-			expectedHash: `414d975b2ef9d112`,
+			expectedFields: []string{
+				"Nerd stuff",
+				"Latest graphs for #GoToSocial <https://gts.superseriousbusiness.org/tags/gotosocial> on Wasm sqlite3 <https://github.com/ncruces/go-sqlite3> with embedded Wasm ffmpeg <https://codeberg.org/gruf/go-ffmpreg>, both running on Wazero <https://wazero.io/>, and configured with a 50MiB db cache target <https://github.com/superseriousbusiness/gotosocial/blob/20fe430ef9ff3012a7a4dc2d01b68020c20e13bb/example/config.yaml#L259-L266>. This is the version we'll be releasing soonish, now we're happy with how we've tamed everything.",
+				"Graph showing GtS using between 150-300 MiB of memory, steadily, over a few days.",
+				"Another media attachment",
+				"Poll option 1",
+				"Poll option 2",
+			},
 		},
 	} {
-		hash := StatusHash(testcase.status)
+		fields := filterableFields(testcase.status)
-		if hash != testcase.expectedHash {
+		assert.Equal(t, testcase.expectedFields, fields)
-			t.Errorf(
-				"test %d expected hash '%s' got '%s'",
-				i, testcase.expectedHash, hash,
-			)
-		}
 	}
 }

scripts/build.sh

@@ -6,7 +6,7 @@ set -e
 log_exec() { echo "$ ${*}"; "$@"; }
 
 # Grab environment variables and set defaults + requirements.
-GO_BUILDTAGS="${GO_BUILDTAGS-} netgo osusergo static_build kvformat timetzdata purego"
+GO_BUILDTAGS="${GO_BUILDTAGS-} netgo osusergo static_build kvformat timetzdata"
 GO_LDFLAGS="${GO_LDFLAGS-} -s -w -extldflags '-static' -X 'main.Version=${VERSION:-$(git describe --tags --abbrev=0)}'"
 GO_GCFLAGS=${GO_GCFLAGS-}
 
@@ -17,7 +17,6 @@ GO_GCFLAGS=${GO_GCFLAGS-}
 # Available Go build tags, with explanation, followed by benefits of enabling it:
 # - kvformat:       enables prettier output of log fields                       (slightly better performance)
 # - timetzdata:     embed timezone database inside binary                       (allow setting local time inside Docker containers, at cost of 450KB)
-# - purego:         disable amd64/arm64 assembly implementation for xxhash      (increase portability at marginal performance cost)
 # - notracing:      disables compiling-in otel tracing support                  (reduced binary size, better performance)
 # - nometrics:      disables compiling-in otel metrics support                  (reduced binary size, better performance)
 # - noerrcaller:    disables caller function prefix in errors                   (slightly better performance, at cost of err readability)

vendor/github.com/cespare/xxhash/LICENSE.txt

@@ -1,22 +0,0 @@
-Copyright (c) 2016 Caleb Spare
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/cespare/xxhash/README.md

@@ -1,50 +0,0 @@
-# xxhash
-
-[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash)
-
-xxhash is a Go implementation of the 64-bit
-[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
-high-quality hashing algorithm that is much faster than anything in the Go
-standard library.
-
-The API is very small, taking its cue from the other hashing packages in the
-standard library:
-
-    $ go doc github.com/cespare/xxhash                                                                                                                                                                                              !
-    package xxhash // import "github.com/cespare/xxhash"
-
-    Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
-    at http://cyan4973.github.io/xxHash/.
-
-    func New() hash.Hash64
-    func Sum64(b []byte) uint64
-    func Sum64String(s string) uint64
-
-This implementation provides a fast pure-Go implementation and an even faster
-assembly implementation for amd64.
-
-## Benchmarks
-
-Here are some quick benchmarks comparing the pure-Go and assembly
-implementations of Sum64 against another popular Go XXH64 implementation,
-[github.com/OneOfOne/xxhash](https://github.com/OneOfOne/xxhash):
-
-| input size | OneOfOne | cespare (purego) | cespare |
-| --- | --- | --- | --- |
-| 5 B   |  416 MB/s | 720 MB/s |  872 MB/s  |
-| 100 B | 3980 MB/s | 5013 MB/s | 5252 MB/s  |
-| 4 KB  | 12727 MB/s | 12999 MB/s | 13026 MB/s |
-| 10 MB | 9879 MB/s | 10775 MB/s | 10913 MB/s  |
-
-These numbers were generated with:
-
-```
-$ go test -benchtime 10s -bench '/OneOfOne,'
-$ go test -tags purego -benchtime 10s -bench '/xxhash,'
-$ go test -benchtime 10s -bench '/xxhash,'
-```
-
-## Projects using this package
-
-- [InfluxDB](https://github.com/influxdata/influxdb)
-- [Prometheus](https://github.com/prometheus/prometheus)

vendor/github.com/cespare/xxhash/rotate.go

@@ -1,14 +0,0 @@
-// +build !go1.9
-
-package xxhash
-
-// TODO(caleb): After Go 1.10 comes out, remove this fallback code.
-
-func rol1(x uint64) uint64  { return (x << 1) | (x >> (64 - 1)) }
-func rol7(x uint64) uint64  { return (x << 7) | (x >> (64 - 7)) }
-func rol11(x uint64) uint64 { return (x << 11) | (x >> (64 - 11)) }
-func rol12(x uint64) uint64 { return (x << 12) | (x >> (64 - 12)) }
-func rol18(x uint64) uint64 { return (x << 18) | (x >> (64 - 18)) }
-func rol23(x uint64) uint64 { return (x << 23) | (x >> (64 - 23)) }
-func rol27(x uint64) uint64 { return (x << 27) | (x >> (64 - 27)) }
-func rol31(x uint64) uint64 { return (x << 31) | (x >> (64 - 31)) }

vendor/github.com/cespare/xxhash/rotate19.go

@@ -1,14 +0,0 @@
-// +build go1.9
-
-package xxhash
-
-import "math/bits"
-
-func rol1(x uint64) uint64  { return bits.RotateLeft64(x, 1) }
-func rol7(x uint64) uint64  { return bits.RotateLeft64(x, 7) }
-func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) }
-func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) }
-func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) }
-func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) }
-func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) }
-func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) }

vendor/github.com/cespare/xxhash/xxhash.go

@@ -1,168 +0,0 @@
-// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
-// at http://cyan4973.github.io/xxHash/.
-package xxhash
-
-import (
-	"encoding/binary"
-	"hash"
-)
-
-const (
-	prime1 uint64 = 11400714785074694791
-	prime2 uint64 = 14029467366897019727
-	prime3 uint64 = 1609587929392839161
-	prime4 uint64 = 9650029242287828579
-	prime5 uint64 = 2870177450012600261
-)
-
-// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
-// possible in the Go code is worth a small (but measurable) performance boost
-// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
-// convenience in the Go code in a few places where we need to intentionally
-// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
-// result overflows a uint64).
-var (
-	prime1v = prime1
-	prime2v = prime2
-	prime3v = prime3
-	prime4v = prime4
-	prime5v = prime5
-)
-
-type xxh struct {
-	v1    uint64
-	v2    uint64
-	v3    uint64
-	v4    uint64
-	total int
-	mem   [32]byte
-	n     int // how much of mem is used
-}
-
-// New creates a new hash.Hash64 that implements the 64-bit xxHash algorithm.
-func New() hash.Hash64 {
-	var x xxh
-	x.Reset()
-	return &x
-}
-
-func (x *xxh) Reset() {
-	x.n = 0
-	x.total = 0
-	x.v1 = prime1v + prime2
-	x.v2 = prime2
-	x.v3 = 0
-	x.v4 = -prime1v
-}
-
-func (x *xxh) Size() int      { return 8 }
-func (x *xxh) BlockSize() int { return 32 }
-
-// Write adds more data to x. It always returns len(b), nil.
-func (x *xxh) Write(b []byte) (n int, err error) {
-	n = len(b)
-	x.total += len(b)
-
-	if x.n+len(b) < 32 {
-		// This new data doesn't even fill the current block.
-		copy(x.mem[x.n:], b)
-		x.n += len(b)
-		return
-	}
-
-	if x.n > 0 {
-		// Finish off the partial block.
-		copy(x.mem[x.n:], b)
-		x.v1 = round(x.v1, u64(x.mem[0:8]))
-		x.v2 = round(x.v2, u64(x.mem[8:16]))
-		x.v3 = round(x.v3, u64(x.mem[16:24]))
-		x.v4 = round(x.v4, u64(x.mem[24:32]))
-		b = b[32-x.n:]
-		x.n = 0
-	}
-
-	if len(b) >= 32 {
-		// One or more full blocks left.
-		b = writeBlocks(x, b)
-	}
-
-	// Store any remaining partial block.
-	copy(x.mem[:], b)
-	x.n = len(b)
-
-	return
-}
-
-func (x *xxh) Sum(b []byte) []byte {
-	s := x.Sum64()
-	return append(
-		b,
-		byte(s>>56),
-		byte(s>>48),
-		byte(s>>40),
-		byte(s>>32),
-		byte(s>>24),
-		byte(s>>16),
-		byte(s>>8),
-		byte(s),
-	)
-}
-
-func (x *xxh) Sum64() uint64 {
-	var h uint64
-
-	if x.total >= 32 {
-		v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4
-		h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
-		h = mergeRound(h, v1)
-		h = mergeRound(h, v2)
-		h = mergeRound(h, v3)
-		h = mergeRound(h, v4)
-	} else {
-		h = x.v3 + prime5
-	}
-
-	h += uint64(x.total)
-
-	i, end := 0, x.n
-	for ; i+8 <= end; i += 8 {
-		k1 := round(0, u64(x.mem[i:i+8]))
-		h ^= k1
-		h = rol27(h)*prime1 + prime4
-	}
-	if i+4 <= end {
-		h ^= uint64(u32(x.mem[i:i+4])) * prime1
-		h = rol23(h)*prime2 + prime3
-		i += 4
-	}
-	for i < end {
-		h ^= uint64(x.mem[i]) * prime5
-		h = rol11(h) * prime1
-		i++
-	}
-
-	h ^= h >> 33
-	h *= prime2
-	h ^= h >> 29
-	h *= prime3
-	h ^= h >> 32
-
-	return h
-}
-
-func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }
-func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
-
-func round(acc, input uint64) uint64 {
-	acc += input * prime2
-	acc = rol31(acc)
-	acc *= prime1
-	return acc
-}
-
-func mergeRound(acc, val uint64) uint64 {
-	val = round(0, val)
-	acc ^= val
-	acc = acc*prime1 + prime4
-	return acc
-}

vendor/github.com/cespare/xxhash/xxhash_amd64.go

@@ -1,12 +0,0 @@
-// +build !appengine
-// +build gc
-// +build !purego
-
-package xxhash
-
-// Sum64 computes the 64-bit xxHash digest of b.
-//
-//go:noescape
-func Sum64(b []byte) uint64
-
-func writeBlocks(x *xxh, b []byte) []byte

vendor/github.com/cespare/xxhash/xxhash_amd64.s

@@ -1,233 +0,0 @@
-// +build !appengine
-// +build gc
-// +build !purego
-
-#include "textflag.h"
-
-// Register allocation:
-// AX	h
-// CX	pointer to advance through b
-// DX	n
-// BX	loop end
-// R8	v1, k1
-// R9	v2
-// R10	v3
-// R11	v4
-// R12	tmp
-// R13	prime1v
-// R14	prime2v
-// R15	prime4v
-
-// round reads from and advances the buffer pointer in CX.
-// It assumes that R13 has prime1v and R14 has prime2v.
-#define round(r) \
-	MOVQ  (CX), R12 \
-	ADDQ  $8, CX    \
-	IMULQ R14, R12  \
-	ADDQ  R12, r    \
-	ROLQ  $31, r    \
-	IMULQ R13, r
-
-// mergeRound applies a merge round on the two registers acc and val.
-// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v.
-#define mergeRound(acc, val) \
-	IMULQ R14, val \
-	ROLQ  $31, val \
-	IMULQ R13, val \
-	XORQ  val, acc \
-	IMULQ R13, acc \
-	ADDQ  R15, acc
-
-// func Sum64(b []byte) uint64
-TEXT ·Sum64(SB), NOSPLIT, $0-32
-	// Load fixed primes.
-	MOVQ ·prime1v(SB), R13
-	MOVQ ·prime2v(SB), R14
-	MOVQ ·prime4v(SB), R15
-
-	// Load slice.
-	MOVQ b_base+0(FP), CX
-	MOVQ b_len+8(FP), DX
-	LEAQ (CX)(DX*1), BX
-
-	// The first loop limit will be len(b)-32.
-	SUBQ $32, BX
-
-	// Check whether we have at least one block.
-	CMPQ DX, $32
-	JLT  noBlocks
-
-	// Set up initial state (v1, v2, v3, v4).
-	MOVQ R13, R8
-	ADDQ R14, R8
-	MOVQ R14, R9
-	XORQ R10, R10
-	XORQ R11, R11
-	SUBQ R13, R11
-
-	// Loop until CX > BX.
-blockLoop:
-	round(R8)
-	round(R9)
-	round(R10)
-	round(R11)
-
-	CMPQ CX, BX
-	JLE  blockLoop
-
-	MOVQ R8, AX
-	ROLQ $1, AX
-	MOVQ R9, R12
-	ROLQ $7, R12
-	ADDQ R12, AX
-	MOVQ R10, R12
-	ROLQ $12, R12
-	ADDQ R12, AX
-	MOVQ R11, R12
-	ROLQ $18, R12
-	ADDQ R12, AX
-
-	mergeRound(AX, R8)
-	mergeRound(AX, R9)
-	mergeRound(AX, R10)
-	mergeRound(AX, R11)
-
-	JMP afterBlocks
-
-noBlocks:
-	MOVQ ·prime5v(SB), AX
-
-afterBlocks:
-	ADDQ DX, AX
-
-	// Right now BX has len(b)-32, and we want to loop until CX > len(b)-8.
-	ADDQ $24, BX
-
-	CMPQ CX, BX
-	JG   fourByte
-
-wordLoop:
-	// Calculate k1.
-	MOVQ  (CX), R8
-	ADDQ  $8, CX
-	IMULQ R14, R8
-	ROLQ  $31, R8
-	IMULQ R13, R8
-
-	XORQ  R8, AX
-	ROLQ  $27, AX
-	IMULQ R13, AX
-	ADDQ  R15, AX
-
-	CMPQ CX, BX
-	JLE  wordLoop
-
-fourByte:
-	ADDQ $4, BX
-	CMPQ CX, BX
-	JG   singles
-
-	MOVL  (CX), R8
-	ADDQ  $4, CX
-	IMULQ R13, R8
-	XORQ  R8, AX
-
-	ROLQ  $23, AX
-	IMULQ R14, AX
-	ADDQ  ·prime3v(SB), AX
-
-singles:
-	ADDQ $4, BX
-	CMPQ CX, BX
-	JGE  finalize
-
-singlesLoop:
-	MOVBQZX (CX), R12
-	ADDQ    $1, CX
-	IMULQ   ·prime5v(SB), R12
-	XORQ    R12, AX
-
-	ROLQ  $11, AX
-	IMULQ R13, AX
-
-	CMPQ CX, BX
-	JL   singlesLoop
-
-finalize:
-	MOVQ  AX, R12
-	SHRQ  $33, R12
-	XORQ  R12, AX
-	IMULQ R14, AX
-	MOVQ  AX, R12
-	SHRQ  $29, R12
-	XORQ  R12, AX
-	IMULQ ·prime3v(SB), AX
-	MOVQ  AX, R12
-	SHRQ  $32, R12
-	XORQ  R12, AX
-
-	MOVQ AX, ret+24(FP)
-	RET
-
-// writeBlocks uses the same registers as above except that it uses AX to store
-// the x pointer.
-
-// func writeBlocks(x *xxh, b []byte) []byte
-TEXT ·writeBlocks(SB), NOSPLIT, $0-56
-	// Load fixed primes needed for round.
-	MOVQ ·prime1v(SB), R13
-	MOVQ ·prime2v(SB), R14
-
-	// Load slice.
-	MOVQ b_base+8(FP), CX
-	MOVQ CX, ret_base+32(FP) // initialize return base pointer; see NOTE below
-	MOVQ b_len+16(FP), DX
-	LEAQ (CX)(DX*1), BX
-	SUBQ $32, BX
-
-	// Load vN from x.
-	MOVQ x+0(FP), AX
-	MOVQ 0(AX), R8   // v1
-	MOVQ 8(AX), R9   // v2
-	MOVQ 16(AX), R10 // v3
-	MOVQ 24(AX), R11 // v4
-
-	// We don't need to check the loop condition here; this function is
-	// always called with at least one block of data to process.
-blockLoop:
-	round(R8)
-	round(R9)
-	round(R10)
-	round(R11)
-
-	CMPQ CX, BX
-	JLE  blockLoop
-
-	// Copy vN back to x.
-	MOVQ R8, 0(AX)
-	MOVQ R9, 8(AX)
-	MOVQ R10, 16(AX)
-	MOVQ R11, 24(AX)
-
-	// Construct return slice.
-	// NOTE: It's important that we don't construct a slice that has a base
-	// pointer off the end of the original slice, as in Go 1.7+ this will
-	// cause runtime crashes. (See discussion in, for example,
-	// https://github.com/golang/go/issues/16772.)
-	// Therefore, we calculate the length/cap first, and if they're zero, we
-	// keep the old base. This is what the compiler does as well if you
-	// write code like
-	//   b = b[len(b):]
-
-	// New length is 32 - (CX - BX) -> BX+32 - CX.
-	ADDQ $32, BX
-	SUBQ CX, BX
-	JZ   afterSetBase
-
-	MOVQ CX, ret_base+32(FP)
-
-afterSetBase:
-	MOVQ BX, ret_len+40(FP)
-	MOVQ BX, ret_cap+48(FP) // set cap == len
-
-	RET

vendor/github.com/cespare/xxhash/xxhash_other.go

@@ -1,75 +0,0 @@
-// +build !amd64 appengine !gc purego
-
-package xxhash
-
-// Sum64 computes the 64-bit xxHash digest of b.
-func Sum64(b []byte) uint64 {
-	// A simpler version would be
-	//   x := New()
-	//   x.Write(b)
-	//   return x.Sum64()
-	// but this is faster, particularly for small inputs.
-
-	n := len(b)
-	var h uint64
-
-	if n >= 32 {
-		v1 := prime1v + prime2
-		v2 := prime2
-		v3 := uint64(0)
-		v4 := -prime1v
-		for len(b) >= 32 {
-			v1 = round(v1, u64(b[0:8:len(b)]))
-			v2 = round(v2, u64(b[8:16:len(b)]))
-			v3 = round(v3, u64(b[16:24:len(b)]))
-			v4 = round(v4, u64(b[24:32:len(b)]))
-			b = b[32:len(b):len(b)]
-		}
-		h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
-		h = mergeRound(h, v1)
-		h = mergeRound(h, v2)
-		h = mergeRound(h, v3)
-		h = mergeRound(h, v4)
-	} else {
-		h = prime5
-	}
-
-	h += uint64(n)
-
-	i, end := 0, len(b)
-	for ; i+8 <= end; i += 8 {
-		k1 := round(0, u64(b[i:i+8:len(b)]))
-		h ^= k1
-		h = rol27(h)*prime1 + prime4
-	}
-	if i+4 <= end {
-		h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
-		h = rol23(h)*prime2 + prime3
-		i += 4
-	}
-	for ; i < end; i++ {
-		h ^= uint64(b[i]) * prime5
-		h = rol11(h) * prime1
-	}
-
-	h ^= h >> 33
-	h *= prime2
-	h ^= h >> 29
-	h *= prime3
-	h ^= h >> 32
-
-	return h
-}
-
-func writeBlocks(x *xxh, b []byte) []byte {
-	v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4
-	for len(b) >= 32 {
-		v1 = round(v1, u64(b[0:8:len(b)]))
-		v2 = round(v2, u64(b[8:16:len(b)]))
-		v3 = round(v3, u64(b[16:24:len(b)]))
-		v4 = round(v4, u64(b[24:32:len(b)]))
-		b = b[32:len(b):len(b)]
-	}
-	x.v1, x.v2, x.v3, x.v4 = v1, v2, v3, v4
-	return b
-}

vendor/github.com/cespare/xxhash/xxhash_safe.go

@@ -1,10 +0,0 @@
-// +build appengine
-
-// This file contains the safe implementations of otherwise unsafe-using code.
-
-package xxhash
-
-// Sum64String computes the 64-bit xxHash digest of s.
-func Sum64String(s string) uint64 {
-	return Sum64([]byte(s))
-}

vendor/github.com/cespare/xxhash/xxhash_unsafe.go

@@ -1,30 +0,0 @@
-// +build !appengine
-
-// This file encapsulates usage of unsafe.
-// xxhash_safe.go contains the safe implementations.
-
-package xxhash
-
-import (
-	"reflect"
-	"unsafe"
-)
-
-// Sum64String computes the 64-bit xxHash digest of s.
-// It may be faster than Sum64([]byte(s)) by avoiding a copy.
-//
-// TODO(caleb): Consider removing this if an optimization is ever added to make
-// it unnecessary: https://golang.org/issue/2205.
-//
-// TODO(caleb): We still have a function call; we could instead write Go/asm
-// copies of Sum64 for strings to squeeze out a bit more speed.
-func Sum64String(s string) uint64 {
-	// See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ
-	// for some discussion about this unsafe conversion.
-	var b []byte
-	bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
-	bh.Len = len(s)
-	bh.Cap = len(s)
-	return Sum64(b)
-}

vendor/modules.txt

@@ -140,9 +140,6 @@ github.com/bytedance/sonic/loader/internal/rt
 # github.com/cenkalti/backoff/v4 v4.3.0
 ## explicit; go 1.18
 github.com/cenkalti/backoff/v4
-# github.com/cespare/xxhash v1.1.0
-## explicit
-github.com/cespare/xxhash
 # github.com/cespare/xxhash/v2 v2.3.0
 ## explicit; go 1.11
 github.com/cespare/xxhash/v2