[feature] Hashtag federation (in/out), hashtag client API endpoints (#2032)

* update go-fed

* do the things

* remove unused columns from tags

* update to latest lingo from main

* further tag shenanigans

* serve stub page at tag endpoint

* we did it lads

* tests, oh tests, ohhh tests, oh tests (doo doo doo doo)

* swagger docs

* document hashtag usage + federation

* instanceGet

* don't bother parsing tag href

* rename whereStartsWith -> whereStartsLike

* remove GetOrCreateTag

* dont cache status tag timelineability

internal/federation/dereferencing/status.go

@@ -288,7 +288,10 @@ func (d *deref) enrichStatus(
 		return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err)
 	}
 
-	// TODO: populateStatusTags()
+	// Ensure the status' tags are populated.
+	if err := d.fetchStatusTags(ctx, requestUser, latestStatus); err != nil {
+		return nil, nil, gtserror.Newf("error populating tags for status %s: %w", uri, err)
+	}
 
 	// Ensure the status' media attachments are populated, passing in existing to check for changes.
 	if err := d.fetchStatusAttachments(ctx, tsport, status, latestStatus); err != nil {
@@ -400,6 +403,55 @@ func (d *deref) fetchStatusMentions(ctx context.Context, requestUser string, exi
 	return nil
 }
 
+func (d *deref) fetchStatusTags(ctx context.Context, requestUser string, status *gtsmodel.Status) error {
+	// Allocate new slice to take the yet-to-be determined tag IDs.
+	status.TagIDs = make([]string, len(status.Tags))
+
+	for i := range status.Tags {
+		placeholder := status.Tags[i]
+
+		// Look for existing tag with this name first.
+		tag, err := d.state.DB.GetTagByName(ctx, placeholder.Name)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			log.Errorf(ctx, "db error getting tag %s: %v", tag.Name, err)
+			continue
+		}
+
+		// No tag with this name yet, create it.
+		if tag == nil {
+			tag = &gtsmodel.Tag{
+				ID:   id.NewULID(),
+				Name: placeholder.Name,
+			}
+
+			if err := d.state.DB.PutTag(ctx, tag); err != nil {
+				log.Errorf(ctx, "db error putting tag %s: %v", tag.Name, err)
+				continue
+			}
+		}
+
+		// Set the *new* tag and ID.
+		status.Tags[i] = tag
+		status.TagIDs[i] = tag.ID
+	}
+
+	// Remove any tag we couldn't get or create.
+	for i := 0; i < len(status.TagIDs); {
+		if status.TagIDs[i] == "" {
+			// This is a failed tag population, likely due
+			// to some database peculiarity / race condition.
+			copy(status.Tags[i:], status.Tags[i+1:])
+			copy(status.TagIDs[i:], status.TagIDs[i+1:])
+			status.Tags = status.Tags[:len(status.Tags)-1]
+			status.TagIDs = status.TagIDs[:len(status.TagIDs)-1]
+			continue
+		}
+		i++
+	}
+
+	return nil
+}
+
 func (d *deref) fetchStatusAttachments(ctx context.Context, tsport transport.Transport, existing, status *gtsmodel.Status) error {
 	// Allocate new slice to take the yet-to-be fetched attachment IDs.
 	status.AttachmentIDs = make([]string, len(status.Attachments))

internal/federation/dereferencing/status_test.go

@@ -123,6 +123,56 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
 	suite.False(*m.Silent)
 }
 
+func (suite *StatusTestSuite) TestDereferenceStatusWithTag() {
+	fetchingAccount := suite.testAccounts["local_account_1"]
+
+	statusURL := testrig.URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7")
+	status, _, err := suite.dereferencer.GetStatusByURI(context.Background(), fetchingAccount.Username, statusURL)
+	suite.NoError(err)
+	suite.NotNil(status)
+
+	// status values should be set
+	suite.Equal("https://unknown-instance.com/users/brand_new_person/statuses/01H641QSRS3TCXSVC10X4GPKW7", status.URI)
+	suite.Equal("https://unknown-instance.com/users/@brand_new_person/01H641QSRS3TCXSVC10X4GPKW7", status.URL)
+	suite.Equal("<p>Babe are you okay, you've hardly touched your <a href=\"https://unknown-instance.com/tags/piss\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>piss</span></a></p>", status.Content)
+	suite.Equal("https://unknown-instance.com/users/brand_new_person", status.AccountURI)
+	suite.False(*status.Local)
+	suite.Empty(status.ContentWarning)
+	suite.Equal(gtsmodel.VisibilityPublic, status.Visibility)
+	suite.Equal(ap.ObjectNote, status.ActivityStreamsType)
+
+	// Ensure tags set + ID'd.
+	suite.Len(status.Tags, 1)
+	suite.Len(status.TagIDs, 1)
+
+	// status should be in the database
+	dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
+	suite.NoError(err)
+	suite.Equal(status.ID, dbStatus.ID)
+	suite.True(*dbStatus.Federated)
+	suite.True(*dbStatus.Boostable)
+	suite.True(*dbStatus.Replyable)
+	suite.True(*dbStatus.Likeable)
+
+	// account should be in the database now too
+	account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)
+	suite.NoError(err)
+	suite.NotNil(account)
+	suite.True(*account.Discoverable)
+	suite.Equal("https://unknown-instance.com/users/brand_new_person", account.URI)
+	suite.Equal("hey I'm a new person, your instance hasn't seen me yet uwu", account.Note)
+	suite.Equal("Geoff Brando New Personson", account.DisplayName)
+	suite.Equal("brand_new_person", account.Username)
+	suite.NotNil(account.PublicKey)
+	suite.Nil(account.PrivateKey)
+
+	// we should have a tag in the database
+	t := &gtsmodel.Tag{}
+	err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "name", Value: "piss"}}, t)
+	suite.NoError(err)
+	suite.NotNil(t)
+}
+
 func (suite *StatusTestSuite) TestDereferenceStatusWithImageAndNoContent() {
 	fetchingAccount := suite.testAccounts["local_account_1"]
 

internal/federation/federatingactor_test.go

@@ -50,6 +50,7 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() {
 		false,
 		nil,
 		nil,
+		nil,
 	)
 	testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), time.Now(), testNote)
 
@@ -97,6 +98,7 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
 		false,
 		nil,
 		nil,
+		nil,
 	)
 	testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), testNote)