more work on parsing statuses

internal/typeutils/asextractionutil.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/go-fed/activity/pub"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 func extractPreferredUsername(i withPreferredUsername) (string, error) {
@@ -184,12 +185,12 @@ func extractImageURL(i withImage) (*url.URL, error) {
 	// here in order to find the first one that meets these criteria:
 	// 1. is an image
 	// 2. has a URL so we can grab it
-	for imageIter := imageProp.Begin(); imageIter != imageProp.End(); imageIter = imageIter.Next() {
+	for iter := imageProp.Begin(); iter != imageProp.End(); iter = iter.Next() {
 		// 1. is an image
-		if !imageIter.IsActivityStreamsImage() {
+		if !iter.IsActivityStreamsImage() {
 			continue
 		}
-		imageValue := imageIter.GetActivityStreamsImage()
+		imageValue := iter.GetActivityStreamsImage()
 		if imageValue == nil {
 			continue
 		}
@@ -210,9 +211,9 @@ func extractSummary(i withSummary) (string, error) {
 		return "", errors.New("summary property was nil")
 	}
 
-	for summaryIter := summaryProp.Begin(); summaryIter != summaryProp.End(); summaryIter = summaryIter.Next() {
-		if summaryIter.IsXMLSchemaString() && summaryIter.GetXMLSchemaString() != "" {
-			return summaryIter.GetXMLSchemaString(), nil
+	for iter := summaryProp.Begin(); iter != summaryProp.End(); iter = iter.Next() {
+		if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" {
+			return iter.GetXMLSchemaString(), nil
 		}
 	}
 
@@ -232,9 +233,9 @@ func extractURL(i withURL) (*url.URL, error) {
 		return nil, errors.New("url property was nil")
 	}
 
-	for urlIter := urlProp.Begin(); urlIter != urlProp.End(); urlIter = urlIter.Next() {
-		if urlIter.IsIRI() && urlIter.GetIRI() != nil {
-			return urlIter.GetIRI(), nil
+	for iter := urlProp.Begin(); iter != urlProp.End(); iter = iter.Next() {
+		if iter.IsIRI() && iter.GetIRI() != nil {
+			return iter.GetIRI(), nil
 		}
 	}
 
@@ -247,8 +248,8 @@ func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKe
 		return nil, nil, errors.New("public key property was nil")
 	}
 
-	for publicKeyIter := publicKeyProp.Begin(); publicKeyIter != publicKeyProp.End(); publicKeyIter = publicKeyIter.Next() {
-		pkey := publicKeyIter.Get()
+	for iter := publicKeyProp.Begin(); iter != publicKeyProp.End(); iter = iter.Next() {
+		pkey := iter.Get()
 		if pkey == nil {
 			continue
 		}
@@ -449,7 +450,79 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
 	if idProp == nil || !idProp.IsIRI() {
 		return nil, errors.New("no id for emoji")
 	}
-	emoji.URI = idProp.GetIRI().String()
+	uri := idProp.GetIRI()
+	emoji.URI = uri.String()
+	emoji.Domain = uri.Host
+
+	name, err := extractName(i)
+	if err != nil {
+		return nil, err
+	}
+	emoji.Shortcode = strings.Trim(name, ":")
+
+	if i.GetActivityStreamsIcon() == nil {
+		return nil, errors.New("no icon for emoji")
+	}
+	imageURL, err := extractIconURL(i)
+	if err != nil {
+		return nil, errors.New("no url for emoji image")
+	}
+	emoji.ImageRemoteURL = imageURL.String()
 
 	return emoji, nil
 }
+
+func extractMentions(i withTag) ([]*gtsmodel.Mention, error) {
+	mentions := []*gtsmodel.Mention{}
+	tagsProp := i.GetActivityStreamsTag()
+	for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
+		t := iter.GetType()
+		if t == nil {
+			continue
+		}
+
+		if t.GetTypeName() != "Mention" {
+			continue
+		}
+
+		mentionable, ok := t.(Mentionable)
+		if !ok {
+			continue
+		}
+
+		mention, err := extractMention(mentionable)
+		if err != nil {
+			continue
+		}
+
+		mentions = append(mentions, mention)
+	}
+	return mentions, nil
+}
+
+func extractMention(i Mentionable) (*gtsmodel.Mention, error) {
+	mention := &gtsmodel.Mention{}
+
+	mentionString, err := extractName(i)
+	if err != nil {
+		return nil, err
+	}
+
+	// just make sure the mention string is valid so we can handle it properly later on...
+	username, domain, err := util.ExtractMentionParts(mentionString)
+	if err != nil {
+		return nil, err
+	}
+	if username == "" || domain == "" {
+		return nil, errors.New("username or domain was empty")
+	}
+
+	// the href prop should be the URL of a user we know, eg https://example.org/@whatever_user
+	hrefProp := i.GetActivityStreamsHref()
+	if hrefProp == nil || !hrefProp.IsIRI() {
+		return nil, errors.New("no href prop")
+	}
+	mention.Href = hrefProp.GetIRI().String()
+
+	return mention, nil
+}

internal/typeutils/asinterfaces.go

@@ -73,13 +73,14 @@ type Attachmentable interface {
 	withFocalPoint
 }
 
-// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag'.
+// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag.
 type Hashtaggable interface {
 	withTypeName
 	withHref
 	withName
 }
 
+// Emojiable represents the minimum interface for an 'emoji' tag.
 type Emojiable interface {
 	withJSONLDId
 	withTypeName
@@ -88,6 +89,12 @@ type Emojiable interface {
 	withIcon
 }
 
+// Mentionable represents the minimum interface for a 'mention' tag.
+type Mentionable interface {
+	withName
+	withHref
+}
+
 type withJSONLDId interface {
 	GetJSONLDId() vocab.JSONLDIdProperty
 }

internal/typeutils/astointernal.go

@@ -185,9 +185,15 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e
 		status.GTSTags = hashtags
 	}
 
-	// emojis, err := extractEmojis(statusable)
+	emojis, err := extractEmojis(statusable)
+	if err == nil {
+		status.GTSEmojis = emojis
+	}
 
-	// mentions, err := extractMentions(statusable)
+	mentions, err := extractMentions(statusable)
+	if err == nil {
+		status.GTSMentions = mentions
+	}
 
 	cw, err := extractSummary(statusable)
 	if err == nil && cw != "" {

internal/typeutils/astointernal_test.go

@@ -37,7 +37,74 @@ type ASToInternalTestSuite struct {
 }
 
 const (
-	statusAsActivityJson = `{
+	statusWithMentionsActivityJson = `{
+		"@context": [
+		  "https://www.w3.org/ns/activitystreams",
+		  {
+			"ostatus": "http://ostatus.org#",
+			"atomUri": "ostatus:atomUri",
+			"inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+			"conversation": "ostatus:conversation",
+			"sensitive": "as:sensitive",
+			"toot": "http://joinmastodon.org/ns#",
+			"votersCount": "toot:votersCount"
+		  }
+		],
+		"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552/activity",
+		"type": "Create",
+		"actor": "https://ondergrond.org/users/dumpsterqueer",
+		"published": "2021-05-12T09:58:38Z",
+		"to": [
+		  "https://ondergrond.org/users/dumpsterqueer/followers"
+		],
+		"cc": [
+		  "https://www.w3.org/ns/activitystreams#Public",
+		  "https://social.pixie.town/users/f0x"
+		],
+		"object": {
+		  "id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552",
+		  "type": "Note",
+		  "summary": null,
+		  "inReplyTo": "https://social.pixie.town/users/f0x/statuses/106221628567855262",
+		  "published": "2021-05-12T09:58:38Z",
+		  "url": "https://ondergrond.org/@dumpsterqueer/106221634728637552",
+		  "attributedTo": "https://ondergrond.org/users/dumpsterqueer",
+		  "to": [
+			"https://ondergrond.org/users/dumpsterqueer/followers"
+		  ],
+		  "cc": [
+			"https://www.w3.org/ns/activitystreams#Public",
+			"https://social.pixie.town/users/f0x"
+		  ],
+		  "sensitive": false,
+		  "atomUri": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552",
+		  "inReplyToAtomUri": "https://social.pixie.town/users/f0x/statuses/106221628567855262",
+		  "conversation": "tag:ondergrond.org,2021-05-12:objectId=1132361:objectType=Conversation",
+		  "content": "<p><span class=\"h-card\"><a href=\"https://social.pixie.town/@f0x\" class=\"u-url mention\">@<span>f0x</span></a></span> nice there it is:</p><p><a href=\"https://social.pixie.town/users/f0x/statuses/106221628567855262/activity\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">social.pixie.town/users/f0x/st</span><span class=\"invisible\">atuses/106221628567855262/activity</span></a></p>",
+		  "contentMap": {
+			"en": "<p><span class=\"h-card\"><a href=\"https://social.pixie.town/@f0x\" class=\"u-url mention\">@<span>f0x</span></a></span> nice there it is:</p><p><a href=\"https://social.pixie.town/users/f0x/statuses/106221628567855262/activity\" rel=\"nofollow noopener noreferrer\" target=\"_blank\"><span class=\"invisible\">https://</span><span class=\"ellipsis\">social.pixie.town/users/f0x/st</span><span class=\"invisible\">atuses/106221628567855262/activity</span></a></p>"
+		  },
+		  "attachment": [],
+		  "tag": [
+			{
+			  "type": "Mention",
+			  "href": "https://social.pixie.town/users/f0x",
+			  "name": "@f0x@pixie.town"
+			}
+		  ],
+		  "replies": {
+			"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552/replies",
+			"type": "Collection",
+			"first": {
+			  "type": "CollectionPage",
+			  "next": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552/replies?only_other_accounts=true&page=true",
+			  "partOf": "https://ondergrond.org/users/dumpsterqueer/statuses/106221634728637552/replies",
+			  "items": []
+			}
+		  }
+		}
+	  }`
+	statusWithEmojisAndTagsAsActivityJson = `{
 		"@context": [
 		  "https://www.w3.org/ns/activitystreams",
 		  {
@@ -309,7 +376,34 @@ func (suite *ASToInternalTestSuite) TestParseGargron() {
 
 func (suite *ASToInternalTestSuite) TestParseStatus() {
 	m := make(map[string]interface{})
-	err := json.Unmarshal([]byte(statusAsActivityJson), &m)
+	err := json.Unmarshal([]byte(statusWithEmojisAndTagsAsActivityJson), &m)
+	assert.NoError(suite.T(), err)
+
+	t, err := streams.ToType(context.Background(), m)
+	assert.NoError(suite.T(), err)
+
+	create, ok := t.(vocab.ActivityStreamsCreate)
+	assert.True(suite.T(), ok)
+
+	obj := create.GetActivityStreamsObject()
+	assert.NotNil(suite.T(), obj)
+
+	first := obj.Begin()
+	assert.NotNil(suite.T(), first)
+
+	rep, ok := first.GetType().(typeutils.Statusable)
+	assert.True(suite.T(), ok)
+
+	status, err := suite.typeconverter.ASStatusToStatus(rep)
+	assert.NoError(suite.T(), err)
+
+	assert.Len(suite.T(), status.GTSEmojis, 3)
+	// assert.Len(suite.T(), status.GTSTags, 2) TODO: implement this first so that it can pick up tags
+}
+
+func (suite *ASToInternalTestSuite) TestParseStatusWithMention() {
+	m := make(map[string]interface{})
+	err := json.Unmarshal([]byte(statusWithMentionsActivityJson), &m)
 	assert.NoError(suite.T(), err)
 
 	t, err := streams.ToType(context.Background(), m)
@@ -331,6 +425,9 @@ func (suite *ASToInternalTestSuite) TestParseStatus() {
 	assert.NoError(suite.T(), err)
 
 	fmt.Printf("%+v", status)
+
+	assert.Len(suite.T(), status.GTSMentions, 1)
+	fmt.Println(status.GTSMentions[0])
 }
 
 func (suite *ASToInternalTestSuite) TearDownTest() {