plodding along

This commit is contained in:
tsmethurst 2021-05-12 13:07:52 +02:00
commit 77736b926b
6 changed files with 472 additions and 40 deletions

View file

@ -76,15 +76,15 @@ type Account struct {
*/
// Does this account need an approval for new followers?
Locked bool `pg:",default:true"`
Locked bool `pg:",default:'true'"`
// Should this account be shown in the instance's profile directory?
Discoverable bool
// Default post privacy for this account
Privacy Visibility
// Set posts from this account to sensitive by default?
Sensitive bool `pg:",default:false"`
Sensitive bool `pg:",default:'false'"`
// What language does this account post in?
Language string `pg:",default:en"`
Language string `pg:",default:'en'"`
/*
ACTIVITYPUB THINGS

View file

@ -24,6 +24,8 @@ import "time"
type Tag struct {
// id of this tag in the database
ID string `pg:",unique,type:uuid,default:gen_random_uuid(),pk,notnull"`
// Href of this tag, eg https://example.org/tags/somehashtag
URL string
// name of this tag -- the tag without the hash part
Name string `pg:",unique,pk,notnull"`
// Which account ID is the first one we saw using this tag?

View file

@ -25,9 +25,11 @@ import (
"errors"
"fmt"
"net/url"
"strings"
"time"
"github.com/go-fed/activity/pub"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func extractPreferredUsername(i withPreferredUsername) (string, error) {
@ -41,16 +43,16 @@ func extractPreferredUsername(i withPreferredUsername) (string, error) {
return u.GetXMLSchemaString(), nil
}
func extractName(i withDisplayName) (string, error) {
func extractName(i withName) (string, error) {
nameProp := i.GetActivityStreamsName()
if nameProp == nil {
return "", errors.New("activityStreamsName not found")
}
// take the first name string we can find
for nameIter := nameProp.Begin(); nameIter != nameProp.End(); nameIter = nameIter.Next() {
if nameIter.IsXMLSchemaString() && nameIter.GetXMLSchemaString() != "" {
return nameIter.GetXMLSchemaString(), nil
for iter := nameProp.Begin(); iter != nameProp.End(); iter = iter.Next() {
if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" {
return iter.GetXMLSchemaString(), nil
}
}
@ -59,10 +61,10 @@ func extractName(i withDisplayName) (string, error) {
func extractInReplyToURI(i withInReplyTo) (*url.URL, error) {
inReplyToProp := i.GetActivityStreamsInReplyTo()
for i := inReplyToProp.Begin(); i != inReplyToProp.End(); i = i.Next() {
if i.IsIRI() {
if i.GetIRI() != nil {
return i.GetIRI(), nil
for iter := inReplyToProp.Begin(); iter != inReplyToProp.End(); iter = iter.Next() {
if iter.IsIRI() {
if iter.GetIRI() != nil {
return iter.GetIRI(), nil
}
}
}
@ -72,10 +74,10 @@ func extractInReplyToURI(i withInReplyTo) (*url.URL, error) {
func extractTos(i withTo) ([]*url.URL, error) {
to := []*url.URL{}
toProp := i.GetActivityStreamsTo()
for i := toProp.Begin(); i != toProp.End(); i = i.Next() {
if i.IsIRI() {
if i.GetIRI() != nil {
to = append(to, i.GetIRI())
for iter := toProp.Begin(); iter != toProp.End(); iter = iter.Next() {
if iter.IsIRI() {
if iter.GetIRI() != nil {
to = append(to, iter.GetIRI())
}
}
}
@ -88,10 +90,10 @@ func extractTos(i withTo) ([]*url.URL, error) {
func extractCCs(i withCC) ([]*url.URL, error) {
cc := []*url.URL{}
ccProp := i.GetActivityStreamsCc()
for i := ccProp.Begin(); i != ccProp.End(); i = i.Next() {
if i.IsIRI() {
if i.GetIRI() != nil {
cc = append(cc, i.GetIRI())
for iter := ccProp.Begin(); iter != ccProp.End(); iter = iter.Next() {
if iter.IsIRI() {
if iter.GetIRI() != nil {
cc = append(cc, iter.GetIRI())
}
}
}
@ -103,10 +105,10 @@ func extractCCs(i withCC) ([]*url.URL, error) {
func extractAttributedTo(i withAttributedTo) (*url.URL, error) {
attributedToProp := i.GetActivityStreamsAttributedTo()
for aIter := attributedToProp.Begin(); aIter != attributedToProp.End(); aIter = aIter.Next() {
if aIter.IsIRI() {
if aIter.GetIRI() != nil {
return aIter.GetIRI(), nil
for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
if iter.IsIRI() {
if iter.GetIRI() != nil {
return iter.GetIRI(), nil
}
}
}
@ -146,12 +148,12 @@ func extractIconURL(i withIcon) (*url.URL, error) {
// here in order to find the first one that meets these criteria:
// 1. is an image
// 2. has a URL so we can grab it
for iconIter := iconProp.Begin(); iconIter != iconProp.End(); iconIter = iconIter.Next() {
for iter := iconProp.Begin(); iter != iconProp.End(); iter = iter.Next() {
// 1. is an image
if !iconIter.IsActivityStreamsImage() {
if !iter.IsActivityStreamsImage() {
continue
}
imageValue := iconIter.GetActivityStreamsImage()
imageValue := iter.GetActivityStreamsImage()
if imageValue == nil {
continue
}
@ -288,3 +290,166 @@ func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKe
}
return nil, nil, errors.New("couldn't find public key")
}
func extractContent(i withContent) (string, error) {
contentProperty := i.GetActivityStreamsContent()
if contentProperty == nil {
return "", errors.New("content property was nil")
}
for iter := contentProperty.Begin(); iter != contentProperty.End(); iter = iter.Next() {
if iter.IsXMLSchemaString() && iter.GetXMLSchemaString() != "" {
return iter.GetXMLSchemaString(), nil
}
}
return "", errors.New("no content found")
}
func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) {
attachments := []*gtsmodel.MediaAttachment{}
attachmentProp := i.GetActivityStreamsAttachment()
for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() {
attachmentable, ok := iter.(Attachmentable)
if !ok {
continue
}
attachment, err := extractAttachment(attachmentable)
if err != nil {
continue
}
attachments = append(attachments, attachment)
}
return attachments, nil
}
func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
attachment := &gtsmodel.MediaAttachment{
File: gtsmodel.File{},
}
attachmentURL, err := extractURL(i)
if err != nil {
return nil, err
}
attachment.RemoteURL = attachmentURL.String()
mediaType := i.GetActivityStreamsMediaType()
if mediaType == nil {
return nil, errors.New("no media type")
}
if mediaType.Get() == "" {
return nil, errors.New("no media type")
}
attachment.File.ContentType = mediaType.Get()
attachment.Type = gtsmodel.FileTypeImage
name, err := extractName(i)
if err == nil {
attachment.Description = name
}
blurhash, err := extractBlurhash(i)
if err == nil {
attachment.Blurhash = blurhash
}
return attachment, nil
}
func extractBlurhash(i withBlurhash) (string, error) {
if i.GetTootBlurhashProperty() == nil {
return "", errors.New("blurhash property was nil")
}
if i.GetTootBlurhashProperty().Get() == "" {
return "", errors.New("empty blurhash string")
}
return i.GetTootBlurhashProperty().Get(), nil
}
func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) {
tags := []*gtsmodel.Tag{}
tagsProp := i.GetActivityStreamsTag()
for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
t := iter.GetType()
if t == nil {
continue
}
if t.GetTypeName() != "Hashtag" {
continue
}
hashtaggable, ok := t.(Hashtaggable)
if !ok {
continue
}
tag, err := extractHashtag(hashtaggable)
if err != nil {
fmt.Println(err)
continue
}
tags = append(tags, tag)
}
return tags, nil
}
func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
tag := &gtsmodel.Tag{}
hrefProp := i.GetActivityStreamsHref()
if hrefProp == nil || !hrefProp.IsIRI() {
return nil, errors.New("no href prop")
}
tag.URL = hrefProp.GetIRI().String()
name, err := extractName(i)
if err != nil {
return nil, err
}
tag.Name = strings.TrimPrefix(name, "#")
return tag, nil
}
func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) {
emojis := []*gtsmodel.Emoji{}
tagsProp := i.GetActivityStreamsTag()
for iter := tagsProp.Begin(); iter != tagsProp.End(); iter = iter.Next() {
t := iter.GetType()
if t == nil {
continue
}
if t.GetTypeName() != "Emoji" {
continue
}
emojiable, ok := t.(Emojiable)
if !ok {
continue
}
emoji, err := extractEmoji(emojiable)
if err != nil {
continue
}
emojis = append(emojis, emoji)
}
return emojis, nil
}
func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
emoji := &gtsmodel.Emoji{}
idProp := i.GetJSONLDId()
if idProp == nil || !idProp.IsIRI() {
return nil, errors.New("no id for emoji")
}
emoji.URI = idProp.GetIRI().String()
return emoji, nil
}

View file

@ -24,11 +24,11 @@ import "github.com/go-fed/activity/streams/vocab"
// This interface is fulfilled by: Person, Application, Organization, Service, and Group
type Accountable interface {
withJSONLDId
withGetTypeName
withTypeName
withPreferredUsername
withIcon
withDisplayName
withName
withImage
withSummary
withDiscoverable
@ -45,7 +45,7 @@ type Accountable interface {
// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
type Statusable interface {
withJSONLDId
withGetTypeName
withTypeName
withSummary
withInReplyTo
@ -62,11 +62,37 @@ type Statusable interface {
withReplies
}
// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'.
// This interface is fulfilled by: Audio, Document, Image, Video
type Attachmentable interface {
withTypeName
withMediaType
withURL
withName
withBlurhash
withFocalPoint
}
// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag'.
type Hashtaggable interface {
withTypeName
withHref
withName
}
type Emojiable interface {
withJSONLDId
withTypeName
withName
withUpdated
withIcon
}
type withJSONLDId interface {
GetJSONLDId() vocab.JSONLDIdProperty
}
type withGetTypeName interface {
type withTypeName interface {
GetTypeName() string
}
@ -78,7 +104,7 @@ type withIcon interface {
GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty
}
type withDisplayName interface {
type withName interface {
GetActivityStreamsName() vocab.ActivityStreamsNameProperty
}
@ -165,3 +191,23 @@ type withTag interface {
type withReplies interface {
GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty
}
type withMediaType interface {
GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty
}
type withBlurhash interface {
GetTootBlurhashProperty() vocab.TootBlurhashProperty
}
type withFocalPoint interface {
// TODO
}
type withHref interface {
GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty
}
type withUpdated interface {
GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty
}

View file

@ -167,6 +167,28 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e
}
status.URI = uriProp.GetIRI().String()
statusURL, err := extractURL(statusable)
if err == nil {
status.URL = statusURL.String()
}
if content, err := extractContent(statusable); err == nil {
status.Content = content
}
attachments, err := extractAttachments(statusable); if err == nil {
status.GTSMediaAttachments = attachments
}
hashtags, err := extractHashtags(statusable)
if err == nil {
status.GTSTags = hashtags
}
// emojis, err := extractEmojis(statusable)
// mentions, err := extractMentions(statusable)
cw, err := extractSummary(statusable)
if err == nil && cw != "" {
status.ContentWarning = cw
@ -185,21 +207,82 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e
status.CreatedAt = published
}
statusURL, err := extractURL(statusable)
if err == nil {
status.URL = statusURL.String()
}
attributedTo, err := extractAttributedTo(statusable)
if err != nil {
return nil, errors.New("attributedTo was empty")
}
// if we don't know the account yet we can dereference it later
statusOwner := &gtsmodel.Status{}
if err := c.db.GetWhere("uri", attributedTo.String(), statusOwner); err != nil {
return nil, fmt.Errorf("cannot attribute %s to an account we know: %s", attributedTo.String(), err)
if err := c.db.GetWhere("uri", attributedTo.String(), statusOwner); err == nil {
status.AccountID = statusOwner.ID
}
status.AccountID = statusOwner.ID
return nil, nil
return status, nil
}
// // id of the status in the database
// ID string `pg:"type:uuid,default:gen_random_uuid(),pk,notnull"`
// // uri at which this status is reachable
// URI string `pg:",unique"`
// // web url for viewing this status
// URL string `pg:",unique"`
// // the html-formatted content of this status
// Content string
// // Database IDs of any media attachments associated with this status
// Attachments []string `pg:",array"`
// // Database IDs of any tags used in this status
// Tags []string `pg:",array"`
// // Database IDs of any accounts mentioned in this status
// Mentions []string `pg:",array"`
// // Database IDs of any emojis used in this status
// Emojis []string `pg:",array"`
// // when was this status created?
// CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
// // when was this status updated?
// UpdatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
// // is this status from a local account?
// Local bool
// // which account posted this status?
// AccountID string
// // id of the status this status is a reply to
// InReplyToID string
// // id of the account that this status replies to
// InReplyToAccountID string
// // id of the status this status is a boost of
// BoostOfID string
// // cw string for this status
// ContentWarning string
// // visibility entry for this status
// Visibility Visibility `pg:",notnull"`
// // mark the status as sensitive?
// Sensitive bool
// // what language is this status written in?
// Language string
// // Which application was used to create this status?
// CreatedWithApplicationID string
// // advanced visibility for this status
// VisibilityAdvanced *VisibilityAdvanced
// // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types
// // Will probably almost always be Note but who knows!.
// ActivityStreamsType ActivityStreamsObject
// // Original text of the status without formatting
// Text string
// // Mentions created in this status
// GTSMentions []*Mention `pg:"-"`
// // Hashtags used in this status
// GTSTags []*Tag `pg:"-"`
// // Emojis used in this status
// GTSEmojis []*Emoji `pg:"-"`
// // MediaAttachments used in this status
// GTSMediaAttachments []*MediaAttachment `pg:"-"`
// // Status being replied to
// GTSReplyToStatus *Status `pg:"-"`
// // Account being replied to
// GTSReplyToAccount *Account `pg:"-"`
// // Status being boosted
// GTSBoostedStatus *Status `pg:"-"`
// // Account of the boosted status
// GTSBoostedAccount *Account `pg:"-"`

View file

@ -25,6 +25,7 @@ import (
"testing"
"github.com/go-fed/activity/streams"
"github.com/go-fed/activity/streams/vocab"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@ -36,6 +37,115 @@ type ASToInternalTestSuite struct {
}
const (
statusAsActivityJson = `{
"@context": [
"https://www.w3.org/ns/activitystreams",
{
"ostatus": "http://ostatus.org#",
"atomUri": "ostatus:atomUri",
"inReplyToAtomUri": "ostatus:inReplyToAtomUri",
"conversation": "ostatus:conversation",
"sensitive": "as:sensitive",
"toot": "http://joinmastodon.org/ns#",
"votersCount": "toot:votersCount",
"Hashtag": "as:Hashtag",
"Emoji": "toot:Emoji",
"focalPoint": {
"@container": "@list",
"@id": "toot:focalPoint"
}
}
],
"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704/activity",
"type": "Create",
"actor": "https://ondergrond.org/users/dumpsterqueer",
"published": "2021-05-12T09:41:38Z",
"to": [
"https://ondergrond.org/users/dumpsterqueer/followers"
],
"cc": [
"https://www.w3.org/ns/activitystreams#Public"
],
"object": {
"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704",
"type": "Note",
"summary": null,
"inReplyTo": null,
"published": "2021-05-12T09:41:38Z",
"url": "https://ondergrond.org/@dumpsterqueer/106221567884565704",
"attributedTo": "https://ondergrond.org/users/dumpsterqueer",
"to": [
"https://ondergrond.org/users/dumpsterqueer/followers"
],
"cc": [
"https://www.w3.org/ns/activitystreams#Public"
],
"sensitive": false,
"atomUri": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704",
"inReplyToAtomUri": null,
"conversation": "tag:ondergrond.org,2021-05-12:objectId=1132361:objectType=Conversation",
"content": "<p>just testing activitypub representations of <a href=\"https://ondergrond.org/tags/tags\" class=\"mention hashtag\" rel=\"tag\">#<span>tags</span></a> and <a href=\"https://ondergrond.org/tags/emoji\" class=\"mention hashtag\" rel=\"tag\">#<span>emoji</span></a> :party_parrot: :amaze: :blobsunglasses: </p><p>don&apos;t mind me....</p>",
"contentMap": {
"en": "<p>just testing activitypub representations of <a href=\"https://ondergrond.org/tags/tags\" class=\"mention hashtag\" rel=\"tag\">#<span>tags</span></a> and <a href=\"https://ondergrond.org/tags/emoji\" class=\"mention hashtag\" rel=\"tag\">#<span>emoji</span></a> :party_parrot: :amaze: :blobsunglasses: </p><p>don&apos;t mind me....</p>"
},
"attachment": [],
"tag": [
{
"type": "Hashtag",
"href": "https://ondergrond.org/tags/tags",
"name": "#tags"
},
{
"type": "Hashtag",
"href": "https://ondergrond.org/tags/emoji",
"name": "#emoji"
},
{
"id": "https://ondergrond.org/emojis/2390",
"type": "Emoji",
"name": ":party_parrot:",
"updated": "2020-11-06T13:42:11Z",
"icon": {
"type": "Image",
"mediaType": "image/gif",
"url": "https://ondergrond.org/system/custom_emojis/images/000/002/390/original/ef133aac7ab23341.gif"
}
},
{
"id": "https://ondergrond.org/emojis/2395",
"type": "Emoji",
"name": ":amaze:",
"updated": "2020-09-26T12:29:56Z",
"icon": {
"type": "Image",
"mediaType": "image/png",
"url": "https://ondergrond.org/system/custom_emojis/images/000/002/395/original/2c7d9345e57367ed.png"
}
},
{
"id": "https://ondergrond.org/emojis/764",
"type": "Emoji",
"name": ":blobsunglasses:",
"updated": "2020-09-26T12:13:23Z",
"icon": {
"type": "Image",
"mediaType": "image/png",
"url": "https://ondergrond.org/system/custom_emojis/images/000/000/764/original/3f8eef9de773c90d.png"
}
}
],
"replies": {
"id": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704/replies",
"type": "Collection",
"first": {
"type": "CollectionPage",
"next": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704/replies?only_other_accounts=true&page=true",
"partOf": "https://ondergrond.org/users/dumpsterqueer/statuses/106221567884565704/replies",
"items": []
}
}
}
}`
gargronAsActivityJson = `{
"@context": [
"https://www.w3.org/ns/activitystreams",
@ -197,6 +307,32 @@ func (suite *ASToInternalTestSuite) TestParseGargron() {
// TODO: write assertions here, rn we're just eyeballing the output
}
func (suite *ASToInternalTestSuite) TestParseStatus() {
m := make(map[string]interface{})
err := json.Unmarshal([]byte(statusAsActivityJson), &m)
assert.NoError(suite.T(), err)
t, err := streams.ToType(context.Background(), m)
assert.NoError(suite.T(), err)
create, ok := t.(vocab.ActivityStreamsCreate)
assert.True(suite.T(), ok)
obj := create.GetActivityStreamsObject()
assert.NotNil(suite.T(), obj)
first := obj.Begin()
assert.NotNil(suite.T(), first)
rep, ok := first.GetType().(typeutils.Statusable)
assert.True(suite.T(), ok)
status, err := suite.typeconverter.ASStatusToStatus(rep)
assert.NoError(suite.T(), err)
fmt.Printf("%+v", status)
}
func (suite *ASToInternalTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
}