From 3ba528120c1a76dcd9986d593858d69313b4453a Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Wed, 12 May 2021 21:47:45 +0200 Subject: [PATCH] fixed a few things --- internal/gtsmodel/mention.go | 10 +- internal/gtsmodel/status.go | 18 ++- internal/typeutils/asextractionutil.go | 7 +- internal/typeutils/astointernal.go | 146 ++++++++++--------------- internal/util/regexes.go | 6 +- internal/util/statustools.go | 11 +- 6 files changed, 92 insertions(+), 106 deletions(-) diff --git a/internal/gtsmodel/mention.go b/internal/gtsmodel/mention.go index c8e94158e..8e56a1b36 100644 --- a/internal/gtsmodel/mention.go +++ b/internal/gtsmodel/mention.go @@ -41,11 +41,11 @@ type Mention struct { // NameString is for putting in the namestring of the mentioned user // before the mention is dereferenced. Should be in a form along the lines of: // @whatever_username@example.org - // - // This will not be put in the database, it's just for convenience. - NameString string `pg:"-"` - // Href is the web URL (not AP uri!) of the user mentioned. // // This will not be put in the database, it's just for convenience. - Href string `pg:"-"` + NameString string `pg:"-"` + // MentionedAccountURI is the AP ID (uri) of the user mentioned. + // + // This will not be put in the database, it's just for convenience. + MentionedAccountURI string `pg:"-"` } diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index 55ae91599..8693bce30 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -71,12 +71,14 @@ type Status struct { Text string /* - NON-DATABASE FIELDS + INTERNAL MODEL NON-DATABASE FIELDS These are for convenience while passing the status around internally, but these fields should *never* be put in the db. */ + // Account that created this status + GTSAccount *Account `pg:"-"` // Mentions created in this status GTSMentions []*Mention `pg:"-"` // Hashtags used in this status @@ -93,6 +95,20 @@ type Status struct { GTSBoostedStatus *Status `pg:"-"` // Account of the boosted status GTSBoostedAccount *Account `pg:"-"` + + /* + AP NON-DATABASE FIELDS + + These are for convenience while passing the status around internally, + but these fields should *never* be put in the db. + */ + + // AP URI of the status being replied to. + // Useful when that status doesn't exist in the database yet and we still need to dereference it. + APReplyToStatusURI string `pg:"-"` + // The AP URI of the owner/creator of the status. + // Useful when that account doesn't exist in the database yet and we still need to dereference it. + APStatusOwnerURI string `pg:"-"` } // Visibility represents the visibility granularity of a status. diff --git a/internal/typeutils/asextractionutil.go b/internal/typeutils/asextractionutil.go index 82628c434..f3d2b1c4f 100644 --- a/internal/typeutils/asextractionutil.go +++ b/internal/typeutils/asextractionutil.go @@ -388,7 +388,6 @@ func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) { tag, err := extractHashtag(hashtaggable) if err != nil { - fmt.Println(err) continue } @@ -516,13 +515,13 @@ func extractMention(i Mentionable) (*gtsmodel.Mention, error) { if username == "" || domain == "" { return nil, errors.New("username or domain was empty") } + mention.NameString = mentionString - // the href prop should be the URL of a user we know, eg https://example.org/@whatever_user + // the href prop should be the AP URI of a user we know, eg https://example.org/users/whatever_user hrefProp := i.GetActivityStreamsHref() if hrefProp == nil || !hrefProp.IsIRI() { return nil, errors.New("no href prop") } - mention.Href = hrefProp.GetIRI().String() - + mention.MentionedAccountURI = hrefProp.GetIRI().String() return mention, nil } diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go index afaddf964..e2017f9b8 100644 --- a/internal/typeutils/astointernal.go +++ b/internal/typeutils/astointernal.go @@ -161,134 +161,108 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable) (*gtsmode func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) { status := >smodel.Status{} + // uri at which this status is reachable uriProp := statusable.GetJSONLDId() if uriProp == nil || !uriProp.IsIRI() { return nil, errors.New("no id property found, or id was not an iri") } status.URI = uriProp.GetIRI().String() - statusURL, err := extractURL(statusable) - if err == nil { + // web url for viewing this status + if statusURL, err := extractURL(statusable); err == nil { status.URL = statusURL.String() } + // the html-formatted content of this status if content, err := extractContent(statusable); err == nil { status.Content = content } - attachments, err := extractAttachments(statusable); if err == nil { + // attachments to dereference and fetch later on (we don't do that here) + if attachments, err := extractAttachments(statusable); err == nil { status.GTSMediaAttachments = attachments } - hashtags, err := extractHashtags(statusable) - if err == nil { + // hashtags to dereference later on + if hashtags, err := extractHashtags(statusable); err == nil { status.GTSTags = hashtags } - emojis, err := extractEmojis(statusable) - if err == nil { + // emojis to dereference and fetch later on + if emojis, err := extractEmojis(statusable); err == nil { status.GTSEmojis = emojis } - mentions, err := extractMentions(statusable) - if err == nil { + // mentions to dereference later on + if mentions, err := extractMentions(statusable); err == nil { status.GTSMentions = mentions } - cw, err := extractSummary(statusable) - if err == nil && cw != "" { + // cw string for this status + if cw, err := extractSummary(statusable); err == nil { status.ContentWarning = cw } - inReplyToURI, err := extractInReplyToURI(statusable) - if err == nil { - inReplyToStatus := >smodel.Status{} - if err := c.db.GetWhere("uri", inReplyToURI.String(), inReplyToStatus); err == nil { - status.InReplyToID = inReplyToStatus.ID - } - } - + // when was this status created? published, err := extractPublished(statusable) if err == nil { status.CreatedAt = published } + // which account posted this status? + // if we don't know the account yet we can dereference it later attributedTo, err := extractAttributedTo(statusable) if err != nil { return nil, errors.New("attributedTo was empty") } + status.APStatusOwnerURI = attributedTo.String() - // if we don't know the account yet we can dereference it later - statusOwner := >smodel.Status{} + statusOwner := >smodel.Account{} if err := c.db.GetWhere("uri", attributedTo.String(), statusOwner); err == nil { status.AccountID = statusOwner.ID + status.GTSAccount = statusOwner } + // check if there's a post that this is a reply to + inReplyToURI, err := extractInReplyToURI(statusable) + if err == nil { + // something is set so we can at least set this field on the + // status and dereference using this later if we need to + status.APReplyToStatusURI = inReplyToURI.String() + + // now we can check if we have the replied-to status in our db already + inReplyToStatus := >smodel.Status{} + if err := c.db.GetWhere("uri", inReplyToURI.String(), inReplyToStatus); err == nil { + // we have the status in our database already + // so we can set these fields here and then... + status.InReplyToID = inReplyToStatus.ID + status.InReplyToAccountID = inReplyToStatus.AccountID + status.GTSReplyToStatus = inReplyToStatus + + // ... check if we've seen the account already + inReplyToAccount := >smodel.Account{} + if err := c.db.GetByID(inReplyToStatus.AccountID, inReplyToAccount); err == nil { + status.GTSReplyToAccount = inReplyToAccount + } + } + } + + // visibility entry for this status + // TODO: if it's just got followers in TO and it's not CC'ed to public, it's followers only + // TODO: if it's CC'ed to public, it's public or unlocked + // TODO: if it's a DM then it's addressed to SPECIFIC ACCOUNTS and not followers or public + // TODO: mentioned SPECIFIC ACCOUNTS also get added to CC'es if it's not a direct message + + // advanced visibility for this status + // TODO: a lot of work to be done here -- a new type needs to be created for this in go-fed/activity using ASTOOL + + // sensitive + // TODO: this is a bool + + // language + // we might be able to extract this from the contentMap field + + status.ActivityStreamsType = gtsmodel.ActivityStreamsObject(statusable.GetTypeName()) return status, nil } - -// // id of the status in the database -// ID string `pg:"type:uuid,default:gen_random_uuid(),pk,notnull"` -// // uri at which this status is reachable -// URI string `pg:",unique"` -// // web url for viewing this status -// URL string `pg:",unique"` -// // the html-formatted content of this status -// Content string -// // Database IDs of any media attachments associated with this status -// Attachments []string `pg:",array"` -// // Database IDs of any tags used in this status -// Tags []string `pg:",array"` -// // Database IDs of any accounts mentioned in this status -// Mentions []string `pg:",array"` -// // Database IDs of any emojis used in this status -// Emojis []string `pg:",array"` -// // when was this status created? -// CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"` -// // when was this status updated? -// UpdatedAt time.Time `pg:"type:timestamp,notnull,default:now()"` -// // is this status from a local account? -// Local bool -// // which account posted this status? -// AccountID string -// // id of the status this status is a reply to -// InReplyToID string -// // id of the account that this status replies to -// InReplyToAccountID string -// // id of the status this status is a boost of -// BoostOfID string -// // cw string for this status -// ContentWarning string -// // visibility entry for this status -// Visibility Visibility `pg:",notnull"` -// // mark the status as sensitive? -// Sensitive bool -// // what language is this status written in? -// Language string -// // Which application was used to create this status? -// CreatedWithApplicationID string -// // advanced visibility for this status -// VisibilityAdvanced *VisibilityAdvanced -// // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types -// // Will probably almost always be Note but who knows!. -// ActivityStreamsType ActivityStreamsObject -// // Original text of the status without formatting -// Text string - -// // Mentions created in this status -// GTSMentions []*Mention `pg:"-"` -// // Hashtags used in this status -// GTSTags []*Tag `pg:"-"` -// // Emojis used in this status -// GTSEmojis []*Emoji `pg:"-"` -// // MediaAttachments used in this status -// GTSMediaAttachments []*MediaAttachment `pg:"-"` -// // Status being replied to -// GTSReplyToStatus *Status `pg:"-"` -// // Account being replied to -// GTSReplyToAccount *Account `pg:"-"` -// // Status being boosted -// GTSBoostedStatus *Status `pg:"-"` -// // Account of the boosted status -// GTSBoostedAccount *Account `pg:"-"` diff --git a/internal/util/regexes.go b/internal/util/regexes.go index 8bdcf8618..adab8c87f 100644 --- a/internal/util/regexes.go +++ b/internal/util/regexes.go @@ -35,8 +35,10 @@ const ( ) var ( - mentionNameRegexString = `@([a-zA-Z0-9_]+)(?:@([a-zA-Z0-9_\-\.]+)?)` - mentionNameRegex = regexp.MustCompile(fmt.Sprintf(`^%s$`, mentionNameRegexString)) + mentionNameRegexString = `^@([a-zA-Z0-9_]+)(?:@([a-zA-Z0-9_\-\.]+)?)$` + // mention name regex captures the username and domain part from a mention string + // such as @whatever_user@example.org, returning whatever_user and example.org (without the @ symbols) + mentionNameRegex = regexp.MustCompile(mentionNameRegexString) // mention regex can be played around with here: https://regex101.com/r/qwM9D3/1 mentionFinderRegexString = `(?: |^|\W)(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?: |\n)` diff --git a/internal/util/statustools.go b/internal/util/statustools.go index 0a85dda41..2c74749e5 100644 --- a/internal/util/statustools.go +++ b/internal/util/statustools.go @@ -62,23 +62,18 @@ func DeriveEmojisFromStatus(status string) []string { return lower(unique(emojis)) } -// ExtractMentionParts extracts the username @test_user and the domain @example.org +// ExtractMentionParts extracts the username test_user and the domain example.org // from a mention string like @test_user@example.org. // -// If no domain is provided, it will return just the username part. -// // If nothing is matched, it will return an error. func ExtractMentionParts(mention string) (username, domain string, err error) { matches := mentionNameRegex.FindStringSubmatch(mention) - if matches == nil { + if matches == nil || len(matches) != 3 { err = fmt.Errorf("could't match mention %s", mention) return } - fmt.Println(matches) username = matches[1] - if len(matches) == 2 { - domain = matches[2] - } + domain = matches[2] return }