[feature] Update attachment format, receive + send focalPoint prop + use it on the frontend (#4052)

* [feature] Update attachment format, receive + send `focalPoint` prop + use it on the frontend

* whoops

* boop

* restore function signature of ExtractAttachments
This commit is contained in:
tobi 2025-04-26 15:03:05 +02:00 committed by GitHub
commit f7323c065a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 617 additions and 72 deletions

View file

@ -634,32 +634,38 @@ func ExtractContent(i WithContent) gtsmodel.Content {
return content
}
// ExtractAttachments attempts to extract barebones MediaAttachment objects from given AS interface type.
// ExtractAttachments attempts to extract barebones
// MediaAttachment objects from given AS interface type.
func ExtractAttachments(i WithAttachment) ([]*gtsmodel.MediaAttachment, error) {
attachmentProp := i.GetActivityStreamsAttachment()
if attachmentProp == nil {
return nil, nil
}
var errs gtserror.MultiError
var (
attachments = make([]*gtsmodel.MediaAttachment, 0, attachmentProp.Len())
errs gtserror.MultiError
)
attachments := make([]*gtsmodel.MediaAttachment, 0, attachmentProp.Len())
for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() {
t := iter.GetType()
if t == nil {
errs.Appendf("nil attachment type")
continue
}
attachmentable, ok := t.(Attachmentable)
attachmentable, ok := ToAttachmentable(t)
if !ok {
errs.Appendf("incorrect attachment type: %T", t)
errs.Appendf("could not cast %T to Attachmentable", t)
continue
}
attachment, err := ExtractAttachment(attachmentable)
if err != nil {
errs.Appendf("error extracting attachment: %w", err)
continue
}
attachments = append(attachments, attachment)
}
@ -681,7 +687,10 @@ func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
RemoteURL: remoteURL.String(),
Description: ExtractDescription(i),
Blurhash: ExtractBlurhash(i),
Processing: gtsmodel.ProcessingStatusReceived,
FileMeta: gtsmodel.FileMeta{
Focus: ExtractFocus(i),
},
Processing: gtsmodel.ProcessingStatusReceived,
}, nil
}
@ -708,6 +717,50 @@ func ExtractBlurhash(i WithBlurhash) string {
return blurhashProp.Get()
}
// ExtractFocus parses a gtsmodel.Focus from the given Attachmentable's
// `focalPoint` property, if Attachmentable can have `focalPoint`, and
// `focalPoint` is set to a valid pair of floats. Otherwise, returns a
// zero gtsmodel.Focus (ie., focus in the centre of the image).
func ExtractFocus(attachmentable Attachmentable) gtsmodel.Focus {
focus := gtsmodel.Focus{}
withFocalPoint, ok := attachmentable.(WithFocalPoint)
if !ok {
return focus
}
focalPointProp := withFocalPoint.GetTootFocalPoint()
if focalPointProp == nil || focalPointProp.Len() != 2 {
return focus
}
xProp := focalPointProp.At(0)
if !xProp.IsXMLSchemaFloat() {
return focus
}
yProp := focalPointProp.At(1)
if !yProp.IsXMLSchemaFloat() {
return focus
}
x := xProp.Get()
if x < -1 || x > 1 {
return focus
}
y := yProp.Get()
if y < -1 || y > 1 {
return focus
}
// Looks good.
focus.X = float32(x)
focus.Y = float32(y)
return focus
}
// ExtractHashtags extracts a slice of minimal gtsmodel.Tags
// from a WithTag. If an entry in the WithTag is not a hashtag,
// or has a name that cannot be normalized, it will be ignored.

View file

@ -0,0 +1,125 @@
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ap_test
import (
"context"
"encoding/json"
"fmt"
"testing"
"code.superseriousbusiness.org/activity/streams"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/ap"
)
type ExtractFocusTestSuite struct {
APTestSuite
}
func (suite *ExtractFocusTestSuite) TestExtractFocus() {
ctx := context.Background()
type test struct {
data string
expectX float32
expectY float32
}
for _, test := range []test{
{
// Fine.
data: "-0.5, 0.5",
expectX: -0.5,
expectY: 0.5,
},
{
// Also fine.
data: "1, 1",
expectX: 1,
expectY: 1,
},
{
// Out of range.
data: "1.5, 1",
expectX: 0,
expectY: 0,
},
{
// Too many points.
data: "1, 1, 0",
expectX: 0,
expectY: 0,
},
{
// Not enough points.
data: "1",
expectX: 0,
expectY: 0,
},
} {
// Wrap provided test.data
// in a minimal Attachmentable.
const fmts = `{
"@context": [
"https://www.w3.org/ns/activitystreams",
{
"focalPoint": {
"@container": "@list",
"@id": "toot:focalPoint"
},
"toot": "http://joinmastodon.org/ns#"
}
],
"focalPoint": [ %s ],
"type": "Image"
}`
// Unmarshal test data.
data := fmt.Sprintf(fmts, test.data)
m := make(map[string]any)
if err := json.Unmarshal([]byte(data), &m); err != nil {
suite.FailNow(err.Error())
}
// Convert to type.
t, err := streams.ToType(ctx, m)
if err != nil {
suite.FailNow(err.Error())
}
// Convert to attachmentable.
attachmentable, ok := t.(ap.Attachmentable)
if !ok {
suite.FailNow("", "%T was not Attachmentable", t)
}
// Check extracted focus.
focus := ap.ExtractFocus(attachmentable)
if focus.X != test.expectX || focus.Y != test.expectY {
suite.Fail("",
"expected x=%.2f y=%.2f got x=%.2f y=%.2f",
test.expectX, test.expectY, focus.X, focus.Y,
)
}
}
}
func TestExtractFocusTestSuite(t *testing.T) {
suite.Run(t, new(ExtractFocusTestSuite))
}

View file

@ -165,6 +165,29 @@ func ToApprovable(t vocab.Type) (Approvable, bool) {
return approvable, true
}
// IsAttachmentable returns whether AS vocab type name
// is something that can be cast to Attachmentable.
func IsAttachmentable(typeName string) bool {
switch typeName {
case ObjectAudio,
ObjectDocument,
ObjectImage,
ObjectVideo:
return true
default:
return false
}
}
// ToAttachmentable safely tries to cast vocab.Type as Attachmentable.
func ToAttachmentable(t vocab.Type) (Attachmentable, bool) {
attachmentable, ok := t.(Attachmentable)
if !ok || !IsAttachmentable(t.GetTypeName()) {
return nil, false
}
return attachmentable, true
}
// Activityable represents the minimum activitypub interface for representing an 'activity'.
// (see: IsActivityable() for types implementing this, though you MUST make sure to check
// the typeName as this bare interface may be implementable by non-Activityable types).
@ -628,9 +651,11 @@ type WithBlurhash interface {
SetTootBlurhash(vocab.TootBlurhashProperty)
}
// type withFocalPoint interface {
// // TODO
// }
// WithFocalPoint represents an object with TootFocalPointProperty.
type WithFocalPoint interface {
GetTootFocalPoint() vocab.TootFocalPointProperty
SetTootFocalPoint(vocab.TootFocalPointProperty)
}
// WithHref represents an activity with ActivityStreamsHrefProperty
type WithHref interface {

View file

@ -560,6 +560,70 @@ func SetApprovedBy(with WithApprovedBy, approvedBy *url.URL) {
abProp.Set(approvedBy)
}
// GetMediaType returns the string contained in
// the MediaType property of 'with', if set.
func GetMediaType(with WithMediaType) string {
mtProp := with.GetActivityStreamsMediaType()
if mtProp == nil || !mtProp.IsRFCRfc2045() {
return ""
}
return mtProp.Get()
}
// SetMediaType sets the given string
// on the MediaType property of 'with'.
func SetMediaType(with WithMediaType, mediaType string) {
mtProp := with.GetActivityStreamsMediaType()
if mtProp == nil {
mtProp = streams.NewActivityStreamsMediaTypeProperty()
with.SetActivityStreamsMediaType(mtProp)
}
mtProp.Set(mediaType)
}
// AppendName appends the given name
// vals to the Name property of 'with'.
func AppendName(with WithName, name ...string) {
if len(name) == 0 {
return
}
nameProp := with.GetActivityStreamsName()
if nameProp == nil {
nameProp = streams.NewActivityStreamsNameProperty()
with.SetActivityStreamsName(nameProp)
}
for _, name := range name {
nameProp.AppendXMLSchemaString(name)
}
}
// AppendSummary appends the given summary
// vals to the Summary property of 'with'.
func AppendSummary(with WithSummary, summary ...string) {
if len(summary) == 0 {
return
}
summaryProp := with.GetActivityStreamsSummary()
if summaryProp == nil {
summaryProp = streams.NewActivityStreamsSummaryProperty()
with.SetActivityStreamsSummary(summaryProp)
}
for _, summary := range summary {
summaryProp.AppendXMLSchemaString(summary)
}
}
// SetBlurhash sets the given string
// on the Blurhash property of 'with'.
func SetBlurhash(with WithBlurhash, mediaType string) {
bProp := with.GetTootBlurhash()
if bProp == nil {
bProp = streams.NewTootBlurhashProperty()
with.SetTootBlurhash(bProp)
}
bProp.Set(mediaType)
}
// extractIRIs extracts just the AP IRIs from an iterable
// property that may contain types (with IRIs) or just IRIs.
//

View file

@ -193,8 +193,8 @@ func (suite *StatusBoostTestSuite) TestPostBoost() {
"id": "01F8MH6NEM8D7527KZAECTCR76",
"meta": {
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
},
"original": {
"aspect": 1.9047619,

View file

@ -950,6 +950,8 @@ func (d *Dereferencer) fetchStatusAttachments(
RemoteURL: &placeholder.RemoteURL,
Description: &placeholder.Description,
Blurhash: &placeholder.Blurhash,
FocusX: &placeholder.FileMeta.Focus.X,
FocusY: &placeholder.FileMeta.Focus.Y,
},
)
if err != nil {

View file

@ -142,7 +142,14 @@ func (f *Filter) StatusableOK(
}
// HEURISTIC 6: Are there any media attachments?
attachments, _ := ap.ExtractAttachments(statusable)
attachments, err := ap.ExtractAttachments(statusable)
if err != nil {
log.Warnf(ctx,
"error(s) extracting attachments for %s: %v",
ap.GetJSONLDId(statusable), err,
)
}
hasAttachments := len(attachments) != 0
if hasAttachments {
err := errors.New("status has attachment(s)")

View file

@ -136,6 +136,7 @@ func LoadTemplates(engine *gin.Engine) error {
var funcMap = template.FuncMap{
"add": add,
"acctInstance": acctInstance,
"objectPosition": objectPosition,
"demojify": demojify,
"deref": deref,
"emojify": emojify,
@ -365,3 +366,12 @@ func deref(i any) any {
return vOf.Elem()
}
// objectPosition formats the given focus coordinates to a
// string suitable for use as a css object-position value.
func objectPosition(focusX float32, focusY float32) string {
const fmts = "%.2f"
xPos := ((focusX / 2) + .5) * 100
yPos := ((focusY / -2) + .5) * 100
return fmt.Sprintf(fmts, xPos) + "%" + " " + fmt.Sprintf(fmts, yPos) + "%"
}

View file

@ -678,22 +678,9 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
status.SetActivityStreamsContent(contentProp)
// attachments
attachmentProp := streams.NewActivityStreamsAttachmentProperty()
attachments := s.Attachments
if len(s.AttachmentIDs) != len(attachments) {
attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
if err != nil {
return nil, gtserror.Newf("error getting attachments from database: %w", err)
}
if err := c.attachAttachments(ctx, s, status); err != nil {
return nil, gtserror.Newf("error attaching attachments: %w", err)
}
for _, a := range attachments {
doc, err := c.AttachmentToAS(ctx, a)
if err != nil {
return nil, gtserror.Newf("error converting attachment: %w", err)
}
attachmentProp.AppendActivityStreamsDocument(doc)
}
status.SetActivityStreamsAttachment(attachmentProp)
// replies
repliesCollection, err := c.StatusToASRepliesCollection(ctx, s, false)
@ -1130,39 +1117,94 @@ func (c *Converter) EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.Too
return emoji, nil
}
// AttachmentToAS converts a gts model media attachment into an activity streams Attachment, suitable for federation
func (c *Converter) AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachment) (vocab.ActivityStreamsDocument, error) {
// type -- Document
doc := streams.NewActivityStreamsDocument()
// mediaType aka mime content type
mediaTypeProp := streams.NewActivityStreamsMediaTypeProperty()
mediaTypeProp.Set(a.File.ContentType)
doc.SetActivityStreamsMediaType(mediaTypeProp)
// url -- for the original image not the thumbnail
urlProp := streams.NewActivityStreamsUrlProperty()
imageURL, err := url.Parse(a.URL)
if err != nil {
return nil, fmt.Errorf("AttachmentToAS: error parsing uri %s: %s", a.URL, err)
// attachAttachments converts the attachments on the given status
// into Attachmentables, and appends them to the given Statusable.
func (c *Converter) attachAttachments(
ctx context.Context,
s *gtsmodel.Status,
statusable ap.Statusable,
) error {
// Ensure status attachments populated.
if len(s.AttachmentIDs) != len(s.Attachments) {
var err error
s.Attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return gtserror.Newf("db error getting attachments: %w", err)
}
}
urlProp.AppendIRI(imageURL)
doc.SetActivityStreamsUrl(urlProp)
// name -- aka image description
nameProp := streams.NewActivityStreamsNameProperty()
nameProp.AppendXMLSchemaString(a.Description)
doc.SetActivityStreamsName(nameProp)
// Prepare attachment property.
attachmentProp := streams.NewActivityStreamsAttachmentProperty()
defer statusable.SetActivityStreamsAttachment(attachmentProp)
// blurhash
blurProp := streams.NewTootBlurhashProperty()
blurProp.Set(a.Blurhash)
doc.SetTootBlurhash(blurProp)
for _, a := range s.Attachments {
// focalpoint
// TODO
// Use appropriate vocab.Type and
// append function for this attachment.
var (
attachmentable ap.Attachmentable
append func()
)
switch a.Type {
return doc, nil
// png, gif, webp, jpeg, etc.
case gtsmodel.FileTypeImage:
t := streams.NewActivityStreamsImage()
attachmentable = t
append = func() { attachmentProp.AppendActivityStreamsImage(t) }
// mp4, m4a, wmv, webm, etc.
case gtsmodel.FileTypeVideo, gtsmodel.FileTypeGifv:
t := streams.NewActivityStreamsVideo()
attachmentable = t
append = func() { attachmentProp.AppendActivityStreamsVideo(t) }
// mp3, flac, ogg, wma, etc.
case gtsmodel.FileTypeAudio:
t := streams.NewActivityStreamsAudio()
attachmentable = t
append = func() { attachmentProp.AppendActivityStreamsAudio(t) }
// Not sure, fall back to Document.
default:
t := streams.NewActivityStreamsDocument()
attachmentable = t
append = func() { attachmentProp.AppendActivityStreamsDocument(t) }
}
// `mediaType` ie., mime content type.
ap.SetMediaType(attachmentable, a.File.ContentType)
// URL of the media file.
imageURL, err := url.Parse(a.URL)
if err != nil {
return gtserror.Newf("error parsing attachment url: %w", err)
}
ap.AppendURL(attachmentable, imageURL)
// `summary` ie., media description / alt text
ap.AppendSummary(attachmentable, a.Description)
// `blurhash`
ap.SetBlurhash(attachmentable, a.Blurhash)
// Set `focalPoint` only if necessary.
if a.FileMeta.Focus.X != 0 && a.FileMeta.Focus.Y != 0 {
if withFocalPoint, ok := attachmentable.(ap.WithFocalPoint); ok {
focalPointProp := streams.NewTootFocalPointProperty()
focalPointProp.AppendXMLSchemaFloat(float64(a.FileMeta.Focus.X))
focalPointProp.AppendXMLSchemaFloat(float64(a.FileMeta.Focus.Y))
withFocalPoint.SetTootFocalPoint(focalPointProp)
}
}
// Done, append
// to Statusable.
append()
}
statusable.SetActivityStreamsAttachment(attachmentProp)
return nil
}
// FaveToAS converts a gts model status fave into an activityStreams LIKE, suitable for federation.

View file

@ -597,6 +597,10 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
"Emoji": "toot:Emoji",
"Hashtag": "as:Hashtag",
"blurhash": "toot:blurhash",
"focalPoint": {
"@container": "@list",
"@id": "toot:focalPoint"
},
"sensitive": "as:sensitive",
"toot": "http://joinmastodon.org/ns#"
}
@ -604,9 +608,13 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
"attachment": [
{
"blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
"focalPoint": [
-0.5,
0.5
],
"mediaType": "image/jpeg",
"name": "Black and white image of some 50's style text saying: Welcome On Board",
"type": "Document",
"summary": "Black and white image of some 50's style text saying: Welcome On Board",
"type": "Image",
"url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
}
],
@ -697,6 +705,10 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
"Emoji": "toot:Emoji",
"Hashtag": "as:Hashtag",
"blurhash": "toot:blurhash",
"focalPoint": {
"@container": "@list",
"@id": "toot:focalPoint"
},
"sensitive": "as:sensitive",
"toot": "http://joinmastodon.org/ns#"
}
@ -704,9 +716,13 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
"attachment": [
{
"blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
"focalPoint": [
-0.5,
0.5
],
"mediaType": "image/jpeg",
"name": "Black and white image of some 50's style text saying: Welcome On Board",
"type": "Document",
"summary": "Black and white image of some 50's style text saying: Welcome On Board",
"type": "Image",
"url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
}
],

View file

@ -553,8 +553,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -701,8 +701,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendHTMLContentWarning
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -851,8 +851,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -1032,8 +1032,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() {
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -1218,8 +1218,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() {
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -1955,8 +1955,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",