[feature] Update attachment format, receive + send focalPoint prop + use it on the frontend

This commit is contained in:
tobi 2025-04-24 17:58:20 +02:00
commit b841e9e9b1
19 changed files with 595 additions and 81 deletions

View file

@ -32,6 +32,7 @@ import (
"codeberg.org/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@ -634,36 +635,37 @@ func ExtractContent(i WithContent) gtsmodel.Content {
return content
}
// ExtractAttachments attempts to extract barebones MediaAttachment objects from given AS interface type.
func ExtractAttachments(i WithAttachment) ([]*gtsmodel.MediaAttachment, error) {
// ExtractAttachments attempts to extract barebones
// MediaAttachment objects from given AS interface type.
func ExtractAttachments(i WithAttachment) []*gtsmodel.MediaAttachment {
attachmentProp := i.GetActivityStreamsAttachment()
if attachmentProp == nil {
return nil, nil
return nil
}
var errs gtserror.MultiError
attachments := make([]*gtsmodel.MediaAttachment, 0, attachmentProp.Len())
for iter := attachmentProp.Begin(); iter != attachmentProp.End(); iter = iter.Next() {
t := iter.GetType()
if t == nil {
errs.Appendf("nil attachment type")
continue
}
attachmentable, ok := t.(Attachmentable)
attachmentable, ok := ToAttachmentable(t)
if !ok {
errs.Appendf("incorrect attachment type: %T", t)
log.Debugf(nil, "could not cast %T to Attachmentable", t)
continue
}
attachment, err := ExtractAttachment(attachmentable)
if err != nil {
errs.Appendf("error extracting attachment: %w", err)
log.Debugf(nil, "error extracting attachment: %v", err)
continue
}
attachments = append(attachments, attachment)
}
return attachments, errs.Combine()
return attachments
}
// ExtractAttachment extracts a minimal gtsmodel.Attachment
@ -681,7 +683,10 @@ func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
RemoteURL: remoteURL.String(),
Description: ExtractDescription(i),
Blurhash: ExtractBlurhash(i),
Processing: gtsmodel.ProcessingStatusReceived,
FileMeta: gtsmodel.FileMeta{
Focus: ExtractFocus(i),
},
Processing: gtsmodel.ProcessingStatusReceived,
}, nil
}
@ -708,6 +713,50 @@ func ExtractBlurhash(i WithBlurhash) string {
return blurhashProp.Get()
}
// ExtractFocus parses a gtsmodel.Focus from the given Attachmentable's
// `focalPoint` property, if Attachmentable can have `focalPoint`, and
// `focalPoint` is set to a valid pair of floats. Otherwise, returns a
// zero gtsmodel.Focus (ie., focus in the centre of the image).
func ExtractFocus(attachmentable Attachmentable) gtsmodel.Focus {
focus := gtsmodel.Focus{}
withFocalPoint, ok := attachmentable.(WithFocalPoint)
if !ok {
return focus
}
focalPointProp := withFocalPoint.GetTootFocalPoint()
if focalPointProp == nil || focalPointProp.Len() != 2 {
return focus
}
xProp := focalPointProp.At(0)
if !xProp.IsXMLSchemaFloat() {
return focus
}
yProp := focalPointProp.At(1)
if !yProp.IsXMLSchemaFloat() {
return focus
}
x := xProp.Get()
if x < -1 || x > 1 {
return focus
}
y := yProp.Get()
if y < -1 || y > 1 {
return focus
}
// Looks good.
focus.X = float32(x)
focus.Y = float32(y)
return focus
}
// ExtractHashtags extracts a slice of minimal gtsmodel.Tags
// from a WithTag. If an entry in the WithTag is not a hashtag,
// or has a name that cannot be normalized, it will be ignored.

View file

@ -0,0 +1,108 @@
package ap_test
import (
"context"
"encoding/json"
"fmt"
"testing"
"codeberg.org/superseriousbusiness/activity/streams"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/ap"
)
type ExtractFocusTestSuite struct {
APTestSuite
}
func (suite *ExtractFocusTestSuite) TestExtractFocus() {
ctx := context.Background()
type test struct {
data string
expectX float32
expectY float32
}
for _, test := range []test{
{
// Fine.
data: "-0.5, 0.5",
expectX: -0.5,
expectY: 0.5,
},
{
// Also fine.
data: "1, 1",
expectX: 1,
expectY: 1,
},
{
// Out of range.
data: "1.5, 1",
expectX: 0,
expectY: 0,
},
{
// Too many points.
data: "1, 1, 0",
expectX: 0,
expectY: 0,
},
{
// Not enough points.
data: "1",
expectX: 0,
expectY: 0,
},
} {
// Wrap provided test.data
// in a minimal Attachmentable.
const fmts = `{
"@context": [
"https://www.w3.org/ns/activitystreams",
{
"focalPoint": {
"@container": "@list",
"@id": "toot:focalPoint"
},
"toot": "http://joinmastodon.org/ns#"
}
],
"focalPoint": [ %s ],
"type": "Image"
}`
// Unmarshal test data.
data := fmt.Sprintf(fmts, test.data)
m := make(map[string]any)
if err := json.Unmarshal([]byte(data), &m); err != nil {
suite.FailNow(err.Error())
}
// Convert to type.
t, err := streams.ToType(ctx, m)
if err != nil {
suite.FailNow(err.Error())
}
// Convert to attachmentable.
attachmentable, ok := t.(ap.Attachmentable)
if !ok {
suite.FailNow("", "%T was not Attachmentable", t)
}
// Check extracted focus.
focus := ap.ExtractFocus(attachmentable)
if focus.X != test.expectX || focus.Y != test.expectY {
suite.Fail("",
"expected x=%.2f y=%.2f got x=%.2f y=%.2f",
test.expectX, test.expectY, focus.X, focus.Y,
)
}
}
}
func TestExtractFocusTestSuite(t *testing.T) {
suite.Run(t, new(ExtractFocusTestSuite))
}

View file

@ -165,6 +165,29 @@ func ToApprovable(t vocab.Type) (Approvable, bool) {
return approvable, true
}
// IsAttachmentable returns whether AS vocab type name
// is something that can be cast to Attachmentable.
func IsAttachmentable(typeName string) bool {
switch typeName {
case ObjectAudio,
ObjectDocument,
ObjectImage,
ObjectVideo:
return true
default:
return false
}
}
// ToAttachmentable safely tries to cast vocab.Type as Attachmentable.
func ToAttachmentable(t vocab.Type) (Attachmentable, bool) {
attachmentable, ok := t.(Attachmentable)
if !ok || !IsAttachmentable(t.GetTypeName()) {
return nil, false
}
return attachmentable, true
}
// Activityable represents the minimum activitypub interface for representing an 'activity'.
// (see: IsActivityable() for types implementing this, though you MUST make sure to check
// the typeName as this bare interface may be implementable by non-Activityable types).
@ -628,9 +651,11 @@ type WithBlurhash interface {
SetTootBlurhash(vocab.TootBlurhashProperty)
}
// type withFocalPoint interface {
// // TODO
// }
// WithFocalPoint represents an object with TootFocalPointProperty.
type WithFocalPoint interface {
GetTootFocalPoint() vocab.TootFocalPointProperty
SetTootFocalPoint(vocab.TootFocalPointProperty)
}
// WithHref represents an activity with ActivityStreamsHrefProperty
type WithHref interface {

View file

@ -560,6 +560,70 @@ func SetApprovedBy(with WithApprovedBy, approvedBy *url.URL) {
abProp.Set(approvedBy)
}
// GetMediaType returns the string contained in
// the MediaType property of 'with', if set.
func GetMediaType(with WithMediaType) string {
mtProp := with.GetActivityStreamsMediaType()
if mtProp == nil || !mtProp.IsRFCRfc2045() {
return ""
}
return mtProp.Get()
}
// SetMediaType sets the given string
// on the MediaType property of 'with'.
func SetMediaType(with WithMediaType, mediaType string) {
mtProp := with.GetActivityStreamsMediaType()
if mtProp == nil {
mtProp = streams.NewActivityStreamsMediaTypeProperty()
with.SetActivityStreamsMediaType(mtProp)
}
mtProp.Set(mediaType)
}
// AppendName appends the given name
// vals to the Name property of 'with'.
func AppendName(with WithName, name ...string) {
if len(name) == 0 {
return
}
nameProp := with.GetActivityStreamsName()
if nameProp == nil {
nameProp = streams.NewActivityStreamsNameProperty()
with.SetActivityStreamsName(nameProp)
}
for _, name := range name {
nameProp.AppendXMLSchemaString(name)
}
}
// AppendSummary appends the given summary
// vals to the Summary property of 'with'.
func AppendSummary(with WithSummary, summary ...string) {
if len(summary) == 0 {
return
}
summaryProp := with.GetActivityStreamsSummary()
if summaryProp == nil {
summaryProp = streams.NewActivityStreamsSummaryProperty()
with.SetActivityStreamsSummary(summaryProp)
}
for _, summary := range summary {
summaryProp.AppendXMLSchemaString(summary)
}
}
// SetBlurhash sets the given string
// on the Blurhash property of 'with'.
func SetBlurhash(with WithBlurhash, mediaType string) {
bProp := with.GetTootBlurhash()
if bProp == nil {
bProp = streams.NewTootBlurhashProperty()
with.SetTootBlurhash(bProp)
}
bProp.Set(mediaType)
}
// extractIRIs extracts just the AP IRIs from an iterable
// property that may contain types (with IRIs) or just IRIs.
//

View file

@ -193,8 +193,8 @@ func (suite *StatusBoostTestSuite) TestPostBoost() {
"id": "01F8MH6NEM8D7527KZAECTCR76",
"meta": {
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
},
"original": {
"aspect": 1.9047619,

View file

@ -950,6 +950,8 @@ func (d *Dereferencer) fetchStatusAttachments(
RemoteURL: &placeholder.RemoteURL,
Description: &placeholder.Description,
Blurhash: &placeholder.Blurhash,
FocusX: &placeholder.FileMeta.Focus.X,
FocusY: &placeholder.FileMeta.Focus.Y,
},
)
if err != nil {

View file

@ -142,7 +142,7 @@ func (f *Filter) StatusableOK(
}
// HEURISTIC 6: Are there any media attachments?
attachments, _ := ap.ExtractAttachments(statusable)
attachments := ap.ExtractAttachments(statusable)
hasAttachments := len(attachments) != 0
if hasAttachments {
err := errors.New("status has attachment(s)")

View file

@ -136,6 +136,7 @@ func LoadTemplates(engine *gin.Engine) error {
var funcMap = template.FuncMap{
"add": add,
"acctInstance": acctInstance,
"objectPosition": objectPosition,
"demojify": demojify,
"deref": deref,
"emojify": emojify,
@ -365,3 +366,12 @@ func deref(i any) any {
return vOf.Elem()
}
// objectPosition formats the given focus coordinates to a
// string suitable for use as a css object-position value.
func objectPosition(focusX float32, focusY float32) string {
const fmts = "%.2f"
xPos := ((focusX / 2) + .5) * 100
yPos := ((focusY / -2) + .5) * 100
return fmt.Sprintf(fmts, xPos) + "%" + " " + fmt.Sprintf(fmts, yPos) + "%"
}

View file

@ -288,10 +288,7 @@ func (c *Converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab
// status.Attachments
//
// Media attachments for later dereferencing.
status.Attachments, err = ap.ExtractAttachments(statusable)
if err != nil {
log.Warnf(ctx, "error(s) extracting attachments for %s: %v", uri, err)
}
status.Attachments = ap.ExtractAttachments(statusable)
// status.Poll
//

View file

@ -678,22 +678,9 @@ func (c *Converter) StatusToAS(ctx context.Context, s *gtsmodel.Status) (ap.Stat
status.SetActivityStreamsContent(contentProp)
// attachments
attachmentProp := streams.NewActivityStreamsAttachmentProperty()
attachments := s.Attachments
if len(s.AttachmentIDs) != len(attachments) {
attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
if err != nil {
return nil, gtserror.Newf("error getting attachments from database: %w", err)
}
if err := c.attachAttachments(ctx, s, status); err != nil {
return nil, gtserror.Newf("error attaching attachments: %w", err)
}
for _, a := range attachments {
doc, err := c.AttachmentToAS(ctx, a)
if err != nil {
return nil, gtserror.Newf("error converting attachment: %w", err)
}
attachmentProp.AppendActivityStreamsDocument(doc)
}
status.SetActivityStreamsAttachment(attachmentProp)
// replies
repliesCollection, err := c.StatusToASRepliesCollection(ctx, s, false)
@ -1130,39 +1117,94 @@ func (c *Converter) EmojiToAS(ctx context.Context, e *gtsmodel.Emoji) (vocab.Too
return emoji, nil
}
// AttachmentToAS converts a gts model media attachment into an activity streams Attachment, suitable for federation
func (c *Converter) AttachmentToAS(ctx context.Context, a *gtsmodel.MediaAttachment) (vocab.ActivityStreamsDocument, error) {
// type -- Document
doc := streams.NewActivityStreamsDocument()
// mediaType aka mime content type
mediaTypeProp := streams.NewActivityStreamsMediaTypeProperty()
mediaTypeProp.Set(a.File.ContentType)
doc.SetActivityStreamsMediaType(mediaTypeProp)
// url -- for the original image not the thumbnail
urlProp := streams.NewActivityStreamsUrlProperty()
imageURL, err := url.Parse(a.URL)
if err != nil {
return nil, fmt.Errorf("AttachmentToAS: error parsing uri %s: %s", a.URL, err)
// attachAttachments converts the attachments on the given status
// into Attachmentables, and appends them to the given Statusable.
func (c *Converter) attachAttachments(
ctx context.Context,
s *gtsmodel.Status,
statusable ap.Statusable,
) error {
// Ensure status attachments populated.
if len(s.AttachmentIDs) != len(s.Attachments) {
var err error
s.Attachments, err = c.state.DB.GetAttachmentsByIDs(ctx, s.AttachmentIDs)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return gtserror.Newf("db error getting attachments: %w", err)
}
}
urlProp.AppendIRI(imageURL)
doc.SetActivityStreamsUrl(urlProp)
// name -- aka image description
nameProp := streams.NewActivityStreamsNameProperty()
nameProp.AppendXMLSchemaString(a.Description)
doc.SetActivityStreamsName(nameProp)
// Prepare attachment property.
attachmentProp := streams.NewActivityStreamsAttachmentProperty()
defer statusable.SetActivityStreamsAttachment(attachmentProp)
// blurhash
blurProp := streams.NewTootBlurhashProperty()
blurProp.Set(a.Blurhash)
doc.SetTootBlurhash(blurProp)
for _, a := range s.Attachments {
// focalpoint
// TODO
// Use appropriate vocab.Type and
// append function for this attachment.
var (
attachmentable ap.Attachmentable
append func()
)
switch a.Type {
return doc, nil
// png, gif, webp, jpeg, etc.
case gtsmodel.FileTypeImage:
t := streams.NewActivityStreamsImage()
attachmentable = t
append = func() { attachmentProp.AppendActivityStreamsImage(t) }
// mp4, m4a, wmv, webm, etc.
case gtsmodel.FileTypeVideo, gtsmodel.FileTypeGifv:
t := streams.NewActivityStreamsVideo()
attachmentable = t
append = func() { attachmentProp.AppendActivityStreamsVideo(t) }
// mp3, flac, ogg, wma, etc.
case gtsmodel.FileTypeAudio:
t := streams.NewActivityStreamsAudio()
attachmentable = t
append = func() { attachmentProp.AppendActivityStreamsAudio(t) }
// Not sure, fall back to Document.
default:
t := streams.NewActivityStreamsDocument()
attachmentable = t
append = func() { attachmentProp.AppendActivityStreamsDocument(t) }
}
// `mediaType` ie., mime content type.
ap.SetMediaType(attachmentable, a.File.ContentType)
// URL of the media file.
imageURL, err := url.Parse(a.URL)
if err != nil {
return gtserror.Newf("error parsing attachment url: %w", err)
}
ap.AppendURL(attachmentable, imageURL)
// `name` ie., image description
ap.AppendName(attachmentable, a.Description)
// `blurhash`
ap.SetBlurhash(attachmentable, a.Blurhash)
// Set `focalPoint` only if necessary.
if a.FileMeta.Focus.X != 0 && a.FileMeta.Focus.Y != 0 {
if withFocalPoint, ok := attachmentable.(ap.WithFocalPoint); ok {
focalPointProp := streams.NewTootFocalPointProperty()
focalPointProp.AppendXMLSchemaFloat(float64(a.FileMeta.Focus.X))
focalPointProp.AppendXMLSchemaFloat(float64(a.FileMeta.Focus.Y))
withFocalPoint.SetTootFocalPoint(focalPointProp)
}
}
// Done, append
// to Statusable.
append()
}
statusable.SetActivityStreamsAttachment(attachmentProp)
return nil
}
// FaveToAS converts a gts model status fave into an activityStreams LIKE, suitable for federation.

View file

@ -597,6 +597,10 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
"Emoji": "toot:Emoji",
"Hashtag": "as:Hashtag",
"blurhash": "toot:blurhash",
"focalPoint": {
"@container": "@list",
"@id": "toot:focalPoint"
},
"sensitive": "as:sensitive",
"toot": "http://joinmastodon.org/ns#"
}
@ -604,9 +608,13 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() {
"attachment": [
{
"blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
"focalPoint": [
-0.5,
0.5
],
"mediaType": "image/jpeg",
"name": "Black and white image of some 50's style text saying: Welcome On Board",
"type": "Document",
"summary": "Black and white image of some 50's style text saying: Welcome On Board",
"type": "Image",
"url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
}
],
@ -697,6 +705,10 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
"Emoji": "toot:Emoji",
"Hashtag": "as:Hashtag",
"blurhash": "toot:blurhash",
"focalPoint": {
"@container": "@list",
"@id": "toot:focalPoint"
},
"sensitive": "as:sensitive",
"toot": "http://joinmastodon.org/ns#"
}
@ -704,9 +716,13 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() {
"attachment": [
{
"blurhash": "LIIE|gRj00WB-;j[t7j[4nWBj[Rj",
"focalPoint": [
-0.5,
0.5
],
"mediaType": "image/jpeg",
"name": "Black and white image of some 50's style text saying: Welcome On Board",
"type": "Document",
"summary": "Black and white image of some 50's style text saying: Welcome On Board",
"type": "Image",
"url": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"
}
],

View file

@ -553,8 +553,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() {
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -701,8 +701,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendHTMLContentWarning
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -851,8 +851,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendApplicationDeleted
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -1032,8 +1032,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredStatusToFrontend() {
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -1218,8 +1218,8 @@ func (suite *InternalToFrontendTestSuite) TestWarnFilteredBoostToFrontend() {
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",
@ -1955,8 +1955,8 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage()
"aspect": 1.9104477
},
"focus": {
"x": 0,
"y": 0
"x": -0.5,
"y": 0.5
}
},
"description": "Black and white image of some 50's style text saying: Welcome On Board",