mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-28 22:42:26 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] add paging support to rss feed endpoint, and support JSON / atom feed types (#4442)

Browse source 
originally based on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4396

hope this is okay https://codeberg.org/zordsdavini !

closes https://codeberg.org/superseriousbusiness/gotosocial/issues/4411
closes https://codeberg.org/superseriousbusiness/gotosocial/issues/3407

Co-authored-by: Arnas Udovic <zordsdavini@gmail.com>
Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4442
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2025-09-18 16:33:23 +02:00 • committed by kim
commit 6607e1c944
13 changed files with 447 additions and 182 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/api/util/mime.go
View file

@ -26,6 +26,8 @@ const (
appXMLText = `text/xml` // AppXML is only *recommended* in RFC7303
AppXMLXRD = `application/xrd+xml`
AppRSSXML = `application/rss+xml`
AppAtomXML = `application/atom+xml`
AppFeedJSON = `application/feed+json`
AppActivityJSON = `application/activity+json`
appActivityLDJSON = `application/ld+json` // without profile
AppActivityLDJSON = appActivityLDJSON + `; profile="https://www.w3.org/ns/activitystreams"`

13
internal/api/util/response.go
View file

@ -71,6 +71,18 @@ func JSONType(c *gin.Context, code int, contentType string, data any) {
EncodeJSONResponse(c.Writer, c.Request, code, contentType, data)
}
// XML calls EncodeJSONResponse() using gin.Context{}, with content-type = AppXML,
// This function handles the case of XML unmarshal errors and pools read buffers.
func XML(c *gin.Context, code int, data any) {
EncodeXMLResponse(c.Writer, c.Request, code, AppXML, data)
}
// XML calls EncodeXMLResponse() using gin.Context{}, with given content-type.
// This function handles the case of XML unmarshal errors and pools read buffers.
func XMLType(c *gin.Context, code int, contentType string, data any) {
EncodeXMLResponse(c.Writer, c.Request, code, contentType, data)
}
// Data calls WriteResponseBytes() using gin.Context{}, with given content-type.
func Data(c *gin.Context, code int, contentType string, data []byte) {
WriteResponseBytes(c.Writer, c.Request, code, contentType, data)
@ -230,6 +242,7 @@ func EncodeCSVResponse(
// Write all the records to the buffer.
if err := csvWriter.WriteAll(records); err == nil {
// Respond with the now-known
// size byte slice within buf.
WriteResponseBytes(rw, r,

2
internal/db/account.go
View file

@ -121,7 +121,7 @@ type Account interface {
// returning statuses that should be visible via the web view of a *LOCAL* account.
//
// In the case of no statuses, this function will return db.ErrNoEntries.
GetAccountWebStatuses(ctx context.Context, account *gtsmodel.Account, mediaOnly bool, limit int, maxID string) ([]*gtsmodel.Status, error)
GetAccountWebStatuses(ctx context.Context, account *gtsmodel.Account, page *paging.Page, mediaOnly bool) ([]*gtsmodel.Status, error)
// GetInstanceAccount returns the instance account for the given domain.
// If domain is empty, this instance account will be returned.

109
internal/db/bundb/account.go
View file

@ -900,7 +900,7 @@ func (a *accountDB) GetAccountFaves(ctx context.Context, accountID string) ([]*g
return *faves, nil
}
func qMediaOnly(q *bun.SelectQuery) *bun.SelectQuery {
func selectOnlyWithMedia(q *bun.SelectQuery) *bun.SelectQuery {
// Attachments are stored as a json object; this
// implementation differs between SQLite and Postgres,
// so we have to be thorough to cover all eventualities
@ -908,14 +908,14 @@ func qMediaOnly(q *bun.SelectQuery) *bun.SelectQuery {
switch d := q.Dialect().Name(); d {
case dialect.PG:
return q.
Where("? IS NOT NULL", bun.Ident("status.attachments")).
Where("? != '{}'", bun.Ident("status.attachments"))
Where("? IS NOT NULL", bun.Ident("attachments")).
Where("? != '{}'", bun.Ident("attachments"))
case dialect.SQLite:
return q.
Where("? IS NOT NULL", bun.Ident("status.attachments")).
Where("? != 'null'", bun.Ident("status.attachments")).
Where("? != '[]'", bun.Ident("status.attachments"))
Where("? IS NOT NULL", bun.Ident("attachments")).
Where("? != 'null'", bun.Ident("attachments")).
Where("? != '[]'", bun.Ident("attachments"))
default:
panic("dialect " + d.String() + " was neither pg nor sqlite")
@ -963,9 +963,9 @@ func (a *accountDB) GetAccountStatuses(ctx context.Context, accountID string, li
q = q.Where("? IS NULL", bun.Ident("status.boost_of_id"))
}
// Respect media-only preference.
if mediaOnly {
q = qMediaOnly(q)
// Respect mediaOnly pref.
q = selectOnlyWithMedia(q)
}
if publicOnly {
@ -1041,12 +1041,16 @@ func (a *accountDB) GetAccountPinnedStatuses(ctx context.Context, accountID stri
return a.state.DB.GetStatusesByIDs(ctx, statusIDs)
}
var webStatusVisibilities = bun.In([]gtsmodel.Visibility{
gtsmodel.VisibilityPublic,
gtsmodel.VisibilityUnlocked,
})
func (a *accountDB) GetAccountWebStatuses(
ctx context.Context,
account *gtsmodel.Account,
page *paging.Page,
mediaOnly bool,
limit int,
maxID string,
) ([]*gtsmodel.Status, error) {
if account.Username == config.GetHost() {
// Instance account
@ -1071,74 +1075,35 @@ func (a *accountDB) GetAccountWebStatuses(
return nil, nil
}
// Ensure reasonable
if limit < 0 {
limit = 0
}
return loadStatusTimelinePage(ctx, a.db, a.state,
// Make educated guess for slice size
statusIDs := make([]string, 0, limit)
// Paging
// params.
page,
q := a.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
// Select only IDs from table
Column("status.id").
Where("? = ?", bun.Ident("status.account_id"), account.ID)
// The actual meat of the account web statuses query.
func(q *bun.SelectQuery) (*bun.SelectQuery, error) {
q = q.Where("? = ?", bun.Ident("account_id"), account.ID)
// Select statuses according to
// account's web visibility prefs.
if publicOnly {
// Only Public statuses.
q = q.Where("? = ?", bun.Ident("status.visibility"), gtsmodel.VisibilityPublic)
} else {
// Public or Unlocked.
visis := []gtsmodel.Visibility{
gtsmodel.VisibilityPublic,
gtsmodel.VisibilityUnlocked,
}
q = q.Where("? IN (?)", bun.Ident("status.visibility"), bun.In(visis))
}
if publicOnly {
q = q.Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityPublic)
} else {
q = q.Where("? IN (?)", bun.Ident("visibility"), webStatusVisibilities)
}
// Don't show replies, boosts, or
// local-only statuses on the web view.
q = q.
Where("? IS NULL", bun.Ident("status.in_reply_to_uri")).
Where("? IS NULL", bun.Ident("status.boost_of_id")).
Where("? = ?", bun.Ident("status.federated"), true)
// Don't show replies, boosts, or local-only in web view.
q = q.Where("? IS NULL", bun.Ident("in_reply_to_uri")).
Where("? IS NULL", bun.Ident("boost_of_id")).
Where("? = ?", bun.Ident("federated"), true)
// Respect media-only preference.
if mediaOnly {
q = qMediaOnly(q)
}
if mediaOnly {
// Respect mediaOnly pref.
q = selectOnlyWithMedia(q)
}
// Return only statuses LOWER (ie., older) than maxID
if maxID == "" {
maxID = id.Highest
}
q = q.Where("? < ?", bun.Ident("status.id"), maxID)
if limit > 0 {
// limit amount of statuses returned
q = q.Limit(limit)
}
if limit > 0 {
// limit amount of statuses returned
q = q.Limit(limit)
}
q = q.Order("status.id DESC")
if err := q.Scan(ctx, &statusIDs); err != nil {
return nil, err
}
if len(statusIDs) == 0 {
return nil, db.ErrNoEntries
}
return a.state.DB.GetStatusesByIDs(ctx, statusIDs)
return q, nil
},
)
}
func (a *accountDB) GetAccountSettings(

2
internal/db/bundb/account_test.go
View file

@ -49,7 +49,7 @@ func (suite *AccountTestSuite) TestGetAccountStatuses() {
}
func (suite *AccountTestSuite) TestGetAccountWebStatusesMediaOnly() {
statuses, err := suite.db.GetAccountWebStatuses(suite.T().Context(), suite.testAccounts["local_account_3"], true, 20, "")
statuses, err := suite.db.GetAccountWebStatuses(suite.T().Context(), suite.testAccounts["local_account_3"], &paging.Page{Limit: 20}, true)
suite.NoError(err)
suite.Len(statuses, 2)
}

73
internal/processing/account/rss.go
View file

@ -20,21 +20,19 @@ package account
import (
"context"
"errors"
"fmt"
"time"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/db"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gorilla/feeds"
)
const (
rssFeedLength = 20
)
var never time.Time
type GetRSSFeed func() (string, gtserror.WithCode)
type GetRSSFeed func() (*feeds.Feed, gtserror.WithCode)
// GetRSSFeedForUsername returns a function to return the RSS feed of a local account
// with the given username, and the last-modified time (time that the account last
@ -45,33 +43,30 @@ type GetRSSFeed func() (string, gtserror.WithCode)
//
// If the account has not yet posted an RSS-eligible status, the returned last-modified
// time will be zero, and the GetRSSFeed func will return a valid RSS xml with no items.
func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string) (GetRSSFeed, time.Time, gtserror.WithCode) {
var (
never = time.Time{}
)
func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string, page *paging.Page) (GetRSSFeed, time.Time, gtserror.WithCode) {
// Fetch local (i.e. empty domain) account from database by username.
account, err := p.state.DB.GetAccountByUsernameDomain(ctx, username, "")
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// Simply no account with this username.
err = gtserror.New("account not found")
return nil, never, gtserror.NewErrorNotFound(err)
}
// Real db error.
err = gtserror.Newf("db error getting account %s: %w", username, err)
err := gtserror.Newf("db error getting account %s: %w", username, err)
return nil, never, gtserror.NewErrorInternalError(err)
}
// Check if exists.
if account == nil {
err := gtserror.New("account not found")
return nil, never, gtserror.NewErrorNotFound(err)
}
// Ensure account has rss feed enabled.
if !*account.Settings.EnableRSS {
err = gtserror.New("account RSS feed not enabled")
err := gtserror.New("account RSS feed not enabled")
return nil, never, gtserror.NewErrorNotFound(err)
}
// Ensure account stats populated.
// Ensure account stats populated for last status fetch information.
if err := p.state.DB.PopulateAccountStats(ctx, account); err != nil {
err = gtserror.Newf("db error getting account stats %s: %w", username, err)
err := gtserror.Newf("db error getting account stats %s: %w", username, err)
return nil, never, gtserror.NewErrorInternalError(err)
}
@ -80,14 +75,14 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
// eligible to appear in the RSS feed; that's fine.
lastPostAt := account.Stats.LastStatusAt
return func() (string, gtserror.WithCode) {
return func() (*feeds.Feed, gtserror.WithCode) {
// Assemble author namestring once only.
author := "@" + account.Username + "@" + config.GetAccountDomain()
// Derive image/thumbnail for this account (may be nil).
// Derive image/thumbnail for this account (may be nil if no media).
image, errWithCode := p.rssImageForAccount(ctx, account, author)
if errWithCode != nil {
return "", errWithCode
return nil, errWithCode
}
feed := &feeds.Feed{
@ -106,7 +101,7 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
// since we already know there's no eligible statuses.
if lastPostAt.IsZero() {
feed.Updated = account.CreatedAt
return stringifyFeed(feed)
return feed, nil
}
// Account has posted at least one status that's
@ -120,32 +115,30 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
//
// Take into account whether the user wants
// their web view laid out in gallery mode.
mediaOnly := account.Settings != nil &&
account.Settings.WebLayout == gtsmodel.WebLayoutGallery
mediaOnly := (account.Settings != nil &&
account.Settings.WebLayout == gtsmodel.WebLayoutGallery)
statuses, err := p.state.DB.GetAccountWebStatuses(
ctx,
account,
page,
mediaOnly,
rssFeedLength,
"", // Latest posts from the top.
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err = fmt.Errorf("db error getting account web statuses: %w", err)
return "", gtserror.NewErrorInternalError(err)
err := gtserror.Newf("db error getting account web statuses: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Add each status to the rss feed.
for _, status := range statuses {
item, err := p.converter.StatusToRSSItem(ctx, status)
if err != nil {
err = gtserror.Newf("error converting status to feed item: %w", err)
return "", gtserror.NewErrorInternalError(err)
err := gtserror.Newf("error converting status to feed item: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
feed.Add(item)
}
return stringifyFeed(feed)
return feed, nil
}, lastPostAt, nil
}
@ -177,15 +170,3 @@ func (p *Processor) rssImageForAccount(ctx context.Context, account *gtsmodel.Ac
Link: account.URL,
}, nil
}
func stringifyFeed(feed *feeds.Feed) (string, gtserror.WithCode) {
// Stringify the feed. Even with no statuses,
// this will still produce valid rss xml.
rss, err := feed.ToRss()
if err != nil {
err := gtserror.Newf("error converting feed to rss string: %w", err)
return "", gtserror.NewErrorInternalError(err)
}
return rss, nil
}

291
internal/processing/account/rss_test.go
View file

@ -19,7 +19,10 @@ package account_test
import (
"testing"
"time"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gorilla/feeds"
"github.com/stretchr/testify/suite"
)
@ -28,13 +31,8 @@ type GetRSSTestSuite struct {
}
func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() {
getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(suite.T().Context(), "admin")
suite.NoError(err)
suite.EqualValues(1634726497, lastModified.Unix())
feed, err := getFeed()
suite.NoError(err)
suite.Equal(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1634726497,
`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Posts from @admin@localhost:8080</title>
<link>http://localhost:8080/@admin</link>
@ -63,17 +61,94 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() {
<source>http://localhost:8080/@admin/feed.rss</source>
</item>
</channel>
</rss>`, feed)
</rss>`)
}
func (suite *GetRSSTestSuite) TestGetAccountAtomAdmin() {
suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToAtom, 1634726497,
`<?xml version="1.0" encoding="UTF-8"?><feed xmlns="http://www.w3.org/2005/Atom">
<title>Posts from @admin@localhost:8080</title>
<id>http://localhost:8080/@admin</id>
<updated>2021-10-20T10:41:37Z</updated>
<subtitle>Posts from @admin@localhost:8080</subtitle>
<link href="http://localhost:8080/@admin"></link>
<entry>
<title>open to see some &lt;strong&gt;puppies&lt;/strong&gt;</title>
<updated>2021-10-20T12:36:45Z</updated>
<id>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</id>
<content type="html">&lt;p&gt;🐕🐕🐕🐕🐕&lt;/p&gt;</content>
<link href="http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37" rel="alternate"></link>
<link href="" rel="enclosure"></link>
<summary type="html">@admin@localhost:8080 made a new post: &#34;🐕🐕🐕🐕🐕&#34;</summary>
<author>
<name>@admin@localhost:8080</name>
</author>
</entry>
<entry>
<title>hello world! #welcome ! first post on the instance :rainbow: !</title>
<updated>2021-10-20T11:36:45Z</updated>
<id>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</id>
<content type="html">&lt;p&gt;hello world! &lt;a href=&#34;http://localhost:8080/tags/welcome&#34; class=&#34;mention hashtag&#34; rel=&#34;tag nofollow noreferrer noopener&#34; target=&#34;_blank&#34;&gt;#&lt;span&gt;welcome&lt;/span&gt;&lt;/a&gt; ! first post on the instance &lt;img src=&#34;http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png&#34; title=&#34;:rainbow:&#34; alt=&#34;:rainbow:&#34; width=&#34;25&#34; height=&#34;25&#34; /&gt; !&lt;/p&gt;</content>
<link href="http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R" rel="alternate"></link>
<link href="http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg" rel="enclosure" type="image/jpeg" length="62529"></link>
<summary type="html">@admin@localhost:8080 posted 1 attachment: &#34;hello world! #welcome ! first post on the instance :rainbow: !&#34;</summary>
<author>
<name>@admin@localhost:8080</name>
</author>
</entry>
</feed>`)
}
func (suite *GetRSSTestSuite) TestGetAccountJSONAdmin() {
suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToJSON, 1634726497,
`{
"version": "https://jsonfeed.org/version/1.1",
"title": "Posts from @admin@localhost:8080",
"home_page_url": "http://localhost:8080/@admin",
"description": "Posts from @admin@localhost:8080",
"items": [
{
"id": "http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37",
"url": "http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37",
"external_url": "http://localhost:8080/@admin/feed.rss",
"title": "open to see some \u003cstrong\u003epuppies\u003c/strong\u003e",
"content_html": "\u003cp\u003e🐕🐕🐕🐕🐕\u003c/p\u003e",
"summary": "@admin@localhost:8080 made a new post: \"🐕🐕🐕🐕🐕\"",
"date_published": "2021-10-20T12:36:45Z",
"author": {
"name": "@admin@localhost:8080"
},
"authors": [
{
"name": "@admin@localhost:8080"
}
]
},
{
"id": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
"url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
"external_url": "http://localhost:8080/@admin/feed.rss",
"title": "hello world! #welcome ! first post on the instance :rainbow: !",
"content_html": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance \u003cimg src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" width=\"25\" height=\"25\" /\u003e !\u003c/p\u003e",
"summary": "@admin@localhost:8080 posted 1 attachment: \"hello world! #welcome ! first post on the instance :rainbow: !\"",
"image": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
"date_published": "2021-10-20T11:36:45Z",
"author": {
"name": "@admin@localhost:8080"
},
"authors": [
{
"name": "@admin@localhost:8080"
}
]
}
]
}`)
}
func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(suite.T().Context(), "the_mighty_zork")
suite.NoError(err)
suite.EqualValues(1730451600, lastModified.Unix())
feed, err := getFeed()
suite.NoError(err)
suite.Equal(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1730451600,
`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Posts from @the_mighty_zork@localhost:8080</title>
<link>http://localhost:8080/@the_mighty_zork</link>
@ -151,7 +226,154 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
<source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
</item>
</channel>
</rss>`, feed)
</rss>`)
}
func (suite *GetRSSTestSuite) TestGetAccountAtomZork() {
suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1730451600,
`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Posts from @the_mighty_zork@localhost:8080</title>
<link>http://localhost:8080/@the_mighty_zork</link>
<description>Posts from @the_mighty_zork@localhost:8080</description>
<pubDate>Fri, 01 Nov 2024 09:00:00 +0000</pubDate>
<lastBuildDate>Fri, 01 Nov 2024 09:00:00 +0000</lastBuildDate>
<image>
<url>http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.webp</url>
<title>Avatar for @the_mighty_zork@localhost:8080</title>
<link>http://localhost:8080/@the_mighty_zork</link>
</image>
<item>
<title>edited status</title>
<link>http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR</link>
<description>@the_mighty_zork@localhost:8080 made a new post: &#34;this is the latest revision of the status, with a content-warning&#34;</description>
<content:encoded><![CDATA[<p>this is the latest revision of the status, with a content-warning</p>]]></content:encoded>
<author>@the_mighty_zork@localhost:8080</author>
<guid isPermaLink="true">http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR</guid>
<pubDate>Fri, 01 Nov 2024 09:00:00 +0000</pubDate>
<source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
</item>
<item>
<title>HTML in post</title>
<link>http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40</link>
<description>@the_mighty_zork@localhost:8080 made a new post: &#34;Here&#39;s a bunch of HTML, read it and weep, weep then!&#xA;&#xA;`+"```"+`html&#xA;&lt;section class=&#34;about-user&#34;&gt;&#xA; &lt;div class=&#34;col-header&#34;&gt;&#xA; &lt;h2&gt;About&lt;/h2&gt;&#xA; &lt;/div&gt; &#xA; &lt;div class=&#34;fields&#34;&gt;&#xA; &lt;h3 class=&#34;sr-only&#34;&gt;Fields&lt;/h3&gt;&#xA; &lt;dl&gt;&#xA;...</description>
<content:encoded><![CDATA[<p>Here's a bunch of HTML, read it and weep, weep then!</p><pre><code class="language-html">&lt;section class=&#34;about-user&#34;&gt;
&lt;div class=&#34;col-header&#34;&gt;
&lt;h2&gt;About&lt;/h2&gt;
&lt;/div&gt;
&lt;div class=&#34;fields&#34;&gt;
&lt;h3 class=&#34;sr-only&#34;&gt;Fields&lt;/h3&gt;
&lt;dl&gt;
&lt;div class=&#34;field&#34;&gt;
&lt;dt&gt;should you follow me?&lt;/dt&gt;
&lt;dd&gt;maybe!&lt;/dd&gt;
&lt;/div&gt;
&lt;div class=&#34;field&#34;&gt;
&lt;dt&gt;age&lt;/dt&gt;
&lt;dd&gt;120&lt;/dd&gt;
&lt;/div&gt;
&lt;/dl&gt;
&lt;/div&gt;
&lt;div class=&#34;bio&#34;&gt;
&lt;h3 class=&#34;sr-only&#34;&gt;Bio&lt;/h3&gt;
&lt;p&gt;i post about things that concern me&lt;/p&gt;
&lt;/div&gt;
&lt;div class=&#34;sr-only&#34; role=&#34;group&#34;&gt;
&lt;h3 class=&#34;sr-only&#34;&gt;Stats&lt;/h3&gt;
&lt;span&gt;Joined in Jun, 2022.&lt;/span&gt;
&lt;span&gt;8 posts.&lt;/span&gt;
&lt;span&gt;Followed by 1.&lt;/span&gt;
&lt;span&gt;Following 1.&lt;/span&gt;
&lt;/div&gt;
&lt;div class=&#34;accountstats&#34; aria-hidden=&#34;true&#34;&gt;
&lt;b&gt;Joined&lt;/b&gt;&lt;time datetime=&#34;2022-06-04T13:12:00.000Z&#34;&gt;Jun, 2022&lt;/time&gt;
&lt;b&gt;Posts&lt;/b&gt;&lt;span&gt;8&lt;/span&gt;
&lt;b&gt;Followed by&lt;/b&gt;&lt;span&gt;1&lt;/span&gt;
&lt;b&gt;Following&lt;/b&gt;&lt;span&gt;1&lt;/span&gt;
&lt;/div&gt;
&lt;/section&gt;
</code></pre><p>There, hope you liked that!</p>]]></content:encoded>
<author>@the_mighty_zork@localhost:8080</author>
<guid isPermaLink="true">http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40</guid>
<pubDate>Sun, 10 Dec 2023 09:24:00 +0000</pubDate>
<source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
</item>
<item>
<title>introduction post</title>
<link>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</link>
<description>@the_mighty_zork@localhost:8080 made a new post: &#34;hello everyone!&#34;</description>
<content:encoded><![CDATA[<p>hello everyone!</p>]]></content:encoded>
<author>@the_mighty_zork@localhost:8080</author>
<guid isPermaLink="true">http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</guid>
<pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>
<source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
</item>
</channel>
</rss>`)
}
func (suite *GetRSSTestSuite) TestGetAccountJSONZork() {
suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToJSON, 1730451600,
`{
"version": "https://jsonfeed.org/version/1.1",
"title": "Posts from @the_mighty_zork@localhost:8080",
"home_page_url": "http://localhost:8080/@the_mighty_zork",
"description": "Posts from @the_mighty_zork@localhost:8080",
"items": [
{
"id": "http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR",
"url": "http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR",
"external_url": "http://localhost:8080/@the_mighty_zork/feed.rss",
"title": "edited status",
"content_html": "\u003cp\u003ethis is the latest revision of the status, with a content-warning\u003c/p\u003e",
"summary": "@the_mighty_zork@localhost:8080 made a new post: \"this is the latest revision of the status, with a content-warning\"",
"date_published": "2024-11-01T09:00:00Z",
"date_modified": "2024-11-01T09:02:00Z",
"author": {
"name": "@the_mighty_zork@localhost:8080"
},
"authors": [
{
"name": "@the_mighty_zork@localhost:8080"
}
]
},
{
"id": "http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40",
"url": "http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40",
"external_url": "http://localhost:8080/@the_mighty_zork/feed.rss",
"title": "HTML in post",
"content_html": "\u003cp\u003eHere's a bunch of HTML, read it and weep, weep then!\u003c/p\u003e\u003cpre\u003e\u003ccode class=\"language-html\"\u003e\u0026lt;section class=\u0026#34;about-user\u0026#34;\u0026gt;\n \u0026lt;div class=\u0026#34;col-header\u0026#34;\u0026gt;\n \u0026lt;h2\u0026gt;About\u0026lt;/h2\u0026gt;\n \u0026lt;/div\u0026gt; \n \u0026lt;div class=\u0026#34;fields\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Fields\u0026lt;/h3\u0026gt;\n \u0026lt;dl\u0026gt;\n \u0026lt;div class=\u0026#34;field\u0026#34;\u0026gt;\n \u0026lt;dt\u0026gt;should you follow me?\u0026lt;/dt\u0026gt;\n \u0026lt;dd\u0026gt;maybe!\u0026lt;/dd\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;field\u0026#34;\u0026gt;\n \u0026lt;dt\u0026gt;age\u0026lt;/dt\u0026gt;\n \u0026lt;dd\u0026gt;120\u0026lt;/dd\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;/dl\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;bio\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Bio\u0026lt;/h3\u0026gt;\n \u0026lt;p\u0026gt;i post about things that concern me\u0026lt;/p\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;sr-only\u0026#34; role=\u0026#34;group\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Stats\u0026lt;/h3\u0026gt;\n \u0026lt;span\u0026gt;Joined in Jun, 2022.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;8 posts.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;Followed by 1.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;Following 1.\u0026lt;/span\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;accountstats\u0026#34; aria-hidden=\u0026#34;true\u0026#34;\u0026gt;\n \u0026lt;b\u0026gt;Joined\u0026lt;/b\u0026gt;\u0026lt;time datetime=\u0026#34;2022-06-04T13:12:00.000Z\u0026#34;\u0026gt;Jun, 2022\u0026lt;/time\u0026gt;\n \u0026lt;b\u0026gt;Posts\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;8\u0026lt;/span\u0026gt;\n \u0026lt;b\u0026gt;Followed by\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;1\u0026lt;/span\u0026gt;\n \u0026lt;b\u0026gt;Following\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;1\u0026lt;/span\u0026gt;\n \u0026lt;/div\u0026gt;\n\u0026lt;/section\u0026gt;\n\u003c/code\u003e\u003c/pre\u003e\u003cp\u003eThere, hope you liked that!\u003c/p\u003e",
"summary": "@the_mighty_zork@localhost:8080 made a new post: \"Here's a bunch of HTML, read it and weep, weep then!\n\n`+"```"+`html\n\u003csection class=\"about-user\"\u003e\n \u003cdiv class=\"col-header\"\u003e\n \u003ch2\u003eAbout\u003c/h2\u003e\n \u003c/div\u003e \n \u003cdiv class=\"fields\"\u003e\n \u003ch3 class=\"sr-only\"\u003eFields\u003c/h3\u003e\n \u003cdl\u003e\n...",
"date_published": "2023-12-10T09:24:00Z",
"author": {
"name": "@the_mighty_zork@localhost:8080"
},
"authors": [
{
"name": "@the_mighty_zork@localhost:8080"
}
]
},
{
"id": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
"url": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
"external_url": "http://localhost:8080/@the_mighty_zork/feed.rss",
"title": "introduction post",
"content_html": "\u003cp\u003ehello everyone!\u003c/p\u003e",
"summary": "@the_mighty_zork@localhost:8080 made a new post: \"hello everyone!\"",
"date_published": "2021-10-20T10:40:37Z",
"author": {
"name": "@the_mighty_zork@localhost:8080"
},
"authors": [
{
"name": "@the_mighty_zork@localhost:8080"
}
]
}
]
}`)
}
func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() {
@ -170,13 +392,10 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() {
}
}
getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(ctx, "the_mighty_zork")
suite.NoError(err)
suite.Empty(lastModified)
var zeroTime time.Time
feed, err := getFeed()
suite.NoError(err)
suite.Equal(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, zeroTime.Unix(),
`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Posts from @the_mighty_zork@localhost:8080</title>
<link>http://localhost:8080/@the_mighty_zork</link>
@ -189,7 +408,33 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() {
<link>http://localhost:8080/@the_mighty_zork</link>
</image>
</channel>
</rss>`, feed)
</rss>`)
}
// func (suite *GetRSSTestSuite) testGetAccountRSSPaging(username string, page *paging.Page, expectIDs []string) {
// ctx := suite.T().Context()
// getFeed, _, errWithCode := suite.accountProcessor.GetRSSFeedForUsername(ctx, username, page)
// suite.NoError(errWithCode)
// feed, errWithCode := getFeed()
// suite.NoError(errWithCode)
// }
func (suite *GetRSSTestSuite) testGetFeedSerializedAs(username string, page *paging.Page, serialize func(*feeds.Feed) (string, error), expectLastMod int64, expectSerialized string) {
ctx := suite.T().Context()
getFeed, lastMod, errWithCode := suite.accountProcessor.GetRSSFeedForUsername(ctx, username, page)
suite.NoError(errWithCode)
suite.Equal(expectLastMod, lastMod.Unix())
feed, errWithCode := getFeed()
suite.NoError(errWithCode)
feedStr, err := serialize(feed)
suite.NoError(err)
suite.Equal(expectSerialized, feedStr)
}
func TestGetRSSTestSuite(t *testing.T) {

11
internal/processing/account/statuses.go
View file

@ -27,6 +27,7 @@ import (
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"code.superseriousbusiness.org/gotosocial/internal/util"
)
@ -156,9 +157,8 @@ func (p *Processor) StatusesGet(
func (p *Processor) WebStatusesGet(
ctx context.Context,
targetAccountID string,
page *paging.Page,
mediaOnly bool,
limit int,
maxID string,
) (*apimodel.PageableResponse, gtserror.WithCode) {
account, err := p.state.DB.GetAccountByID(ctx, targetAccountID)
if err != nil {
@ -174,14 +174,13 @@ func (p *Processor) WebStatusesGet(
return nil, gtserror.NewErrorNotFound(err)
}
statuses, err := p.state.DB.GetAccountWebStatuses(
ctx,
statuses, err := p.state.DB.GetAccountWebStatuses(ctx,
account,
page,
mediaOnly,
limit,
maxID,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err := gtserror.Newf("db error getting statuses: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}

2
internal/web/assets.go
View file

@ -82,7 +82,7 @@ func getAssetETag(
return cachedETag.eTag, nil
}
eTag, err := generateEtag(file)
eTag, err := generateETag(file)
if err != nil {
return "", fmt.Errorf("error generating etag: %s", err)
}

19
internal/web/etag.go
View file

@ -27,6 +27,7 @@ import (
"code.superseriousbusiness.org/gotosocial/internal/log"
"codeberg.org/gruf/go-cache/v3"
"codeberg.org/gruf/go-fastcopy"
)
type withETagCache interface {
@ -47,13 +48,19 @@ type eTagCacheEntry struct {
lastModified time.Time
}
// generateEtag generates a strong (byte-for-byte) etag using
// the entirety of the provided reader.
func generateEtag(r io.Reader) (string, error) {
// nolint:gosec
hash := sha1.New()
// generateEtag generates a strong (byte-for-byte) etag
// using the entirety of the provided reader.
func generateETag(r io.Reader) (string, error) {
return generateETagFrom(func(w io.Writer) error {
_, err := fastcopy.Copy(w, r)
return err
})
}
if _, err := io.Copy(hash, r); err != nil {
func generateETagFrom(writeTo func(io.Writer) error) (string, error) {
hash := sha1.New() // nolint:gosec
if err := writeTo(hash); err != nil {
return "", err
}

13
internal/web/profile.go
View file

@ -27,6 +27,7 @@ import (
apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gin-gonic/gin"
)
@ -122,14 +123,15 @@ func (m *Module) prepareProfile(c *gin.Context) *profile {
// Check if paging.
maxStatusID := apiutil.ParseMaxID(c.Query(apiutil.MaxIDKey), "")
paging := maxStatusID != ""
doPaging := (maxStatusID != "")
// If not paging, load pinned statuses.
var (
mediaOnly = account.WebLayout == "gallery"
pinnedStatuses []*apimodel.WebStatus
)
if !paging {
if !doPaging {
// If not paging, load pinned statuses.
var errWithCode gtserror.WithCode
pinnedStatuses, errWithCode = m.processor.Account().WebStatusesGetPinned(
ctx,
@ -156,9 +158,8 @@ func (m *Module) prepareProfile(c *gin.Context) *profile {
statusResp, errWithCode := m.processor.Account().WebStatusesGet(
ctx,
account.ID,
&paging.Page{Max: paging.MaxID(maxStatusID), Limit: limit},
mediaOnly,
limit,
maxStatusID,
)
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, instanceGet)
@ -172,7 +173,7 @@ func (m *Module) prepareProfile(c *gin.Context) *profile {
robotsMeta: robotsMeta,
pinnedStatuses: pinnedStatuses,
statusResp: statusResp,
paging: paging,
paging: doPaging,
}
}

86
internal/web/rss.go
View file

@ -18,7 +18,6 @@
package web
import (
"bytes"
"net/http"
"strings"
"time"
@ -26,13 +25,26 @@ import (
apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gin-gonic/gin"
"github.com/gorilla/feeds"
)
const appRSSUTF8 = string(apiutil.AppRSSXML) + "; charset=utf-8"
const (
charsetUTF8 = "; charset=utf-8"
appRSSUTF8 = string(apiutil.AppRSSXML) + charsetUTF8
appAtomUTF8 = string(apiutil.AppAtomXML) + charsetUTF8
appJSONUTF8 = string(apiutil.AppFeedJSON) + charsetUTF8
)
func (m *Module) rssFeedGETHandler(c *gin.Context) {
if _, err := apiutil.NegotiateAccept(c, apiutil.AppRSSXML); err != nil {
contentType, err := apiutil.NegotiateAccept(c,
apiutil.AppRSSXML,
apiutil.AppAtomXML,
apiutil.AppFeedJSON,
apiutil.AppJSON,
)
if err != nil {
apiutil.WebErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1)
return
}
@ -49,21 +61,34 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) {
// todo: https://codeberg.org/superseriousbusiness/gotosocial/issues/1813
username = strings.ToLower(username)
// Retrieve the getRSSFeed function from the processor.
// We'll only call the function if we need to, to save db calls.
// lastPostAt may be a zero time if account has never posted.
getRSSFeed, lastPostAt, errWithCode := m.processor.Account().GetRSSFeedForUsername(c.Request.Context(), username)
// Parse paging parameters from request.
page, errWithCode := paging.ParseIDPage(c,
1, // min limit
40, // max limit
20, // default limit
)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
getFunc, lastPostAt, errWithCode := m.processor.Account().GetRSSFeedForUsername(
c.Request.Context(),
username,
page,
)
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
var (
rssFeed string // Stringified rss feed.
var feed *feeds.Feed
cacheKey = c.Request.URL.Path
cacheEntry, wasCached = m.eTagCache.Get(cacheKey)
)
// Key to use in etag cache (note content-type suffix).
cacheKey := c.Request.URL.Path + "#" + contentType
// Check etag cache for an existing entry under key.
cacheEntry, wasCached := m.eTagCache.Get(cacheKey)
if !wasCached || unixAfter(lastPostAt, cacheEntry.lastModified) {
// We either have no ETag cache entry for this account's feed,
@ -72,15 +97,16 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) {
//
// As such, we need to generate a new ETag, and for that we need
// the string representation of the RSS feed.
rssFeed, errWithCode = getRSSFeed()
feed, errWithCode = getFunc()
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
eTag, err := generateEtag(bytes.NewBufferString(rssFeed))
etag, err := generateFeedETag(feed, contentType)
if err != nil {
apiutil.WebErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGetV1)
errWithCode := gtserror.NewErrorInternalError(err)
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
@ -96,7 +122,7 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) {
// Store the new cache entry.
cacheEntry = eTagCacheEntry{
eTag: eTag,
eTag: etag,
lastModified: lastModified,
}
m.eTagCache.Set(cacheKey, cacheEntry)
@ -149,15 +175,37 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) {
// If we had a cache hit earlier, we may not have called the
// getRSSFeed function yet; if that's the case then do call it
// now because we definitely need it.
if rssFeed == "" {
rssFeed, errWithCode = getRSSFeed()
if feed == nil {
feed, errWithCode = getFunc()
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
}
c.Data(http.StatusOK, appRSSUTF8, []byte(rssFeed))
// Encode response.
switch contentType {
case apiutil.AppRSSXML:
apiutil.XMLType(c, http.StatusOK, appRSSUTF8, &feeds.Rss{feed})
case apiutil.AppAtomXML:
apiutil.XMLType(c, http.StatusOK, appAtomUTF8, &feeds.Atom{feed})
case apiutil.AppFeedJSON, apiutil.AppJSON:
apiutil.JSONType(c, http.StatusOK, appJSONUTF8, (&feeds.JSON{feed}).JSONFeed())
}
}
// generateFeedETag generates feed etag for appropriate content-type encoding.
func generateFeedETag(feed *feeds.Feed, contentType string) (string, error) {
switch contentType {
case apiutil.AppRSSXML:
return generateETagFrom(feed.WriteRss)
case apiutil.AppAtomXML:
return generateETagFrom(feed.WriteAtom)
case apiutil.AppFeedJSON, apiutil.AppJSON:
return generateETagFrom(feed.WriteJSON)
default:
panic("unreachable")
}
}
// unixAfter returns true if the unix value of t1