[feature] add paging support to rss feed endpoint, and support JSON / atom feed types (#4442)

originally based on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4396

hope this is okay https://codeberg.org/zordsdavini !

closes https://codeberg.org/superseriousbusiness/gotosocial/issues/4411
closes https://codeberg.org/superseriousbusiness/gotosocial/issues/3407

Co-authored-by: Arnas Udovic <zordsdavini@gmail.com>
Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4442
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2025-09-18 16:33:23 +02:00 committed by kim
commit 6607e1c944
13 changed files with 447 additions and 182 deletions

View file

@ -12,4 +12,8 @@ When enabled, the RSS feed for your account will be available at `https://[your-
## Which posts are shared via RSS?
Only your latest 20 Public posts are shared via RSS. Replies and reblogs/boosts are not included. Unlisted posts are not included. In other words, the only posts visible via RSS will be the same ones that are visible when you open your profile in a browser.
Only your latest 20 Public posts are shared via RSS by default. Replies and reblogs/boosts are not included. Unlisted posts are not included. In other words, the only posts visible via RSS will be the same ones that are visible when you open your profile in a browser.
If you want to see more posts, you can provide our standard set of timeline paging parameters ([as per our swagger documentation](https://docs.gotosocial.org/en/latest/api/swagger)) to see beyond the first page.
You can also access Atom and JSON feeds from this same endpoint, but providing the appropriate request content-type header. i.e. `application/atom+xml` for an Atom feed, or `application/feed+json` for a JSON feed.

View file

@ -26,6 +26,8 @@ const (
appXMLText = `text/xml` // AppXML is only *recommended* in RFC7303
AppXMLXRD = `application/xrd+xml`
AppRSSXML = `application/rss+xml`
AppAtomXML = `application/atom+xml`
AppFeedJSON = `application/feed+json`
AppActivityJSON = `application/activity+json`
appActivityLDJSON = `application/ld+json` // without profile
AppActivityLDJSON = appActivityLDJSON + `; profile="https://www.w3.org/ns/activitystreams"`

View file

@ -71,6 +71,18 @@ func JSONType(c *gin.Context, code int, contentType string, data any) {
EncodeJSONResponse(c.Writer, c.Request, code, contentType, data)
}
// XML calls EncodeJSONResponse() using gin.Context{}, with content-type = AppXML,
// This function handles the case of XML unmarshal errors and pools read buffers.
func XML(c *gin.Context, code int, data any) {
EncodeXMLResponse(c.Writer, c.Request, code, AppXML, data)
}
// XML calls EncodeXMLResponse() using gin.Context{}, with given content-type.
// This function handles the case of XML unmarshal errors and pools read buffers.
func XMLType(c *gin.Context, code int, contentType string, data any) {
EncodeXMLResponse(c.Writer, c.Request, code, contentType, data)
}
// Data calls WriteResponseBytes() using gin.Context{}, with given content-type.
func Data(c *gin.Context, code int, contentType string, data []byte) {
WriteResponseBytes(c.Writer, c.Request, code, contentType, data)
@ -230,6 +242,7 @@ func EncodeCSVResponse(
// Write all the records to the buffer.
if err := csvWriter.WriteAll(records); err == nil {
// Respond with the now-known
// size byte slice within buf.
WriteResponseBytes(rw, r,

View file

@ -121,7 +121,7 @@ type Account interface {
// returning statuses that should be visible via the web view of a *LOCAL* account.
//
// In the case of no statuses, this function will return db.ErrNoEntries.
GetAccountWebStatuses(ctx context.Context, account *gtsmodel.Account, mediaOnly bool, limit int, maxID string) ([]*gtsmodel.Status, error)
GetAccountWebStatuses(ctx context.Context, account *gtsmodel.Account, page *paging.Page, mediaOnly bool) ([]*gtsmodel.Status, error)
// GetInstanceAccount returns the instance account for the given domain.
// If domain is empty, this instance account will be returned.

View file

@ -900,7 +900,7 @@ func (a *accountDB) GetAccountFaves(ctx context.Context, accountID string) ([]*g
return *faves, nil
}
func qMediaOnly(q *bun.SelectQuery) *bun.SelectQuery {
func selectOnlyWithMedia(q *bun.SelectQuery) *bun.SelectQuery {
// Attachments are stored as a json object; this
// implementation differs between SQLite and Postgres,
// so we have to be thorough to cover all eventualities
@ -908,14 +908,14 @@ func qMediaOnly(q *bun.SelectQuery) *bun.SelectQuery {
switch d := q.Dialect().Name(); d {
case dialect.PG:
return q.
Where("? IS NOT NULL", bun.Ident("status.attachments")).
Where("? != '{}'", bun.Ident("status.attachments"))
Where("? IS NOT NULL", bun.Ident("attachments")).
Where("? != '{}'", bun.Ident("attachments"))
case dialect.SQLite:
return q.
Where("? IS NOT NULL", bun.Ident("status.attachments")).
Where("? != 'null'", bun.Ident("status.attachments")).
Where("? != '[]'", bun.Ident("status.attachments"))
Where("? IS NOT NULL", bun.Ident("attachments")).
Where("? != 'null'", bun.Ident("attachments")).
Where("? != '[]'", bun.Ident("attachments"))
default:
panic("dialect " + d.String() + " was neither pg nor sqlite")
@ -963,9 +963,9 @@ func (a *accountDB) GetAccountStatuses(ctx context.Context, accountID string, li
q = q.Where("? IS NULL", bun.Ident("status.boost_of_id"))
}
// Respect media-only preference.
if mediaOnly {
q = qMediaOnly(q)
// Respect mediaOnly pref.
q = selectOnlyWithMedia(q)
}
if publicOnly {
@ -1041,12 +1041,16 @@ func (a *accountDB) GetAccountPinnedStatuses(ctx context.Context, accountID stri
return a.state.DB.GetStatusesByIDs(ctx, statusIDs)
}
var webStatusVisibilities = bun.In([]gtsmodel.Visibility{
gtsmodel.VisibilityPublic,
gtsmodel.VisibilityUnlocked,
})
func (a *accountDB) GetAccountWebStatuses(
ctx context.Context,
account *gtsmodel.Account,
page *paging.Page,
mediaOnly bool,
limit int,
maxID string,
) ([]*gtsmodel.Status, error) {
if account.Username == config.GetHost() {
// Instance account
@ -1071,74 +1075,35 @@ func (a *accountDB) GetAccountWebStatuses(
return nil, nil
}
// Ensure reasonable
if limit < 0 {
limit = 0
}
return loadStatusTimelinePage(ctx, a.db, a.state,
// Make educated guess for slice size
statusIDs := make([]string, 0, limit)
// Paging
// params.
page,
q := a.db.
NewSelect().
TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")).
// Select only IDs from table
Column("status.id").
Where("? = ?", bun.Ident("status.account_id"), account.ID)
// The actual meat of the account web statuses query.
func(q *bun.SelectQuery) (*bun.SelectQuery, error) {
q = q.Where("? = ?", bun.Ident("account_id"), account.ID)
// Select statuses according to
// account's web visibility prefs.
if publicOnly {
// Only Public statuses.
q = q.Where("? = ?", bun.Ident("status.visibility"), gtsmodel.VisibilityPublic)
} else {
// Public or Unlocked.
visis := []gtsmodel.Visibility{
gtsmodel.VisibilityPublic,
gtsmodel.VisibilityUnlocked,
}
q = q.Where("? IN (?)", bun.Ident("status.visibility"), bun.In(visis))
}
if publicOnly {
q = q.Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityPublic)
} else {
q = q.Where("? IN (?)", bun.Ident("visibility"), webStatusVisibilities)
}
// Don't show replies, boosts, or
// local-only statuses on the web view.
q = q.
Where("? IS NULL", bun.Ident("status.in_reply_to_uri")).
Where("? IS NULL", bun.Ident("status.boost_of_id")).
Where("? = ?", bun.Ident("status.federated"), true)
// Don't show replies, boosts, or local-only in web view.
q = q.Where("? IS NULL", bun.Ident("in_reply_to_uri")).
Where("? IS NULL", bun.Ident("boost_of_id")).
Where("? = ?", bun.Ident("federated"), true)
// Respect media-only preference.
if mediaOnly {
q = qMediaOnly(q)
}
if mediaOnly {
// Respect mediaOnly pref.
q = selectOnlyWithMedia(q)
}
// Return only statuses LOWER (ie., older) than maxID
if maxID == "" {
maxID = id.Highest
}
q = q.Where("? < ?", bun.Ident("status.id"), maxID)
if limit > 0 {
// limit amount of statuses returned
q = q.Limit(limit)
}
if limit > 0 {
// limit amount of statuses returned
q = q.Limit(limit)
}
q = q.Order("status.id DESC")
if err := q.Scan(ctx, &statusIDs); err != nil {
return nil, err
}
if len(statusIDs) == 0 {
return nil, db.ErrNoEntries
}
return a.state.DB.GetStatusesByIDs(ctx, statusIDs)
return q, nil
},
)
}
func (a *accountDB) GetAccountSettings(

View file

@ -49,7 +49,7 @@ func (suite *AccountTestSuite) TestGetAccountStatuses() {
}
func (suite *AccountTestSuite) TestGetAccountWebStatusesMediaOnly() {
statuses, err := suite.db.GetAccountWebStatuses(suite.T().Context(), suite.testAccounts["local_account_3"], true, 20, "")
statuses, err := suite.db.GetAccountWebStatuses(suite.T().Context(), suite.testAccounts["local_account_3"], &paging.Page{Limit: 20}, true)
suite.NoError(err)
suite.Len(statuses, 2)
}

View file

@ -20,21 +20,19 @@ package account
import (
"context"
"errors"
"fmt"
"time"
"code.superseriousbusiness.org/gotosocial/internal/config"
"code.superseriousbusiness.org/gotosocial/internal/db"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gorilla/feeds"
)
const (
rssFeedLength = 20
)
var never time.Time
type GetRSSFeed func() (string, gtserror.WithCode)
type GetRSSFeed func() (*feeds.Feed, gtserror.WithCode)
// GetRSSFeedForUsername returns a function to return the RSS feed of a local account
// with the given username, and the last-modified time (time that the account last
@ -45,33 +43,30 @@ type GetRSSFeed func() (string, gtserror.WithCode)
//
// If the account has not yet posted an RSS-eligible status, the returned last-modified
// time will be zero, and the GetRSSFeed func will return a valid RSS xml with no items.
func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string) (GetRSSFeed, time.Time, gtserror.WithCode) {
var (
never = time.Time{}
)
func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string, page *paging.Page) (GetRSSFeed, time.Time, gtserror.WithCode) {
// Fetch local (i.e. empty domain) account from database by username.
account, err := p.state.DB.GetAccountByUsernameDomain(ctx, username, "")
if err != nil {
if errors.Is(err, db.ErrNoEntries) {
// Simply no account with this username.
err = gtserror.New("account not found")
return nil, never, gtserror.NewErrorNotFound(err)
}
// Real db error.
err = gtserror.Newf("db error getting account %s: %w", username, err)
err := gtserror.Newf("db error getting account %s: %w", username, err)
return nil, never, gtserror.NewErrorInternalError(err)
}
// Check if exists.
if account == nil {
err := gtserror.New("account not found")
return nil, never, gtserror.NewErrorNotFound(err)
}
// Ensure account has rss feed enabled.
if !*account.Settings.EnableRSS {
err = gtserror.New("account RSS feed not enabled")
err := gtserror.New("account RSS feed not enabled")
return nil, never, gtserror.NewErrorNotFound(err)
}
// Ensure account stats populated.
// Ensure account stats populated for last status fetch information.
if err := p.state.DB.PopulateAccountStats(ctx, account); err != nil {
err = gtserror.Newf("db error getting account stats %s: %w", username, err)
err := gtserror.Newf("db error getting account stats %s: %w", username, err)
return nil, never, gtserror.NewErrorInternalError(err)
}
@ -80,14 +75,14 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
// eligible to appear in the RSS feed; that's fine.
lastPostAt := account.Stats.LastStatusAt
return func() (string, gtserror.WithCode) {
return func() (*feeds.Feed, gtserror.WithCode) {
// Assemble author namestring once only.
author := "@" + account.Username + "@" + config.GetAccountDomain()
// Derive image/thumbnail for this account (may be nil).
// Derive image/thumbnail for this account (may be nil if no media).
image, errWithCode := p.rssImageForAccount(ctx, account, author)
if errWithCode != nil {
return "", errWithCode
return nil, errWithCode
}
feed := &feeds.Feed{
@ -106,7 +101,7 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
// since we already know there's no eligible statuses.
if lastPostAt.IsZero() {
feed.Updated = account.CreatedAt
return stringifyFeed(feed)
return feed, nil
}
// Account has posted at least one status that's
@ -120,32 +115,30 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string)
//
// Take into account whether the user wants
// their web view laid out in gallery mode.
mediaOnly := account.Settings != nil &&
account.Settings.WebLayout == gtsmodel.WebLayoutGallery
mediaOnly := (account.Settings != nil &&
account.Settings.WebLayout == gtsmodel.WebLayoutGallery)
statuses, err := p.state.DB.GetAccountWebStatuses(
ctx,
account,
page,
mediaOnly,
rssFeedLength,
"", // Latest posts from the top.
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err = fmt.Errorf("db error getting account web statuses: %w", err)
return "", gtserror.NewErrorInternalError(err)
err := gtserror.Newf("db error getting account web statuses: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
// Add each status to the rss feed.
for _, status := range statuses {
item, err := p.converter.StatusToRSSItem(ctx, status)
if err != nil {
err = gtserror.Newf("error converting status to feed item: %w", err)
return "", gtserror.NewErrorInternalError(err)
err := gtserror.Newf("error converting status to feed item: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}
feed.Add(item)
}
return stringifyFeed(feed)
return feed, nil
}, lastPostAt, nil
}
@ -177,15 +170,3 @@ func (p *Processor) rssImageForAccount(ctx context.Context, account *gtsmodel.Ac
Link: account.URL,
}, nil
}
func stringifyFeed(feed *feeds.Feed) (string, gtserror.WithCode) {
// Stringify the feed. Even with no statuses,
// this will still produce valid rss xml.
rss, err := feed.ToRss()
if err != nil {
err := gtserror.Newf("error converting feed to rss string: %w", err)
return "", gtserror.NewErrorInternalError(err)
}
return rss, nil
}

View file

@ -19,7 +19,10 @@ package account_test
import (
"testing"
"time"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gorilla/feeds"
"github.com/stretchr/testify/suite"
)
@ -28,13 +31,8 @@ type GetRSSTestSuite struct {
}
func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() {
getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(suite.T().Context(), "admin")
suite.NoError(err)
suite.EqualValues(1634726497, lastModified.Unix())
feed, err := getFeed()
suite.NoError(err)
suite.Equal(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1634726497,
`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Posts from @admin@localhost:8080</title>
<link>http://localhost:8080/@admin</link>
@ -63,17 +61,94 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() {
<source>http://localhost:8080/@admin/feed.rss</source>
</item>
</channel>
</rss>`, feed)
</rss>`)
}
func (suite *GetRSSTestSuite) TestGetAccountAtomAdmin() {
suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToAtom, 1634726497,
`<?xml version="1.0" encoding="UTF-8"?><feed xmlns="http://www.w3.org/2005/Atom">
<title>Posts from @admin@localhost:8080</title>
<id>http://localhost:8080/@admin</id>
<updated>2021-10-20T10:41:37Z</updated>
<subtitle>Posts from @admin@localhost:8080</subtitle>
<link href="http://localhost:8080/@admin"></link>
<entry>
<title>open to see some &lt;strong&gt;puppies&lt;/strong&gt;</title>
<updated>2021-10-20T12:36:45Z</updated>
<id>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</id>
<content type="html">&lt;p&gt;🐕🐕🐕🐕🐕&lt;/p&gt;</content>
<link href="http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37" rel="alternate"></link>
<link href="" rel="enclosure"></link>
<summary type="html">@admin@localhost:8080 made a new post: &#34;🐕🐕🐕🐕🐕&#34;</summary>
<author>
<name>@admin@localhost:8080</name>
</author>
</entry>
<entry>
<title>hello world! #welcome ! first post on the instance :rainbow: !</title>
<updated>2021-10-20T11:36:45Z</updated>
<id>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</id>
<content type="html">&lt;p&gt;hello world! &lt;a href=&#34;http://localhost:8080/tags/welcome&#34; class=&#34;mention hashtag&#34; rel=&#34;tag nofollow noreferrer noopener&#34; target=&#34;_blank&#34;&gt;#&lt;span&gt;welcome&lt;/span&gt;&lt;/a&gt; ! first post on the instance &lt;img src=&#34;http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png&#34; title=&#34;:rainbow:&#34; alt=&#34;:rainbow:&#34; width=&#34;25&#34; height=&#34;25&#34; /&gt; !&lt;/p&gt;</content>
<link href="http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R" rel="alternate"></link>
<link href="http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg" rel="enclosure" type="image/jpeg" length="62529"></link>
<summary type="html">@admin@localhost:8080 posted 1 attachment: &#34;hello world! #welcome ! first post on the instance :rainbow: !&#34;</summary>
<author>
<name>@admin@localhost:8080</name>
</author>
</entry>
</feed>`)
}
func (suite *GetRSSTestSuite) TestGetAccountJSONAdmin() {
suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToJSON, 1634726497,
`{
"version": "https://jsonfeed.org/version/1.1",
"title": "Posts from @admin@localhost:8080",
"home_page_url": "http://localhost:8080/@admin",
"description": "Posts from @admin@localhost:8080",
"items": [
{
"id": "http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37",
"url": "http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37",
"external_url": "http://localhost:8080/@admin/feed.rss",
"title": "open to see some \u003cstrong\u003epuppies\u003c/strong\u003e",
"content_html": "\u003cp\u003e🐕🐕🐕🐕🐕\u003c/p\u003e",
"summary": "@admin@localhost:8080 made a new post: \"🐕🐕🐕🐕🐕\"",
"date_published": "2021-10-20T12:36:45Z",
"author": {
"name": "@admin@localhost:8080"
},
"authors": [
{
"name": "@admin@localhost:8080"
}
]
},
{
"id": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
"url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R",
"external_url": "http://localhost:8080/@admin/feed.rss",
"title": "hello world! #welcome ! first post on the instance :rainbow: !",
"content_html": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance \u003cimg src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" width=\"25\" height=\"25\" /\u003e !\u003c/p\u003e",
"summary": "@admin@localhost:8080 posted 1 attachment: \"hello world! #welcome ! first post on the instance :rainbow: !\"",
"image": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg",
"date_published": "2021-10-20T11:36:45Z",
"author": {
"name": "@admin@localhost:8080"
},
"authors": [
{
"name": "@admin@localhost:8080"
}
]
}
]
}`)
}
func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(suite.T().Context(), "the_mighty_zork")
suite.NoError(err)
suite.EqualValues(1730451600, lastModified.Unix())
feed, err := getFeed()
suite.NoError(err)
suite.Equal(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1730451600,
`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Posts from @the_mighty_zork@localhost:8080</title>
<link>http://localhost:8080/@the_mighty_zork</link>
@ -151,7 +226,154 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() {
<source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
</item>
</channel>
</rss>`, feed)
</rss>`)
}
func (suite *GetRSSTestSuite) TestGetAccountAtomZork() {
suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1730451600,
`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Posts from @the_mighty_zork@localhost:8080</title>
<link>http://localhost:8080/@the_mighty_zork</link>
<description>Posts from @the_mighty_zork@localhost:8080</description>
<pubDate>Fri, 01 Nov 2024 09:00:00 +0000</pubDate>
<lastBuildDate>Fri, 01 Nov 2024 09:00:00 +0000</lastBuildDate>
<image>
<url>http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.webp</url>
<title>Avatar for @the_mighty_zork@localhost:8080</title>
<link>http://localhost:8080/@the_mighty_zork</link>
</image>
<item>
<title>edited status</title>
<link>http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR</link>
<description>@the_mighty_zork@localhost:8080 made a new post: &#34;this is the latest revision of the status, with a content-warning&#34;</description>
<content:encoded><![CDATA[<p>this is the latest revision of the status, with a content-warning</p>]]></content:encoded>
<author>@the_mighty_zork@localhost:8080</author>
<guid isPermaLink="true">http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR</guid>
<pubDate>Fri, 01 Nov 2024 09:00:00 +0000</pubDate>
<source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
</item>
<item>
<title>HTML in post</title>
<link>http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40</link>
<description>@the_mighty_zork@localhost:8080 made a new post: &#34;Here&#39;s a bunch of HTML, read it and weep, weep then!&#xA;&#xA;`+"```"+`html&#xA;&lt;section class=&#34;about-user&#34;&gt;&#xA; &lt;div class=&#34;col-header&#34;&gt;&#xA; &lt;h2&gt;About&lt;/h2&gt;&#xA; &lt;/div&gt; &#xA; &lt;div class=&#34;fields&#34;&gt;&#xA; &lt;h3 class=&#34;sr-only&#34;&gt;Fields&lt;/h3&gt;&#xA; &lt;dl&gt;&#xA;...</description>
<content:encoded><![CDATA[<p>Here's a bunch of HTML, read it and weep, weep then!</p><pre><code class="language-html">&lt;section class=&#34;about-user&#34;&gt;
&lt;div class=&#34;col-header&#34;&gt;
&lt;h2&gt;About&lt;/h2&gt;
&lt;/div&gt;
&lt;div class=&#34;fields&#34;&gt;
&lt;h3 class=&#34;sr-only&#34;&gt;Fields&lt;/h3&gt;
&lt;dl&gt;
&lt;div class=&#34;field&#34;&gt;
&lt;dt&gt;should you follow me?&lt;/dt&gt;
&lt;dd&gt;maybe!&lt;/dd&gt;
&lt;/div&gt;
&lt;div class=&#34;field&#34;&gt;
&lt;dt&gt;age&lt;/dt&gt;
&lt;dd&gt;120&lt;/dd&gt;
&lt;/div&gt;
&lt;/dl&gt;
&lt;/div&gt;
&lt;div class=&#34;bio&#34;&gt;
&lt;h3 class=&#34;sr-only&#34;&gt;Bio&lt;/h3&gt;
&lt;p&gt;i post about things that concern me&lt;/p&gt;
&lt;/div&gt;
&lt;div class=&#34;sr-only&#34; role=&#34;group&#34;&gt;
&lt;h3 class=&#34;sr-only&#34;&gt;Stats&lt;/h3&gt;
&lt;span&gt;Joined in Jun, 2022.&lt;/span&gt;
&lt;span&gt;8 posts.&lt;/span&gt;
&lt;span&gt;Followed by 1.&lt;/span&gt;
&lt;span&gt;Following 1.&lt;/span&gt;
&lt;/div&gt;
&lt;div class=&#34;accountstats&#34; aria-hidden=&#34;true&#34;&gt;
&lt;b&gt;Joined&lt;/b&gt;&lt;time datetime=&#34;2022-06-04T13:12:00.000Z&#34;&gt;Jun, 2022&lt;/time&gt;
&lt;b&gt;Posts&lt;/b&gt;&lt;span&gt;8&lt;/span&gt;
&lt;b&gt;Followed by&lt;/b&gt;&lt;span&gt;1&lt;/span&gt;
&lt;b&gt;Following&lt;/b&gt;&lt;span&gt;1&lt;/span&gt;
&lt;/div&gt;
&lt;/section&gt;
</code></pre><p>There, hope you liked that!</p>]]></content:encoded>
<author>@the_mighty_zork@localhost:8080</author>
<guid isPermaLink="true">http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40</guid>
<pubDate>Sun, 10 Dec 2023 09:24:00 +0000</pubDate>
<source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
</item>
<item>
<title>introduction post</title>
<link>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</link>
<description>@the_mighty_zork@localhost:8080 made a new post: &#34;hello everyone!&#34;</description>
<content:encoded><![CDATA[<p>hello everyone!</p>]]></content:encoded>
<author>@the_mighty_zork@localhost:8080</author>
<guid isPermaLink="true">http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</guid>
<pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>
<source>http://localhost:8080/@the_mighty_zork/feed.rss</source>
</item>
</channel>
</rss>`)
}
func (suite *GetRSSTestSuite) TestGetAccountJSONZork() {
suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToJSON, 1730451600,
`{
"version": "https://jsonfeed.org/version/1.1",
"title": "Posts from @the_mighty_zork@localhost:8080",
"home_page_url": "http://localhost:8080/@the_mighty_zork",
"description": "Posts from @the_mighty_zork@localhost:8080",
"items": [
{
"id": "http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR",
"url": "http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR",
"external_url": "http://localhost:8080/@the_mighty_zork/feed.rss",
"title": "edited status",
"content_html": "\u003cp\u003ethis is the latest revision of the status, with a content-warning\u003c/p\u003e",
"summary": "@the_mighty_zork@localhost:8080 made a new post: \"this is the latest revision of the status, with a content-warning\"",
"date_published": "2024-11-01T09:00:00Z",
"date_modified": "2024-11-01T09:02:00Z",
"author": {
"name": "@the_mighty_zork@localhost:8080"
},
"authors": [
{
"name": "@the_mighty_zork@localhost:8080"
}
]
},
{
"id": "http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40",
"url": "http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40",
"external_url": "http://localhost:8080/@the_mighty_zork/feed.rss",
"title": "HTML in post",
"content_html": "\u003cp\u003eHere's a bunch of HTML, read it and weep, weep then!\u003c/p\u003e\u003cpre\u003e\u003ccode class=\"language-html\"\u003e\u0026lt;section class=\u0026#34;about-user\u0026#34;\u0026gt;\n \u0026lt;div class=\u0026#34;col-header\u0026#34;\u0026gt;\n \u0026lt;h2\u0026gt;About\u0026lt;/h2\u0026gt;\n \u0026lt;/div\u0026gt; \n \u0026lt;div class=\u0026#34;fields\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Fields\u0026lt;/h3\u0026gt;\n \u0026lt;dl\u0026gt;\n \u0026lt;div class=\u0026#34;field\u0026#34;\u0026gt;\n \u0026lt;dt\u0026gt;should you follow me?\u0026lt;/dt\u0026gt;\n \u0026lt;dd\u0026gt;maybe!\u0026lt;/dd\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;field\u0026#34;\u0026gt;\n \u0026lt;dt\u0026gt;age\u0026lt;/dt\u0026gt;\n \u0026lt;dd\u0026gt;120\u0026lt;/dd\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;/dl\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;bio\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Bio\u0026lt;/h3\u0026gt;\n \u0026lt;p\u0026gt;i post about things that concern me\u0026lt;/p\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;sr-only\u0026#34; role=\u0026#34;group\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Stats\u0026lt;/h3\u0026gt;\n \u0026lt;span\u0026gt;Joined in Jun, 2022.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;8 posts.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;Followed by 1.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;Following 1.\u0026lt;/span\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;accountstats\u0026#34; aria-hidden=\u0026#34;true\u0026#34;\u0026gt;\n \u0026lt;b\u0026gt;Joined\u0026lt;/b\u0026gt;\u0026lt;time datetime=\u0026#34;2022-06-04T13:12:00.000Z\u0026#34;\u0026gt;Jun, 2022\u0026lt;/time\u0026gt;\n \u0026lt;b\u0026gt;Posts\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;8\u0026lt;/span\u0026gt;\n \u0026lt;b\u0026gt;Followed by\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;1\u0026lt;/span\u0026gt;\n \u0026lt;b\u0026gt;Following\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;1\u0026lt;/span\u0026gt;\n \u0026lt;/div\u0026gt;\n\u0026lt;/section\u0026gt;\n\u003c/code\u003e\u003c/pre\u003e\u003cp\u003eThere, hope you liked that!\u003c/p\u003e",
"summary": "@the_mighty_zork@localhost:8080 made a new post: \"Here's a bunch of HTML, read it and weep, weep then!\n\n`+"```"+`html\n\u003csection class=\"about-user\"\u003e\n \u003cdiv class=\"col-header\"\u003e\n \u003ch2\u003eAbout\u003c/h2\u003e\n \u003c/div\u003e \n \u003cdiv class=\"fields\"\u003e\n \u003ch3 class=\"sr-only\"\u003eFields\u003c/h3\u003e\n \u003cdl\u003e\n...",
"date_published": "2023-12-10T09:24:00Z",
"author": {
"name": "@the_mighty_zork@localhost:8080"
},
"authors": [
{
"name": "@the_mighty_zork@localhost:8080"
}
]
},
{
"id": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
"url": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
"external_url": "http://localhost:8080/@the_mighty_zork/feed.rss",
"title": "introduction post",
"content_html": "\u003cp\u003ehello everyone!\u003c/p\u003e",
"summary": "@the_mighty_zork@localhost:8080 made a new post: \"hello everyone!\"",
"date_published": "2021-10-20T10:40:37Z",
"author": {
"name": "@the_mighty_zork@localhost:8080"
},
"authors": [
{
"name": "@the_mighty_zork@localhost:8080"
}
]
}
]
}`)
}
func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() {
@ -170,13 +392,10 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() {
}
}
getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(ctx, "the_mighty_zork")
suite.NoError(err)
suite.Empty(lastModified)
var zeroTime time.Time
feed, err := getFeed()
suite.NoError(err)
suite.Equal(`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, zeroTime.Unix(),
`<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Posts from @the_mighty_zork@localhost:8080</title>
<link>http://localhost:8080/@the_mighty_zork</link>
@ -189,7 +408,33 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() {
<link>http://localhost:8080/@the_mighty_zork</link>
</image>
</channel>
</rss>`, feed)
</rss>`)
}
// func (suite *GetRSSTestSuite) testGetAccountRSSPaging(username string, page *paging.Page, expectIDs []string) {
// ctx := suite.T().Context()
// getFeed, _, errWithCode := suite.accountProcessor.GetRSSFeedForUsername(ctx, username, page)
// suite.NoError(errWithCode)
// feed, errWithCode := getFeed()
// suite.NoError(errWithCode)
// }
func (suite *GetRSSTestSuite) testGetFeedSerializedAs(username string, page *paging.Page, serialize func(*feeds.Feed) (string, error), expectLastMod int64, expectSerialized string) {
ctx := suite.T().Context()
getFeed, lastMod, errWithCode := suite.accountProcessor.GetRSSFeedForUsername(ctx, username, page)
suite.NoError(errWithCode)
suite.Equal(expectLastMod, lastMod.Unix())
feed, errWithCode := getFeed()
suite.NoError(errWithCode)
feedStr, err := serialize(feed)
suite.NoError(err)
suite.Equal(expectSerialized, feedStr)
}
func TestGetRSSTestSuite(t *testing.T) {

View file

@ -27,6 +27,7 @@ import (
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/gtsmodel"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"code.superseriousbusiness.org/gotosocial/internal/util"
)
@ -156,9 +157,8 @@ func (p *Processor) StatusesGet(
func (p *Processor) WebStatusesGet(
ctx context.Context,
targetAccountID string,
page *paging.Page,
mediaOnly bool,
limit int,
maxID string,
) (*apimodel.PageableResponse, gtserror.WithCode) {
account, err := p.state.DB.GetAccountByID(ctx, targetAccountID)
if err != nil {
@ -174,14 +174,13 @@ func (p *Processor) WebStatusesGet(
return nil, gtserror.NewErrorNotFound(err)
}
statuses, err := p.state.DB.GetAccountWebStatuses(
ctx,
statuses, err := p.state.DB.GetAccountWebStatuses(ctx,
account,
page,
mediaOnly,
limit,
maxID,
)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
err := gtserror.Newf("db error getting statuses: %w", err)
return nil, gtserror.NewErrorInternalError(err)
}

View file

@ -82,7 +82,7 @@ func getAssetETag(
return cachedETag.eTag, nil
}
eTag, err := generateEtag(file)
eTag, err := generateETag(file)
if err != nil {
return "", fmt.Errorf("error generating etag: %s", err)
}

View file

@ -27,6 +27,7 @@ import (
"code.superseriousbusiness.org/gotosocial/internal/log"
"codeberg.org/gruf/go-cache/v3"
"codeberg.org/gruf/go-fastcopy"
)
type withETagCache interface {
@ -47,13 +48,19 @@ type eTagCacheEntry struct {
lastModified time.Time
}
// generateEtag generates a strong (byte-for-byte) etag using
// the entirety of the provided reader.
func generateEtag(r io.Reader) (string, error) {
// nolint:gosec
hash := sha1.New()
// generateEtag generates a strong (byte-for-byte) etag
// using the entirety of the provided reader.
func generateETag(r io.Reader) (string, error) {
return generateETagFrom(func(w io.Writer) error {
_, err := fastcopy.Copy(w, r)
return err
})
}
if _, err := io.Copy(hash, r); err != nil {
func generateETagFrom(writeTo func(io.Writer) error) (string, error) {
hash := sha1.New() // nolint:gosec
if err := writeTo(hash); err != nil {
return "", err
}

View file

@ -27,6 +27,7 @@ import (
apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gin-gonic/gin"
)
@ -122,14 +123,15 @@ func (m *Module) prepareProfile(c *gin.Context) *profile {
// Check if paging.
maxStatusID := apiutil.ParseMaxID(c.Query(apiutil.MaxIDKey), "")
paging := maxStatusID != ""
doPaging := (maxStatusID != "")
// If not paging, load pinned statuses.
var (
mediaOnly = account.WebLayout == "gallery"
pinnedStatuses []*apimodel.WebStatus
)
if !paging {
if !doPaging {
// If not paging, load pinned statuses.
var errWithCode gtserror.WithCode
pinnedStatuses, errWithCode = m.processor.Account().WebStatusesGetPinned(
ctx,
@ -156,9 +158,8 @@ func (m *Module) prepareProfile(c *gin.Context) *profile {
statusResp, errWithCode := m.processor.Account().WebStatusesGet(
ctx,
account.ID,
&paging.Page{Max: paging.MaxID(maxStatusID), Limit: limit},
mediaOnly,
limit,
maxStatusID,
)
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, instanceGet)
@ -172,7 +173,7 @@ func (m *Module) prepareProfile(c *gin.Context) *profile {
robotsMeta: robotsMeta,
pinnedStatuses: pinnedStatuses,
statusResp: statusResp,
paging: paging,
paging: doPaging,
}
}

View file

@ -18,7 +18,6 @@
package web
import (
"bytes"
"net/http"
"strings"
"time"
@ -26,13 +25,26 @@ import (
apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util"
"code.superseriousbusiness.org/gotosocial/internal/gtserror"
"code.superseriousbusiness.org/gotosocial/internal/log"
"code.superseriousbusiness.org/gotosocial/internal/paging"
"github.com/gin-gonic/gin"
"github.com/gorilla/feeds"
)
const appRSSUTF8 = string(apiutil.AppRSSXML) + "; charset=utf-8"
const (
charsetUTF8 = "; charset=utf-8"
appRSSUTF8 = string(apiutil.AppRSSXML) + charsetUTF8
appAtomUTF8 = string(apiutil.AppAtomXML) + charsetUTF8
appJSONUTF8 = string(apiutil.AppFeedJSON) + charsetUTF8
)
func (m *Module) rssFeedGETHandler(c *gin.Context) {
if _, err := apiutil.NegotiateAccept(c, apiutil.AppRSSXML); err != nil {
contentType, err := apiutil.NegotiateAccept(c,
apiutil.AppRSSXML,
apiutil.AppAtomXML,
apiutil.AppFeedJSON,
apiutil.AppJSON,
)
if err != nil {
apiutil.WebErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1)
return
}
@ -49,21 +61,34 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) {
// todo: https://codeberg.org/superseriousbusiness/gotosocial/issues/1813
username = strings.ToLower(username)
// Retrieve the getRSSFeed function from the processor.
// We'll only call the function if we need to, to save db calls.
// lastPostAt may be a zero time if account has never posted.
getRSSFeed, lastPostAt, errWithCode := m.processor.Account().GetRSSFeedForUsername(c.Request.Context(), username)
// Parse paging parameters from request.
page, errWithCode := paging.ParseIDPage(c,
1, // min limit
40, // max limit
20, // default limit
)
if errWithCode != nil {
apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
getFunc, lastPostAt, errWithCode := m.processor.Account().GetRSSFeedForUsername(
c.Request.Context(),
username,
page,
)
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
var (
rssFeed string // Stringified rss feed.
var feed *feeds.Feed
cacheKey = c.Request.URL.Path
cacheEntry, wasCached = m.eTagCache.Get(cacheKey)
)
// Key to use in etag cache (note content-type suffix).
cacheKey := c.Request.URL.Path + "#" + contentType
// Check etag cache for an existing entry under key.
cacheEntry, wasCached := m.eTagCache.Get(cacheKey)
if !wasCached || unixAfter(lastPostAt, cacheEntry.lastModified) {
// We either have no ETag cache entry for this account's feed,
@ -72,15 +97,16 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) {
//
// As such, we need to generate a new ETag, and for that we need
// the string representation of the RSS feed.
rssFeed, errWithCode = getRSSFeed()
feed, errWithCode = getFunc()
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
eTag, err := generateEtag(bytes.NewBufferString(rssFeed))
etag, err := generateFeedETag(feed, contentType)
if err != nil {
apiutil.WebErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGetV1)
errWithCode := gtserror.NewErrorInternalError(err)
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
@ -96,7 +122,7 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) {
// Store the new cache entry.
cacheEntry = eTagCacheEntry{
eTag: eTag,
eTag: etag,
lastModified: lastModified,
}
m.eTagCache.Set(cacheKey, cacheEntry)
@ -149,15 +175,37 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) {
// If we had a cache hit earlier, we may not have called the
// getRSSFeed function yet; if that's the case then do call it
// now because we definitely need it.
if rssFeed == "" {
rssFeed, errWithCode = getRSSFeed()
if feed == nil {
feed, errWithCode = getFunc()
if errWithCode != nil {
apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
return
}
}
c.Data(http.StatusOK, appRSSUTF8, []byte(rssFeed))
// Encode response.
switch contentType {
case apiutil.AppRSSXML:
apiutil.XMLType(c, http.StatusOK, appRSSUTF8, &feeds.Rss{feed})
case apiutil.AppAtomXML:
apiutil.XMLType(c, http.StatusOK, appAtomUTF8, &feeds.Atom{feed})
case apiutil.AppFeedJSON, apiutil.AppJSON:
apiutil.JSONType(c, http.StatusOK, appJSONUTF8, (&feeds.JSON{feed}).JSONFeed())
}
}
// generateFeedETag generates feed etag for appropriate content-type encoding.
func generateFeedETag(feed *feeds.Feed, contentType string) (string, error) {
switch contentType {
case apiutil.AppRSSXML:
return generateETagFrom(feed.WriteRss)
case apiutil.AppAtomXML:
return generateETagFrom(feed.WriteAtom)
case apiutil.AppFeedJSON, apiutil.AppJSON:
return generateETagFrom(feed.WriteJSON)
default:
panic("unreachable")
}
}
// unixAfter returns true if the unix value of t1