From 6607e1c9444d0814b72762a46814ff0812d96343 Mon Sep 17 00:00:00 2001 From: kim Date: Thu, 18 Sep 2025 16:33:23 +0200 Subject: [PATCH] [feature] add paging support to rss feed endpoint, and support JSON / atom feed types (#4442) originally based on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4396 hope this is okay https://codeberg.org/zordsdavini ! closes https://codeberg.org/superseriousbusiness/gotosocial/issues/4411 closes https://codeberg.org/superseriousbusiness/gotosocial/issues/3407 Co-authored-by: Arnas Udovic Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4442 Co-authored-by: kim Co-committed-by: kim --- docs/user_guide/rss.md | 6 +- internal/api/util/mime.go | 2 + internal/api/util/response.go | 13 ++ internal/db/account.go | 2 +- internal/db/bundb/account.go | 109 +++------ internal/db/bundb/account_test.go | 2 +- internal/processing/account/rss.go | 73 +++--- internal/processing/account/rss_test.go | 291 ++++++++++++++++++++++-- internal/processing/account/statuses.go | 11 +- internal/web/assets.go | 2 +- internal/web/etag.go | 19 +- internal/web/profile.go | 13 +- internal/web/rss.go | 86 +++++-- 13 files changed, 447 insertions(+), 182 deletions(-) diff --git a/docs/user_guide/rss.md b/docs/user_guide/rss.md index 1ec543cfb..5bcdb43c3 100644 --- a/docs/user_guide/rss.md +++ b/docs/user_guide/rss.md @@ -12,4 +12,8 @@ When enabled, the RSS feed for your account will be available at `https://[your- ## Which posts are shared via RSS? -Only your latest 20 Public posts are shared via RSS. Replies and reblogs/boosts are not included. Unlisted posts are not included. In other words, the only posts visible via RSS will be the same ones that are visible when you open your profile in a browser. +Only your latest 20 Public posts are shared via RSS by default. Replies and reblogs/boosts are not included. Unlisted posts are not included. In other words, the only posts visible via RSS will be the same ones that are visible when you open your profile in a browser. + +If you want to see more posts, you can provide our standard set of timeline paging parameters ([as per our swagger documentation](https://docs.gotosocial.org/en/latest/api/swagger)) to see beyond the first page. + +You can also access Atom and JSON feeds from this same endpoint, but providing the appropriate request content-type header. i.e. `application/atom+xml` for an Atom feed, or `application/feed+json` for a JSON feed. diff --git a/internal/api/util/mime.go b/internal/api/util/mime.go index da96be786..6159d4b79 100644 --- a/internal/api/util/mime.go +++ b/internal/api/util/mime.go @@ -26,6 +26,8 @@ const ( appXMLText = `text/xml` // AppXML is only *recommended* in RFC7303 AppXMLXRD = `application/xrd+xml` AppRSSXML = `application/rss+xml` + AppAtomXML = `application/atom+xml` + AppFeedJSON = `application/feed+json` AppActivityJSON = `application/activity+json` appActivityLDJSON = `application/ld+json` // without profile AppActivityLDJSON = appActivityLDJSON + `; profile="https://www.w3.org/ns/activitystreams"` diff --git a/internal/api/util/response.go b/internal/api/util/response.go index 105537f36..ff58b68e3 100644 --- a/internal/api/util/response.go +++ b/internal/api/util/response.go @@ -71,6 +71,18 @@ func JSONType(c *gin.Context, code int, contentType string, data any) { EncodeJSONResponse(c.Writer, c.Request, code, contentType, data) } +// XML calls EncodeJSONResponse() using gin.Context{}, with content-type = AppXML, +// This function handles the case of XML unmarshal errors and pools read buffers. +func XML(c *gin.Context, code int, data any) { + EncodeXMLResponse(c.Writer, c.Request, code, AppXML, data) +} + +// XML calls EncodeXMLResponse() using gin.Context{}, with given content-type. +// This function handles the case of XML unmarshal errors and pools read buffers. +func XMLType(c *gin.Context, code int, contentType string, data any) { + EncodeXMLResponse(c.Writer, c.Request, code, contentType, data) +} + // Data calls WriteResponseBytes() using gin.Context{}, with given content-type. func Data(c *gin.Context, code int, contentType string, data []byte) { WriteResponseBytes(c.Writer, c.Request, code, contentType, data) @@ -230,6 +242,7 @@ func EncodeCSVResponse( // Write all the records to the buffer. if err := csvWriter.WriteAll(records); err == nil { + // Respond with the now-known // size byte slice within buf. WriteResponseBytes(rw, r, diff --git a/internal/db/account.go b/internal/db/account.go index 59ea9ff1a..45893a5cc 100644 --- a/internal/db/account.go +++ b/internal/db/account.go @@ -121,7 +121,7 @@ type Account interface { // returning statuses that should be visible via the web view of a *LOCAL* account. // // In the case of no statuses, this function will return db.ErrNoEntries. - GetAccountWebStatuses(ctx context.Context, account *gtsmodel.Account, mediaOnly bool, limit int, maxID string) ([]*gtsmodel.Status, error) + GetAccountWebStatuses(ctx context.Context, account *gtsmodel.Account, page *paging.Page, mediaOnly bool) ([]*gtsmodel.Status, error) // GetInstanceAccount returns the instance account for the given domain. // If domain is empty, this instance account will be returned. diff --git a/internal/db/bundb/account.go b/internal/db/bundb/account.go index 603740f17..8276016d7 100644 --- a/internal/db/bundb/account.go +++ b/internal/db/bundb/account.go @@ -900,7 +900,7 @@ func (a *accountDB) GetAccountFaves(ctx context.Context, accountID string) ([]*g return *faves, nil } -func qMediaOnly(q *bun.SelectQuery) *bun.SelectQuery { +func selectOnlyWithMedia(q *bun.SelectQuery) *bun.SelectQuery { // Attachments are stored as a json object; this // implementation differs between SQLite and Postgres, // so we have to be thorough to cover all eventualities @@ -908,14 +908,14 @@ func qMediaOnly(q *bun.SelectQuery) *bun.SelectQuery { switch d := q.Dialect().Name(); d { case dialect.PG: return q. - Where("? IS NOT NULL", bun.Ident("status.attachments")). - Where("? != '{}'", bun.Ident("status.attachments")) + Where("? IS NOT NULL", bun.Ident("attachments")). + Where("? != '{}'", bun.Ident("attachments")) case dialect.SQLite: return q. - Where("? IS NOT NULL", bun.Ident("status.attachments")). - Where("? != 'null'", bun.Ident("status.attachments")). - Where("? != '[]'", bun.Ident("status.attachments")) + Where("? IS NOT NULL", bun.Ident("attachments")). + Where("? != 'null'", bun.Ident("attachments")). + Where("? != '[]'", bun.Ident("attachments")) default: panic("dialect " + d.String() + " was neither pg nor sqlite") @@ -963,9 +963,9 @@ func (a *accountDB) GetAccountStatuses(ctx context.Context, accountID string, li q = q.Where("? IS NULL", bun.Ident("status.boost_of_id")) } - // Respect media-only preference. if mediaOnly { - q = qMediaOnly(q) + // Respect mediaOnly pref. + q = selectOnlyWithMedia(q) } if publicOnly { @@ -1041,12 +1041,16 @@ func (a *accountDB) GetAccountPinnedStatuses(ctx context.Context, accountID stri return a.state.DB.GetStatusesByIDs(ctx, statusIDs) } +var webStatusVisibilities = bun.In([]gtsmodel.Visibility{ + gtsmodel.VisibilityPublic, + gtsmodel.VisibilityUnlocked, +}) + func (a *accountDB) GetAccountWebStatuses( ctx context.Context, account *gtsmodel.Account, + page *paging.Page, mediaOnly bool, - limit int, - maxID string, ) ([]*gtsmodel.Status, error) { if account.Username == config.GetHost() { // Instance account @@ -1071,74 +1075,35 @@ func (a *accountDB) GetAccountWebStatuses( return nil, nil } - // Ensure reasonable - if limit < 0 { - limit = 0 - } + return loadStatusTimelinePage(ctx, a.db, a.state, - // Make educated guess for slice size - statusIDs := make([]string, 0, limit) + // Paging + // params. + page, - q := a.db. - NewSelect(). - TableExpr("? AS ?", bun.Ident("statuses"), bun.Ident("status")). - // Select only IDs from table - Column("status.id"). - Where("? = ?", bun.Ident("status.account_id"), account.ID) + // The actual meat of the account web statuses query. + func(q *bun.SelectQuery) (*bun.SelectQuery, error) { + q = q.Where("? = ?", bun.Ident("account_id"), account.ID) - // Select statuses according to - // account's web visibility prefs. - if publicOnly { - // Only Public statuses. - q = q.Where("? = ?", bun.Ident("status.visibility"), gtsmodel.VisibilityPublic) - } else { - // Public or Unlocked. - visis := []gtsmodel.Visibility{ - gtsmodel.VisibilityPublic, - gtsmodel.VisibilityUnlocked, - } - q = q.Where("? IN (?)", bun.Ident("status.visibility"), bun.In(visis)) - } + if publicOnly { + q = q.Where("? = ?", bun.Ident("visibility"), gtsmodel.VisibilityPublic) + } else { + q = q.Where("? IN (?)", bun.Ident("visibility"), webStatusVisibilities) + } - // Don't show replies, boosts, or - // local-only statuses on the web view. - q = q. - Where("? IS NULL", bun.Ident("status.in_reply_to_uri")). - Where("? IS NULL", bun.Ident("status.boost_of_id")). - Where("? = ?", bun.Ident("status.federated"), true) + // Don't show replies, boosts, or local-only in web view. + q = q.Where("? IS NULL", bun.Ident("in_reply_to_uri")). + Where("? IS NULL", bun.Ident("boost_of_id")). + Where("? = ?", bun.Ident("federated"), true) - // Respect media-only preference. - if mediaOnly { - q = qMediaOnly(q) - } + if mediaOnly { + // Respect mediaOnly pref. + q = selectOnlyWithMedia(q) + } - // Return only statuses LOWER (ie., older) than maxID - if maxID == "" { - maxID = id.Highest - } - q = q.Where("? < ?", bun.Ident("status.id"), maxID) - - if limit > 0 { - // limit amount of statuses returned - q = q.Limit(limit) - } - - if limit > 0 { - // limit amount of statuses returned - q = q.Limit(limit) - } - - q = q.Order("status.id DESC") - - if err := q.Scan(ctx, &statusIDs); err != nil { - return nil, err - } - - if len(statusIDs) == 0 { - return nil, db.ErrNoEntries - } - - return a.state.DB.GetStatusesByIDs(ctx, statusIDs) + return q, nil + }, + ) } func (a *accountDB) GetAccountSettings( diff --git a/internal/db/bundb/account_test.go b/internal/db/bundb/account_test.go index 4c4cad3dd..fb86b5d5d 100644 --- a/internal/db/bundb/account_test.go +++ b/internal/db/bundb/account_test.go @@ -49,7 +49,7 @@ func (suite *AccountTestSuite) TestGetAccountStatuses() { } func (suite *AccountTestSuite) TestGetAccountWebStatusesMediaOnly() { - statuses, err := suite.db.GetAccountWebStatuses(suite.T().Context(), suite.testAccounts["local_account_3"], true, 20, "") + statuses, err := suite.db.GetAccountWebStatuses(suite.T().Context(), suite.testAccounts["local_account_3"], &paging.Page{Limit: 20}, true) suite.NoError(err) suite.Len(statuses, 2) } diff --git a/internal/processing/account/rss.go b/internal/processing/account/rss.go index 495aa2e54..205027528 100644 --- a/internal/processing/account/rss.go +++ b/internal/processing/account/rss.go @@ -20,21 +20,19 @@ package account import ( "context" "errors" - "fmt" "time" "code.superseriousbusiness.org/gotosocial/internal/config" "code.superseriousbusiness.org/gotosocial/internal/db" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" + "code.superseriousbusiness.org/gotosocial/internal/paging" "github.com/gorilla/feeds" ) -const ( - rssFeedLength = 20 -) +var never time.Time -type GetRSSFeed func() (string, gtserror.WithCode) +type GetRSSFeed func() (*feeds.Feed, gtserror.WithCode) // GetRSSFeedForUsername returns a function to return the RSS feed of a local account // with the given username, and the last-modified time (time that the account last @@ -45,33 +43,30 @@ type GetRSSFeed func() (string, gtserror.WithCode) // // If the account has not yet posted an RSS-eligible status, the returned last-modified // time will be zero, and the GetRSSFeed func will return a valid RSS xml with no items. -func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string) (GetRSSFeed, time.Time, gtserror.WithCode) { - var ( - never = time.Time{} - ) +func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string, page *paging.Page) (GetRSSFeed, time.Time, gtserror.WithCode) { + // Fetch local (i.e. empty domain) account from database by username. account, err := p.state.DB.GetAccountByUsernameDomain(ctx, username, "") if err != nil { - if errors.Is(err, db.ErrNoEntries) { - // Simply no account with this username. - err = gtserror.New("account not found") - return nil, never, gtserror.NewErrorNotFound(err) - } - - // Real db error. - err = gtserror.Newf("db error getting account %s: %w", username, err) + err := gtserror.Newf("db error getting account %s: %w", username, err) return nil, never, gtserror.NewErrorInternalError(err) } + // Check if exists. + if account == nil { + err := gtserror.New("account not found") + return nil, never, gtserror.NewErrorNotFound(err) + } + // Ensure account has rss feed enabled. if !*account.Settings.EnableRSS { - err = gtserror.New("account RSS feed not enabled") + err := gtserror.New("account RSS feed not enabled") return nil, never, gtserror.NewErrorNotFound(err) } - // Ensure account stats populated. + // Ensure account stats populated for last status fetch information. if err := p.state.DB.PopulateAccountStats(ctx, account); err != nil { - err = gtserror.Newf("db error getting account stats %s: %w", username, err) + err := gtserror.Newf("db error getting account stats %s: %w", username, err) return nil, never, gtserror.NewErrorInternalError(err) } @@ -80,14 +75,14 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string) // eligible to appear in the RSS feed; that's fine. lastPostAt := account.Stats.LastStatusAt - return func() (string, gtserror.WithCode) { + return func() (*feeds.Feed, gtserror.WithCode) { // Assemble author namestring once only. author := "@" + account.Username + "@" + config.GetAccountDomain() - // Derive image/thumbnail for this account (may be nil). + // Derive image/thumbnail for this account (may be nil if no media). image, errWithCode := p.rssImageForAccount(ctx, account, author) if errWithCode != nil { - return "", errWithCode + return nil, errWithCode } feed := &feeds.Feed{ @@ -106,7 +101,7 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string) // since we already know there's no eligible statuses. if lastPostAt.IsZero() { feed.Updated = account.CreatedAt - return stringifyFeed(feed) + return feed, nil } // Account has posted at least one status that's @@ -120,32 +115,30 @@ func (p *Processor) GetRSSFeedForUsername(ctx context.Context, username string) // // Take into account whether the user wants // their web view laid out in gallery mode. - mediaOnly := account.Settings != nil && - account.Settings.WebLayout == gtsmodel.WebLayoutGallery + mediaOnly := (account.Settings != nil && + account.Settings.WebLayout == gtsmodel.WebLayoutGallery) statuses, err := p.state.DB.GetAccountWebStatuses( ctx, account, + page, mediaOnly, - rssFeedLength, - "", // Latest posts from the top. ) if err != nil && !errors.Is(err, db.ErrNoEntries) { - err = fmt.Errorf("db error getting account web statuses: %w", err) - return "", gtserror.NewErrorInternalError(err) + err := gtserror.Newf("db error getting account web statuses: %w", err) + return nil, gtserror.NewErrorInternalError(err) } // Add each status to the rss feed. for _, status := range statuses { item, err := p.converter.StatusToRSSItem(ctx, status) if err != nil { - err = gtserror.Newf("error converting status to feed item: %w", err) - return "", gtserror.NewErrorInternalError(err) + err := gtserror.Newf("error converting status to feed item: %w", err) + return nil, gtserror.NewErrorInternalError(err) } - feed.Add(item) } - return stringifyFeed(feed) + return feed, nil }, lastPostAt, nil } @@ -177,15 +170,3 @@ func (p *Processor) rssImageForAccount(ctx context.Context, account *gtsmodel.Ac Link: account.URL, }, nil } - -func stringifyFeed(feed *feeds.Feed) (string, gtserror.WithCode) { - // Stringify the feed. Even with no statuses, - // this will still produce valid rss xml. - rss, err := feed.ToRss() - if err != nil { - err := gtserror.Newf("error converting feed to rss string: %w", err) - return "", gtserror.NewErrorInternalError(err) - } - - return rss, nil -} diff --git a/internal/processing/account/rss_test.go b/internal/processing/account/rss_test.go index 0b64e8464..b053a3795 100644 --- a/internal/processing/account/rss_test.go +++ b/internal/processing/account/rss_test.go @@ -19,7 +19,10 @@ package account_test import ( "testing" + "time" + "code.superseriousbusiness.org/gotosocial/internal/paging" + "github.com/gorilla/feeds" "github.com/stretchr/testify/suite" ) @@ -28,13 +31,8 @@ type GetRSSTestSuite struct { } func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { - getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(suite.T().Context(), "admin") - suite.NoError(err) - suite.EqualValues(1634726497, lastModified.Unix()) - - feed, err := getFeed() - suite.NoError(err) - suite.Equal(` + suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1634726497, + ` Posts from @admin@localhost:8080 http://localhost:8080/@admin @@ -63,17 +61,94 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { http://localhost:8080/@admin/feed.rss -`, feed) +`) +} + +func (suite *GetRSSTestSuite) TestGetAccountAtomAdmin() { + suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToAtom, 1634726497, + ` + Posts from @admin@localhost:8080 + http://localhost:8080/@admin + 2021-10-20T10:41:37Z + Posts from @admin@localhost:8080 + + + open to see some <strong>puppies</strong> + 2021-10-20T12:36:45Z + http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37 + <p>🐕🐕🐕🐕🐕</p> + + + @admin@localhost:8080 made a new post: "🐕🐕🐕🐕🐕" + + @admin@localhost:8080 + + + + hello world! #welcome ! first post on the instance :rainbow: ! + 2021-10-20T11:36:45Z + http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R + <p>hello world! <a href="http://localhost:8080/tags/welcome" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>welcome</span></a> ! first post on the instance <img src="http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png" title=":rainbow:" alt=":rainbow:" width="25" height="25" /> !</p> + + + @admin@localhost:8080 posted 1 attachment: "hello world! #welcome ! first post on the instance :rainbow: !" + + @admin@localhost:8080 + + +`) +} + +func (suite *GetRSSTestSuite) TestGetAccountJSONAdmin() { + suite.testGetFeedSerializedAs("admin", &paging.Page{Limit: 20}, (*feeds.Feed).ToJSON, 1634726497, + `{ + "version": "https://jsonfeed.org/version/1.1", + "title": "Posts from @admin@localhost:8080", + "home_page_url": "http://localhost:8080/@admin", + "description": "Posts from @admin@localhost:8080", + "items": [ + { + "id": "http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37", + "url": "http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37", + "external_url": "http://localhost:8080/@admin/feed.rss", + "title": "open to see some \u003cstrong\u003epuppies\u003c/strong\u003e", + "content_html": "\u003cp\u003e🐕🐕🐕🐕🐕\u003c/p\u003e", + "summary": "@admin@localhost:8080 made a new post: \"🐕🐕🐕🐕🐕\"", + "date_published": "2021-10-20T12:36:45Z", + "author": { + "name": "@admin@localhost:8080" + }, + "authors": [ + { + "name": "@admin@localhost:8080" + } + ] + }, + { + "id": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R", + "url": "http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R", + "external_url": "http://localhost:8080/@admin/feed.rss", + "title": "hello world! #welcome ! first post on the instance :rainbow: !", + "content_html": "\u003cp\u003ehello world! \u003ca href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\"\u003e#\u003cspan\u003ewelcome\u003c/span\u003e\u003c/a\u003e ! first post on the instance \u003cimg src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" width=\"25\" height=\"25\" /\u003e !\u003c/p\u003e", + "summary": "@admin@localhost:8080 posted 1 attachment: \"hello world! #welcome ! first post on the instance :rainbow: !\"", + "image": "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", + "date_published": "2021-10-20T11:36:45Z", + "author": { + "name": "@admin@localhost:8080" + }, + "authors": [ + { + "name": "@admin@localhost:8080" + } + ] + } + ] +}`) } func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { - getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(suite.T().Context(), "the_mighty_zork") - suite.NoError(err) - suite.EqualValues(1730451600, lastModified.Unix()) - - feed, err := getFeed() - suite.NoError(err) - suite.Equal(` + suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1730451600, + ` Posts from @the_mighty_zork@localhost:8080 http://localhost:8080/@the_mighty_zork @@ -151,7 +226,154 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { http://localhost:8080/@the_mighty_zork/feed.rss -`, feed) +`) +} + +func (suite *GetRSSTestSuite) TestGetAccountAtomZork() { + suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, 1730451600, + ` + + Posts from @the_mighty_zork@localhost:8080 + http://localhost:8080/@the_mighty_zork + Posts from @the_mighty_zork@localhost:8080 + Fri, 01 Nov 2024 09:00:00 +0000 + Fri, 01 Nov 2024 09:00:00 +0000 + + http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.webp + Avatar for @the_mighty_zork@localhost:8080 + http://localhost:8080/@the_mighty_zork + + + edited status + http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR + @the_mighty_zork@localhost:8080 made a new post: "this is the latest revision of the status, with a content-warning" + this is the latest revision of the status, with a content-warning

]]> + @the_mighty_zork@localhost:8080 + http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR + Fri, 01 Nov 2024 09:00:00 +0000 + http://localhost:8080/@the_mighty_zork/feed.rss + + + HTML in post + http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40 + @the_mighty_zork@localhost:8080 made a new post: "Here's a bunch of HTML, read it and weep, weep then! `+"```"+`html <section class="about-user"> <div class="col-header"> <h2>About</h2> </div> <div class="fields"> <h3 class="sr-only">Fields</h3> <dl> ... + Here's a bunch of HTML, read it and weep, weep then!

<section class="about-user">
+    <div class="col-header">
+        <h2>About</h2>
+    </div>            
+    <div class="fields">
+        <h3 class="sr-only">Fields</h3>
+        <dl>
+            <div class="field">
+                <dt>should you follow me?</dt>
+                <dd>maybe!</dd>
+            </div>
+            <div class="field">
+                <dt>age</dt>
+                <dd>120</dd>
+            </div>
+        </dl>
+    </div>
+    <div class="bio">
+        <h3 class="sr-only">Bio</h3>
+        <p>i post about things that concern me</p>
+    </div>
+    <div class="sr-only" role="group">
+        <h3 class="sr-only">Stats</h3>
+        <span>Joined in Jun, 2022.</span>
+        <span>8 posts.</span>
+        <span>Followed by 1.</span>
+        <span>Following 1.</span>
+    </div>
+    <div class="accountstats" aria-hidden="true">
+        <b>Joined</b><time datetime="2022-06-04T13:12:00.000Z">Jun, 2022</time>
+        <b>Posts</b><span>8</span>
+        <b>Followed by</b><span>1</span>
+        <b>Following</b><span>1</span>
+    </div>
+</section>
+

There, hope you liked that!

]]> + @the_mighty_zork@localhost:8080 + http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40 + Sun, 10 Dec 2023 09:24:00 +0000 + http://localhost:8080/@the_mighty_zork/feed.rss + + + introduction post + http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY + @the_mighty_zork@localhost:8080 made a new post: "hello everyone!" + hello everyone!

]]> + @the_mighty_zork@localhost:8080 + http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY + Wed, 20 Oct 2021 10:40:37 +0000 + http://localhost:8080/@the_mighty_zork/feed.rss + + +`) +} + +func (suite *GetRSSTestSuite) TestGetAccountJSONZork() { + suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToJSON, 1730451600, + `{ + "version": "https://jsonfeed.org/version/1.1", + "title": "Posts from @the_mighty_zork@localhost:8080", + "home_page_url": "http://localhost:8080/@the_mighty_zork", + "description": "Posts from @the_mighty_zork@localhost:8080", + "items": [ + { + "id": "http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR", + "url": "http://localhost:8080/@the_mighty_zork/statuses/01JDPZC707CKDN8N4QVWM4Z1NR", + "external_url": "http://localhost:8080/@the_mighty_zork/feed.rss", + "title": "edited status", + "content_html": "\u003cp\u003ethis is the latest revision of the status, with a content-warning\u003c/p\u003e", + "summary": "@the_mighty_zork@localhost:8080 made a new post: \"this is the latest revision of the status, with a content-warning\"", + "date_published": "2024-11-01T09:00:00Z", + "date_modified": "2024-11-01T09:02:00Z", + "author": { + "name": "@the_mighty_zork@localhost:8080" + }, + "authors": [ + { + "name": "@the_mighty_zork@localhost:8080" + } + ] + }, + { + "id": "http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40", + "url": "http://localhost:8080/@the_mighty_zork/statuses/01HH9KYNQPA416TNJ53NSATP40", + "external_url": "http://localhost:8080/@the_mighty_zork/feed.rss", + "title": "HTML in post", + "content_html": "\u003cp\u003eHere's a bunch of HTML, read it and weep, weep then!\u003c/p\u003e\u003cpre\u003e\u003ccode class=\"language-html\"\u003e\u0026lt;section class=\u0026#34;about-user\u0026#34;\u0026gt;\n \u0026lt;div class=\u0026#34;col-header\u0026#34;\u0026gt;\n \u0026lt;h2\u0026gt;About\u0026lt;/h2\u0026gt;\n \u0026lt;/div\u0026gt; \n \u0026lt;div class=\u0026#34;fields\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Fields\u0026lt;/h3\u0026gt;\n \u0026lt;dl\u0026gt;\n \u0026lt;div class=\u0026#34;field\u0026#34;\u0026gt;\n \u0026lt;dt\u0026gt;should you follow me?\u0026lt;/dt\u0026gt;\n \u0026lt;dd\u0026gt;maybe!\u0026lt;/dd\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;field\u0026#34;\u0026gt;\n \u0026lt;dt\u0026gt;age\u0026lt;/dt\u0026gt;\n \u0026lt;dd\u0026gt;120\u0026lt;/dd\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;/dl\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;bio\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Bio\u0026lt;/h3\u0026gt;\n \u0026lt;p\u0026gt;i post about things that concern me\u0026lt;/p\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;sr-only\u0026#34; role=\u0026#34;group\u0026#34;\u0026gt;\n \u0026lt;h3 class=\u0026#34;sr-only\u0026#34;\u0026gt;Stats\u0026lt;/h3\u0026gt;\n \u0026lt;span\u0026gt;Joined in Jun, 2022.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;8 posts.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;Followed by 1.\u0026lt;/span\u0026gt;\n \u0026lt;span\u0026gt;Following 1.\u0026lt;/span\u0026gt;\n \u0026lt;/div\u0026gt;\n \u0026lt;div class=\u0026#34;accountstats\u0026#34; aria-hidden=\u0026#34;true\u0026#34;\u0026gt;\n \u0026lt;b\u0026gt;Joined\u0026lt;/b\u0026gt;\u0026lt;time datetime=\u0026#34;2022-06-04T13:12:00.000Z\u0026#34;\u0026gt;Jun, 2022\u0026lt;/time\u0026gt;\n \u0026lt;b\u0026gt;Posts\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;8\u0026lt;/span\u0026gt;\n \u0026lt;b\u0026gt;Followed by\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;1\u0026lt;/span\u0026gt;\n \u0026lt;b\u0026gt;Following\u0026lt;/b\u0026gt;\u0026lt;span\u0026gt;1\u0026lt;/span\u0026gt;\n \u0026lt;/div\u0026gt;\n\u0026lt;/section\u0026gt;\n\u003c/code\u003e\u003c/pre\u003e\u003cp\u003eThere, hope you liked that!\u003c/p\u003e", + "summary": "@the_mighty_zork@localhost:8080 made a new post: \"Here's a bunch of HTML, read it and weep, weep then!\n\n`+"```"+`html\n\u003csection class=\"about-user\"\u003e\n \u003cdiv class=\"col-header\"\u003e\n \u003ch2\u003eAbout\u003c/h2\u003e\n \u003c/div\u003e \n \u003cdiv class=\"fields\"\u003e\n \u003ch3 class=\"sr-only\"\u003eFields\u003c/h3\u003e\n \u003cdl\u003e\n...", + "date_published": "2023-12-10T09:24:00Z", + "author": { + "name": "@the_mighty_zork@localhost:8080" + }, + "authors": [ + { + "name": "@the_mighty_zork@localhost:8080" + } + ] + }, + { + "id": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY", + "url": "http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY", + "external_url": "http://localhost:8080/@the_mighty_zork/feed.rss", + "title": "introduction post", + "content_html": "\u003cp\u003ehello everyone!\u003c/p\u003e", + "summary": "@the_mighty_zork@localhost:8080 made a new post: \"hello everyone!\"", + "date_published": "2021-10-20T10:40:37Z", + "author": { + "name": "@the_mighty_zork@localhost:8080" + }, + "authors": [ + { + "name": "@the_mighty_zork@localhost:8080" + } + ] + } + ] +}`) } func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() { @@ -170,13 +392,10 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() { } } - getFeed, lastModified, err := suite.accountProcessor.GetRSSFeedForUsername(ctx, "the_mighty_zork") - suite.NoError(err) - suite.Empty(lastModified) + var zeroTime time.Time - feed, err := getFeed() - suite.NoError(err) - suite.Equal(` + suite.testGetFeedSerializedAs("the_mighty_zork", &paging.Page{Limit: 20}, (*feeds.Feed).ToRss, zeroTime.Unix(), + ` Posts from @the_mighty_zork@localhost:8080 http://localhost:8080/@the_mighty_zork @@ -189,7 +408,33 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZorkNoPosts() { http://localhost:8080/@the_mighty_zork -`, feed) +`) +} + +// func (suite *GetRSSTestSuite) testGetAccountRSSPaging(username string, page *paging.Page, expectIDs []string) { +// ctx := suite.T().Context() + +// getFeed, _, errWithCode := suite.accountProcessor.GetRSSFeedForUsername(ctx, username, page) +// suite.NoError(errWithCode) + +// feed, errWithCode := getFeed() +// suite.NoError(errWithCode) + +// } + +func (suite *GetRSSTestSuite) testGetFeedSerializedAs(username string, page *paging.Page, serialize func(*feeds.Feed) (string, error), expectLastMod int64, expectSerialized string) { + ctx := suite.T().Context() + + getFeed, lastMod, errWithCode := suite.accountProcessor.GetRSSFeedForUsername(ctx, username, page) + suite.NoError(errWithCode) + suite.Equal(expectLastMod, lastMod.Unix()) + + feed, errWithCode := getFeed() + suite.NoError(errWithCode) + + feedStr, err := serialize(feed) + suite.NoError(err) + suite.Equal(expectSerialized, feedStr) } func TestGetRSSTestSuite(t *testing.T) { diff --git a/internal/processing/account/statuses.go b/internal/processing/account/statuses.go index e55c1e81c..870019f41 100644 --- a/internal/processing/account/statuses.go +++ b/internal/processing/account/statuses.go @@ -27,6 +27,7 @@ import ( "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/gtsmodel" "code.superseriousbusiness.org/gotosocial/internal/log" + "code.superseriousbusiness.org/gotosocial/internal/paging" "code.superseriousbusiness.org/gotosocial/internal/util" ) @@ -156,9 +157,8 @@ func (p *Processor) StatusesGet( func (p *Processor) WebStatusesGet( ctx context.Context, targetAccountID string, + page *paging.Page, mediaOnly bool, - limit int, - maxID string, ) (*apimodel.PageableResponse, gtserror.WithCode) { account, err := p.state.DB.GetAccountByID(ctx, targetAccountID) if err != nil { @@ -174,14 +174,13 @@ func (p *Processor) WebStatusesGet( return nil, gtserror.NewErrorNotFound(err) } - statuses, err := p.state.DB.GetAccountWebStatuses( - ctx, + statuses, err := p.state.DB.GetAccountWebStatuses(ctx, account, + page, mediaOnly, - limit, - maxID, ) if err != nil && !errors.Is(err, db.ErrNoEntries) { + err := gtserror.Newf("db error getting statuses: %w", err) return nil, gtserror.NewErrorInternalError(err) } diff --git a/internal/web/assets.go b/internal/web/assets.go index 8e453850d..f58568be3 100644 --- a/internal/web/assets.go +++ b/internal/web/assets.go @@ -82,7 +82,7 @@ func getAssetETag( return cachedETag.eTag, nil } - eTag, err := generateEtag(file) + eTag, err := generateETag(file) if err != nil { return "", fmt.Errorf("error generating etag: %s", err) } diff --git a/internal/web/etag.go b/internal/web/etag.go index fcd55603b..42a6083f0 100644 --- a/internal/web/etag.go +++ b/internal/web/etag.go @@ -27,6 +27,7 @@ import ( "code.superseriousbusiness.org/gotosocial/internal/log" "codeberg.org/gruf/go-cache/v3" + "codeberg.org/gruf/go-fastcopy" ) type withETagCache interface { @@ -47,13 +48,19 @@ type eTagCacheEntry struct { lastModified time.Time } -// generateEtag generates a strong (byte-for-byte) etag using -// the entirety of the provided reader. -func generateEtag(r io.Reader) (string, error) { - // nolint:gosec - hash := sha1.New() +// generateEtag generates a strong (byte-for-byte) etag +// using the entirety of the provided reader. +func generateETag(r io.Reader) (string, error) { + return generateETagFrom(func(w io.Writer) error { + _, err := fastcopy.Copy(w, r) + return err + }) +} - if _, err := io.Copy(hash, r); err != nil { +func generateETagFrom(writeTo func(io.Writer) error) (string, error) { + hash := sha1.New() // nolint:gosec + + if err := writeTo(hash); err != nil { return "", err } diff --git a/internal/web/profile.go b/internal/web/profile.go index 98abe5741..458557b8b 100644 --- a/internal/web/profile.go +++ b/internal/web/profile.go @@ -27,6 +27,7 @@ import ( apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/log" + "code.superseriousbusiness.org/gotosocial/internal/paging" "github.com/gin-gonic/gin" ) @@ -122,14 +123,15 @@ func (m *Module) prepareProfile(c *gin.Context) *profile { // Check if paging. maxStatusID := apiutil.ParseMaxID(c.Query(apiutil.MaxIDKey), "") - paging := maxStatusID != "" + doPaging := (maxStatusID != "") - // If not paging, load pinned statuses. var ( mediaOnly = account.WebLayout == "gallery" pinnedStatuses []*apimodel.WebStatus ) - if !paging { + + if !doPaging { + // If not paging, load pinned statuses. var errWithCode gtserror.WithCode pinnedStatuses, errWithCode = m.processor.Account().WebStatusesGetPinned( ctx, @@ -156,9 +158,8 @@ func (m *Module) prepareProfile(c *gin.Context) *profile { statusResp, errWithCode := m.processor.Account().WebStatusesGet( ctx, account.ID, + &paging.Page{Max: paging.MaxID(maxStatusID), Limit: limit}, mediaOnly, - limit, - maxStatusID, ) if errWithCode != nil { apiutil.WebErrorHandler(c, errWithCode, instanceGet) @@ -172,7 +173,7 @@ func (m *Module) prepareProfile(c *gin.Context) *profile { robotsMeta: robotsMeta, pinnedStatuses: pinnedStatuses, statusResp: statusResp, - paging: paging, + paging: doPaging, } } diff --git a/internal/web/rss.go b/internal/web/rss.go index 006ba4ec1..d812ddc33 100644 --- a/internal/web/rss.go +++ b/internal/web/rss.go @@ -18,7 +18,6 @@ package web import ( - "bytes" "net/http" "strings" "time" @@ -26,13 +25,26 @@ import ( apiutil "code.superseriousbusiness.org/gotosocial/internal/api/util" "code.superseriousbusiness.org/gotosocial/internal/gtserror" "code.superseriousbusiness.org/gotosocial/internal/log" + "code.superseriousbusiness.org/gotosocial/internal/paging" "github.com/gin-gonic/gin" + "github.com/gorilla/feeds" ) -const appRSSUTF8 = string(apiutil.AppRSSXML) + "; charset=utf-8" +const ( + charsetUTF8 = "; charset=utf-8" + appRSSUTF8 = string(apiutil.AppRSSXML) + charsetUTF8 + appAtomUTF8 = string(apiutil.AppAtomXML) + charsetUTF8 + appJSONUTF8 = string(apiutil.AppFeedJSON) + charsetUTF8 +) func (m *Module) rssFeedGETHandler(c *gin.Context) { - if _, err := apiutil.NegotiateAccept(c, apiutil.AppRSSXML); err != nil { + contentType, err := apiutil.NegotiateAccept(c, + apiutil.AppRSSXML, + apiutil.AppAtomXML, + apiutil.AppFeedJSON, + apiutil.AppJSON, + ) + if err != nil { apiutil.WebErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGetV1) return } @@ -49,21 +61,34 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) { // todo: https://codeberg.org/superseriousbusiness/gotosocial/issues/1813 username = strings.ToLower(username) - // Retrieve the getRSSFeed function from the processor. - // We'll only call the function if we need to, to save db calls. - // lastPostAt may be a zero time if account has never posted. - getRSSFeed, lastPostAt, errWithCode := m.processor.Account().GetRSSFeedForUsername(c.Request.Context(), username) + // Parse paging parameters from request. + page, errWithCode := paging.ParseIDPage(c, + 1, // min limit + 40, // max limit + 20, // default limit + ) + if errWithCode != nil { + apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1) + return + } + + getFunc, lastPostAt, errWithCode := m.processor.Account().GetRSSFeedForUsername( + c.Request.Context(), + username, + page, + ) if errWithCode != nil { apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1) return } - var ( - rssFeed string // Stringified rss feed. + var feed *feeds.Feed - cacheKey = c.Request.URL.Path - cacheEntry, wasCached = m.eTagCache.Get(cacheKey) - ) + // Key to use in etag cache (note content-type suffix). + cacheKey := c.Request.URL.Path + "#" + contentType + + // Check etag cache for an existing entry under key. + cacheEntry, wasCached := m.eTagCache.Get(cacheKey) if !wasCached || unixAfter(lastPostAt, cacheEntry.lastModified) { // We either have no ETag cache entry for this account's feed, @@ -72,15 +97,16 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) { // // As such, we need to generate a new ETag, and for that we need // the string representation of the RSS feed. - rssFeed, errWithCode = getRSSFeed() + feed, errWithCode = getFunc() if errWithCode != nil { apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1) return } - eTag, err := generateEtag(bytes.NewBufferString(rssFeed)) + etag, err := generateFeedETag(feed, contentType) if err != nil { - apiutil.WebErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGetV1) + errWithCode := gtserror.NewErrorInternalError(err) + apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1) return } @@ -96,7 +122,7 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) { // Store the new cache entry. cacheEntry = eTagCacheEntry{ - eTag: eTag, + eTag: etag, lastModified: lastModified, } m.eTagCache.Set(cacheKey, cacheEntry) @@ -149,15 +175,37 @@ func (m *Module) rssFeedGETHandler(c *gin.Context) { // If we had a cache hit earlier, we may not have called the // getRSSFeed function yet; if that's the case then do call it // now because we definitely need it. - if rssFeed == "" { - rssFeed, errWithCode = getRSSFeed() + if feed == nil { + feed, errWithCode = getFunc() if errWithCode != nil { apiutil.WebErrorHandler(c, errWithCode, m.processor.InstanceGetV1) return } } - c.Data(http.StatusOK, appRSSUTF8, []byte(rssFeed)) + // Encode response. + switch contentType { + case apiutil.AppRSSXML: + apiutil.XMLType(c, http.StatusOK, appRSSUTF8, &feeds.Rss{feed}) + case apiutil.AppAtomXML: + apiutil.XMLType(c, http.StatusOK, appAtomUTF8, &feeds.Atom{feed}) + case apiutil.AppFeedJSON, apiutil.AppJSON: + apiutil.JSONType(c, http.StatusOK, appJSONUTF8, (&feeds.JSON{feed}).JSONFeed()) + } +} + +// generateFeedETag generates feed etag for appropriate content-type encoding. +func generateFeedETag(feed *feeds.Feed, contentType string) (string, error) { + switch contentType { + case apiutil.AppRSSXML: + return generateETagFrom(feed.WriteRss) + case apiutil.AppAtomXML: + return generateETagFrom(feed.WriteAtom) + case apiutil.AppFeedJSON, apiutil.AppJSON: + return generateETagFrom(feed.WriteJSON) + default: + panic("unreachable") + } } // unixAfter returns true if the unix value of t1