| 
									
										
										
										
											2023-03-12 16:00:57 +01:00
										 |  |  | // GoToSocial | 
					
						
							|  |  |  | // Copyright (C) GoToSocial Authors admin@gotosocial.org | 
					
						
							|  |  |  | // SPDX-License-Identifier: AGPL-3.0-or-later | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // This program is free software: you can redistribute it and/or modify | 
					
						
							|  |  |  | // it under the terms of the GNU Affero General Public License as published by | 
					
						
							|  |  |  | // the Free Software Foundation, either version 3 of the License, or | 
					
						
							|  |  |  | // (at your option) any later version. | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // This program is distributed in the hope that it will be useful, | 
					
						
							|  |  |  | // but WITHOUT ANY WARRANTY; without even the implied warranty of | 
					
						
							|  |  |  | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | 
					
						
							|  |  |  | // GNU Affero General Public License for more details. | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // You should have received a copy of the GNU Affero General Public License | 
					
						
							|  |  |  | // along with this program.  If not, see <http://www.gnu.org/licenses/>. | 
					
						
							| 
									
										
										
										
											2021-05-30 13:12:00 +02:00
										 |  |  | 
 | 
					
						
							|  |  |  | package processing | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 
 | 
					
						
							|  |  |  | import ( | 
					
						
							| 
									
										
										
										
											2021-08-25 15:34:33 +02:00
										 |  |  | 	"context" | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 	"errors" | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 	"fmt" | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" | 
					
						
							| 
									
										
										
										
											2021-05-30 13:12:00 +02:00
										 |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/db" | 
					
						
							| 
									
										
										
										
											2021-06-13 18:42:28 +02:00
										 |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/gtserror" | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" | 
					
						
							| 
									
										
										
										
											2022-07-19 09:47:55 +01:00
										 |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/log" | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/oauth" | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/timeline" | 
					
						
							|  |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/typeutils" | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/util" | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/visibility" | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | ) | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | const boostReinsertionDepth = 50 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // StatusGrabFunction returns a function that satisfies the GrabFunction interface in internal/timeline. | 
					
						
							|  |  |  | func StatusGrabFunction(database db.DB) timeline.GrabFunction { | 
					
						
							|  |  |  | 	return func(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int) ([]timeline.Timelineable, bool, error) { | 
					
						
							|  |  |  | 		statuses, err := database.GetHomeTimeline(ctx, timelineAccountID, maxID, sinceID, minID, limit, false) | 
					
						
							|  |  |  | 		if err != nil { | 
					
						
							|  |  |  | 			if err == db.ErrNoEntries { | 
					
						
							|  |  |  | 				return nil, true, nil // we just don't have enough statuses left in the db so return stop = true | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 			return nil, false, fmt.Errorf("statusGrabFunction: error getting statuses from db: %s", err) | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | 		items := make([]timeline.Timelineable, len(statuses)) | 
					
						
							|  |  |  | 		for i, s := range statuses { | 
					
						
							|  |  |  | 			items[i] = s | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		return items, false, nil | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // StatusFilterFunction returns a function that satisfies the FilterFunction interface in internal/timeline. | 
					
						
							| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | func StatusFilterFunction(database db.DB, filter *visibility.Filter) timeline.FilterFunction { | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 	return func(ctx context.Context, timelineAccountID string, item timeline.Timelineable) (shouldIndex bool, err error) { | 
					
						
							|  |  |  | 		status, ok := item.(*gtsmodel.Status) | 
					
						
							|  |  |  | 		if !ok { | 
					
						
							|  |  |  | 			return false, errors.New("statusFilterFunction: could not convert item to *gtsmodel.Status") | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		requestingAccount, err := database.GetAccountByID(ctx, timelineAccountID) | 
					
						
							|  |  |  | 		if err != nil { | 
					
						
							|  |  |  | 			return false, fmt.Errorf("statusFilterFunction: error getting account with id %s", timelineAccountID) | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | 		timelineable, err := filter.StatusHomeTimelineable(ctx, requestingAccount, status) | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 		if err != nil { | 
					
						
							| 
									
										
										
										
											2023-02-17 12:02:29 +01:00
										 |  |  | 			log.Warnf(ctx, "error checking hometimelineability of status %s for account %s: %s", status.ID, timelineAccountID, err) | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		return timelineable, nil // we don't return the error here because we want to just skip this item if something goes wrong | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // StatusPrepareFunction returns a function that satisfies the PrepareFunction interface in internal/timeline. | 
					
						
							|  |  |  | func StatusPrepareFunction(database db.DB, tc typeutils.TypeConverter) timeline.PrepareFunction { | 
					
						
							|  |  |  | 	return func(ctx context.Context, timelineAccountID string, itemID string) (timeline.Preparable, error) { | 
					
						
							|  |  |  | 		status, err := database.GetStatusByID(ctx, itemID) | 
					
						
							|  |  |  | 		if err != nil { | 
					
						
							|  |  |  | 			return nil, fmt.Errorf("statusPrepareFunction: error getting status with id %s", itemID) | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		requestingAccount, err := database.GetAccountByID(ctx, timelineAccountID) | 
					
						
							|  |  |  | 		if err != nil { | 
					
						
							|  |  |  | 			return nil, fmt.Errorf("statusPrepareFunction: error getting account with id %s", timelineAccountID) | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		return tc.StatusToAPIStatus(ctx, status, requestingAccount) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | // StatusSkipInsertFunction returns a function that satisifes the SkipInsertFunction interface in internal/timeline. | 
					
						
							|  |  |  | func StatusSkipInsertFunction() timeline.SkipInsertFunction { | 
					
						
							|  |  |  | 	return func( | 
					
						
							|  |  |  | 		ctx context.Context, | 
					
						
							|  |  |  | 		newItemID string, | 
					
						
							|  |  |  | 		newItemAccountID string, | 
					
						
							|  |  |  | 		newItemBoostOfID string, | 
					
						
							|  |  |  | 		newItemBoostOfAccountID string, | 
					
						
							|  |  |  | 		nextItemID string, | 
					
						
							|  |  |  | 		nextItemAccountID string, | 
					
						
							|  |  |  | 		nextItemBoostOfID string, | 
					
						
							|  |  |  | 		nextItemBoostOfAccountID string, | 
					
						
							| 
									
										
										
										
											2022-05-30 13:41:24 +01:00
										 |  |  | 		depth int, | 
					
						
							|  |  |  | 	) (bool, error) { | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 		// make sure we don't insert a duplicate | 
					
						
							|  |  |  | 		if newItemID == nextItemID { | 
					
						
							|  |  |  | 			return true, nil | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		// check if it's a boost | 
					
						
							|  |  |  | 		if newItemBoostOfID != "" { | 
					
						
							|  |  |  | 			// skip if we've recently put another boost of this status in the timeline | 
					
						
							|  |  |  | 			if newItemBoostOfID == nextItemBoostOfID { | 
					
						
							|  |  |  | 				if depth < boostReinsertionDepth { | 
					
						
							|  |  |  | 					return true, nil | 
					
						
							|  |  |  | 				} | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 			// skip if we've recently put the original status in the timeline | 
					
						
							|  |  |  | 			if newItemBoostOfID == nextItemID { | 
					
						
							|  |  |  | 				if depth < boostReinsertionDepth { | 
					
						
							|  |  |  | 					return true, nil | 
					
						
							|  |  |  | 				} | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		// insert the item | 
					
						
							|  |  |  | 		return false, nil | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-02-22 16:05:26 +01:00
										 |  |  | func (p *Processor) HomeTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.PageableResponse, gtserror.WithCode) { | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 	preparedItems, err := p.statusTimelines.GetTimeline(ctx, authed.Account.ID, maxID, sinceID, minID, limit, local) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 	if err != nil { | 
					
						
							|  |  |  | 		return nil, gtserror.NewErrorInternalError(err) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 	count := len(preparedItems) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	if count == 0 { | 
					
						
							|  |  |  | 		return util.EmptyPageableResponse(), nil | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 	items := []interface{}{} | 
					
						
							|  |  |  | 	nextMaxIDValue := "" | 
					
						
							|  |  |  | 	prevMinIDValue := "" | 
					
						
							|  |  |  | 	for i, item := range preparedItems { | 
					
						
							|  |  |  | 		if i == count-1 { | 
					
						
							|  |  |  | 			nextMaxIDValue = item.GetID() | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 		} | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 
 | 
					
						
							|  |  |  | 		if i == 0 { | 
					
						
							|  |  |  | 			prevMinIDValue = item.GetID() | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 		items = append(items, item) | 
					
						
							| 
									
										
										
										
											2022-02-05 12:47:38 +01:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 	return util.PackagePageableResponse(util.PageableResponseParams{ | 
					
						
							|  |  |  | 		Items:          items, | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 		Path:           "api/v1/timelines/home", | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 		NextMaxIDValue: nextMaxIDValue, | 
					
						
							|  |  |  | 		PrevMinIDValue: prevMinIDValue, | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 		Limit:          limit, | 
					
						
							|  |  |  | 	}) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-02-22 16:05:26 +01:00
										 |  |  | func (p *Processor) PublicTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.PageableResponse, gtserror.WithCode) { | 
					
						
							| 
									
										
										
										
											2023-03-01 18:26:53 +00:00
										 |  |  | 	statuses, err := p.state.DB.GetPublicTimeline(ctx, maxID, sinceID, minID, limit, local) | 
					
						
							| 
									
										
										
										
											2021-05-31 17:36:35 +02:00
										 |  |  | 	if err != nil { | 
					
						
							| 
									
										
										
										
											2021-08-20 12:26:56 +02:00
										 |  |  | 		if err == db.ErrNoEntries { | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 			// there are just no entries left | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 			return util.EmptyPageableResponse(), nil | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 		} | 
					
						
							|  |  |  | 		// there's an actual error | 
					
						
							|  |  |  | 		return nil, gtserror.NewErrorInternalError(err) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 	filtered, err := p.filterPublicStatuses(ctx, authed, statuses) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 	if err != nil { | 
					
						
							|  |  |  | 		return nil, gtserror.NewErrorInternalError(err) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 	count := len(filtered) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	if count == 0 { | 
					
						
							|  |  |  | 		return util.EmptyPageableResponse(), nil | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 	items := []interface{}{} | 
					
						
							|  |  |  | 	nextMaxIDValue := "" | 
					
						
							|  |  |  | 	prevMinIDValue := "" | 
					
						
							|  |  |  | 	for i, item := range filtered { | 
					
						
							|  |  |  | 		if i == count-1 { | 
					
						
							|  |  |  | 			nextMaxIDValue = item.GetID() | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		if i == 0 { | 
					
						
							|  |  |  | 			prevMinIDValue = item.GetID() | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 		items = append(items, item) | 
					
						
							| 
									
										
										
										
											2021-08-25 15:34:33 +02:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 	return util.PackagePageableResponse(util.PageableResponseParams{ | 
					
						
							|  |  |  | 		Items:          items, | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 		Path:           "api/v1/timelines/public", | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 		NextMaxIDValue: nextMaxIDValue, | 
					
						
							|  |  |  | 		PrevMinIDValue: prevMinIDValue, | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 		Limit:          limit, | 
					
						
							|  |  |  | 	}) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-02-22 16:05:26 +01:00
										 |  |  | func (p *Processor) FavedTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, minID string, limit int) (*apimodel.PageableResponse, gtserror.WithCode) { | 
					
						
							| 
									
										
										
										
											2023-03-01 18:26:53 +00:00
										 |  |  | 	statuses, nextMaxID, prevMinID, err := p.state.DB.GetFavedTimeline(ctx, authed.Account.ID, maxID, minID, limit) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 	if err != nil { | 
					
						
							| 
									
										
										
										
											2021-08-20 12:26:56 +02:00
										 |  |  | 		if err == db.ErrNoEntries { | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 			// there are just no entries left | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 			return util.EmptyPageableResponse(), nil | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 		} | 
					
						
							|  |  |  | 		// there's an actual error | 
					
						
							| 
									
										
										
										
											2021-06-13 18:42:28 +02:00
										 |  |  | 		return nil, gtserror.NewErrorInternalError(err) | 
					
						
							| 
									
										
										
										
											2021-05-31 17:36:35 +02:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 	filtered, err := p.filterFavedStatuses(ctx, authed, statuses) | 
					
						
							| 
									
										
										
										
											2021-05-31 17:36:35 +02:00
										 |  |  | 	if err != nil { | 
					
						
							| 
									
										
										
										
											2021-06-13 18:42:28 +02:00
										 |  |  | 		return nil, gtserror.NewErrorInternalError(err) | 
					
						
							| 
									
										
										
										
											2021-05-31 17:36:35 +02:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 	if len(filtered) == 0 { | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 		return util.EmptyPageableResponse(), nil | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 	items := []interface{}{} | 
					
						
							|  |  |  | 	for _, item := range filtered { | 
					
						
							|  |  |  | 		items = append(items, item) | 
					
						
							| 
									
										
										
										
											2021-08-25 15:34:33 +02:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-10-10 15:52:49 +02:00
										 |  |  | 	return util.PackagePageableResponse(util.PageableResponseParams{ | 
					
						
							|  |  |  | 		Items:          items, | 
					
						
							| 
									
										
										
										
											2022-06-08 20:22:49 +02:00
										 |  |  | 		Path:           "api/v1/favourites", | 
					
						
							|  |  |  | 		NextMaxIDValue: nextMaxID, | 
					
						
							|  |  |  | 		PrevMinIDValue: prevMinID, | 
					
						
							|  |  |  | 		Limit:          limit, | 
					
						
							|  |  |  | 	}) | 
					
						
							| 
									
										
										
										
											2021-05-31 17:36:35 +02:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-02-22 16:05:26 +01:00
										 |  |  | func (p *Processor) filterPublicStatuses(ctx context.Context, authed *oauth.Auth, statuses []*gtsmodel.Status) ([]*apimodel.Status, error) { | 
					
						
							| 
									
										
										
										
											2021-06-13 18:42:28 +02:00
										 |  |  | 	apiStatuses := []*apimodel.Status{} | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 	for _, s := range statuses { | 
					
						
							| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | 		if _, err := p.state.DB.GetAccountByID(ctx, s.AccountID); err != nil { | 
					
						
							| 
									
										
										
										
											2021-08-20 12:26:56 +02:00
										 |  |  | 			if err == db.ErrNoEntries { | 
					
						
							| 
									
										
										
										
											2023-02-17 12:02:29 +01:00
										 |  |  | 				log.Debugf(ctx, "skipping status %s because account %s can't be found in the db", s.ID, s.AccountID) | 
					
						
							| 
									
										
										
										
											2021-05-30 13:12:00 +02:00
										 |  |  | 				continue | 
					
						
							|  |  |  | 			} | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 			return nil, gtserror.NewErrorInternalError(fmt.Errorf("filterPublicStatuses: error getting status author: %s", err)) | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | 		timelineable, err := p.filter.StatusPublicTimelineable(ctx, authed.Account, s) | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 		if err != nil { | 
					
						
							| 
									
										
										
										
											2023-02-17 12:02:29 +01:00
										 |  |  | 			log.Debugf(ctx, "skipping status %s because of an error checking status visibility: %s", s.ID, err) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 			continue | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 		if !timelineable { | 
					
						
							|  |  |  | 			continue | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-10-04 15:24:19 +02:00
										 |  |  | 		apiStatus, err := p.tc.StatusToAPIStatus(ctx, s, authed.Account) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 		if err != nil { | 
					
						
							| 
									
										
										
										
											2023-02-17 12:02:29 +01:00
										 |  |  | 			log.Debugf(ctx, "skipping status %s because it couldn't be converted to its api representation: %s", s.ID, err) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 			continue | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		apiStatuses = append(apiStatuses, apiStatus) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	return apiStatuses, nil | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2023-02-22 16:05:26 +01:00
										 |  |  | func (p *Processor) filterFavedStatuses(ctx context.Context, authed *oauth.Auth, statuses []*gtsmodel.Status) ([]*apimodel.Status, error) { | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 	apiStatuses := []*apimodel.Status{} | 
					
						
							|  |  |  | 	for _, s := range statuses { | 
					
						
							| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | 		if _, err := p.state.DB.GetAccountByID(ctx, s.AccountID); err != nil { | 
					
						
							| 
									
										
										
										
											2021-08-20 12:26:56 +02:00
										 |  |  | 			if err == db.ErrNoEntries { | 
					
						
							| 
									
										
										
										
											2023-02-17 12:02:29 +01:00
										 |  |  | 				log.Debugf(ctx, "skipping status %s because account %s can't be found in the db", s.ID, s.AccountID) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 				continue | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 			return nil, gtserror.NewErrorInternalError(fmt.Errorf("filterPublicStatuses: error getting status author: %s", err)) | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | 		timelineable, err := p.filter.StatusVisible(ctx, authed.Account, s) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 		if err != nil { | 
					
						
							| 
									
										
										
										
											2023-02-17 12:02:29 +01:00
										 |  |  | 			log.Debugf(ctx, "skipping status %s because of an error checking status visibility: %s", s.ID, err) | 
					
						
							| 
									
										
										
										
											2021-07-09 18:32:48 +02:00
										 |  |  | 			continue | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 		} | 
					
						
							| 
									
										
										
										
											2021-06-17 18:02:33 +02:00
										 |  |  | 		if !timelineable { | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 			continue | 
					
						
							|  |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-10-04 15:24:19 +02:00
										 |  |  | 		apiStatus, err := p.tc.StatusToAPIStatus(ctx, s, authed.Account) | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 		if err != nil { | 
					
						
							| 
									
										
										
										
											2023-02-17 12:02:29 +01:00
										 |  |  | 			log.Debugf(ctx, "skipping status %s because it couldn't be converted to its api representation: %s", s.ID, err) | 
					
						
							| 
									
										
										
										
											2021-05-30 13:12:00 +02:00
										 |  |  | 			continue | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 		} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-06-13 18:42:28 +02:00
										 |  |  | 		apiStatuses = append(apiStatuses, apiStatus) | 
					
						
							| 
									
										
										
										
											2021-05-21 23:04:59 +02:00
										 |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	return apiStatuses, nil | 
					
						
							|  |  |  | } |