| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | // GoToSocial | 
					
						
							|  |  |  | // Copyright (C) GoToSocial Authors admin@gotosocial.org | 
					
						
							|  |  |  | // SPDX-License-Identifier: AGPL-3.0-or-later | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // This program is free software: you can redistribute it and/or modify | 
					
						
							|  |  |  | // it under the terms of the GNU Affero General Public License as published by | 
					
						
							|  |  |  | // the Free Software Foundation, either version 3 of the License, or | 
					
						
							|  |  |  | // (at your option) any later version. | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // This program is distributed in the hope that it will be useful, | 
					
						
							|  |  |  | // but WITHOUT ANY WARRANTY; without even the implied warranty of | 
					
						
							|  |  |  | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the | 
					
						
							|  |  |  | // GNU Affero General Public License for more details. | 
					
						
							|  |  |  | // | 
					
						
							|  |  |  | // You should have received a copy of the GNU Affero General Public License | 
					
						
							|  |  |  | // along with this program.  If not, see <http://www.gnu.org/licenses/>. | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | package migrations | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | import ( | 
					
						
							|  |  |  | 	"context" | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel" | 
					
						
							|  |  |  | 	"github.com/uptrace/bun" | 
					
						
							|  |  |  | ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | func init() { | 
					
						
							|  |  |  | 	up := func(ctx context.Context, db *bun.DB) error { | 
					
						
							| 
									
										
										
										
											2023-03-28 22:55:51 +02:00
										 |  |  | 		// To update not null constraint on public key, we need to migrate accounts into a new table. | 
					
						
							| 
									
										
											  
											
												[performance] refactoring + add fave / follow / request / visibility caching (#1607)
* refactor visibility checking, add caching for visibility
* invalidate visibility cache items on account / status deletes
* fix requester ID passed to visibility cache nil ptr
* de-interface caches, fix home / public timeline caching + visibility
* finish adding code comments for visibility filter
* fix angry goconst linter warnings
* actually finish adding filter visibility code comments for timeline functions
* move home timeline status author check to after visibility
* remove now-unused code
* add more code comments
* add TODO code comment, update printed cache start names
* update printed cache names on stop
* start adding separate follow(request) delete db functions, add specific visibility cache tests
* add relationship type caching
* fix getting local account follows / followed-bys, other small codebase improvements
* simplify invalidation using cache hooks, add more GetAccountBy___() functions
* fix boosting to return 404 if not boostable but no error (to not leak status ID)
* remove dead code
* improved placement of cache invalidation
* update license headers
* add example follow, follow-request config entries
* add example visibility cache configuration to config file
* use specific PutFollowRequest() instead of just Put()
* add tests for all GetAccountBy()
* add GetBlockBy() tests
* update block to check primitive fields
* update and finish adding Get{Account,Block,Follow,FollowRequest}By() tests
* fix copy-pasted code
* update envparsing test
* whitespace
* fix bun struct tag
* add license header to gtscontext
* fix old license header
* improved error creation to not use fmt.Errorf() when not needed
* fix various rebase conflicts, fix account test
* remove commented-out code, fix-up mention caching
* fix mention select bun statement
* ensure mention target account populated, pass in context to customrenderer logging
* remove more uncommented code, fix typeutil test
* add statusfave database model caching
* add status fave cache configuration
* add status fave cache example config
* woops, catch missed error. nice catch linter!
* add back testrig panic on nil db
* update example configuration to match defaults, slight tweak to cache configuration defaults
* update envparsing test with new defaults
* fetch followingget to use the follow target account
* use accounnt.IsLocal() instead of empty domain check
* use constants for the cache visibility type check
* use bun.In() for notification type restriction in db query
* include replies when fetching PublicTimeline() (to account for single-author threads in Visibility{}.StatusPublicTimelineable())
* use bun query building for nested select statements to ensure working with postgres
* update public timeline future status checks to match visibility filter
* same as previous, for home timeline
* update public timeline tests to dynamically check for appropriate statuses
* migrate accounts to allow unique constraint on public_key
* provide minimal account with publicKey
---------
Signed-off-by: kim <grufwub@gmail.com>
Co-authored-by: tsmethurst <tobi.smethurst@protonmail.com>
											
										 
											2023-03-28 14:03:14 +01:00
										 |  |  | 		// See section 7 here: https://www.sqlite.org/lang_altertable.html | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { | 
					
						
							|  |  |  | 			// Create the new accounts table. | 
					
						
							|  |  |  | 			if _, err := tx. | 
					
						
							|  |  |  | 				NewCreateTable(). | 
					
						
							|  |  |  | 				ModelTableExpr("new_accounts"). | 
					
						
							|  |  |  | 				Model(>smodel.Account{}). | 
					
						
							|  |  |  | 				Exec(ctx); err != nil { | 
					
						
							|  |  |  | 				return err | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 			// If we don't specify columns explicitly, | 
					
						
							|  |  |  | 			// Postgres gives the following error when | 
					
						
							|  |  |  | 			// transferring accounts to new_accounts: | 
					
						
							|  |  |  | 			// | 
					
						
							|  |  |  | 			//	ERROR:  column "fetched_at" is of type timestamp with time zone but expression is of type character varying at character 35 | 
					
						
							|  |  |  | 			//	HINT:  You will need to rewrite or cast the expression. | 
					
						
							|  |  |  | 			// | 
					
						
							|  |  |  | 			// Rather than do funky casting to fix this, | 
					
						
							|  |  |  | 			// it's simpler to just specify all columns. | 
					
						
							|  |  |  | 			columns := []string{ | 
					
						
							|  |  |  | 				"id", | 
					
						
							|  |  |  | 				"created_at", | 
					
						
							|  |  |  | 				"updated_at", | 
					
						
							|  |  |  | 				"fetched_at", | 
					
						
							|  |  |  | 				"username", | 
					
						
							|  |  |  | 				"domain", | 
					
						
							|  |  |  | 				"avatar_media_attachment_id", | 
					
						
							|  |  |  | 				"avatar_remote_url", | 
					
						
							|  |  |  | 				"header_media_attachment_id", | 
					
						
							|  |  |  | 				"header_remote_url", | 
					
						
							|  |  |  | 				"display_name", | 
					
						
							|  |  |  | 				"emojis", | 
					
						
							|  |  |  | 				"fields", | 
					
						
							|  |  |  | 				"note", | 
					
						
							|  |  |  | 				"note_raw", | 
					
						
							|  |  |  | 				"memorial", | 
					
						
							|  |  |  | 				"also_known_as", | 
					
						
							|  |  |  | 				"moved_to_account_id", | 
					
						
							|  |  |  | 				"bot", | 
					
						
							|  |  |  | 				"reason", | 
					
						
							|  |  |  | 				"locked", | 
					
						
							|  |  |  | 				"discoverable", | 
					
						
							|  |  |  | 				"privacy", | 
					
						
							|  |  |  | 				"sensitive", | 
					
						
							|  |  |  | 				"language", | 
					
						
							|  |  |  | 				"status_content_type", | 
					
						
							|  |  |  | 				"custom_css", | 
					
						
							|  |  |  | 				"uri", | 
					
						
							|  |  |  | 				"url", | 
					
						
							|  |  |  | 				"inbox_uri", | 
					
						
							|  |  |  | 				"shared_inbox_uri", | 
					
						
							|  |  |  | 				"outbox_uri", | 
					
						
							|  |  |  | 				"following_uri", | 
					
						
							|  |  |  | 				"followers_uri", | 
					
						
							|  |  |  | 				"featured_collection_uri", | 
					
						
							|  |  |  | 				"actor_type", | 
					
						
							|  |  |  | 				"private_key", | 
					
						
							|  |  |  | 				"public_key", | 
					
						
							|  |  |  | 				"public_key_uri", | 
					
						
							|  |  |  | 				"sensitized_at", | 
					
						
							|  |  |  | 				"silenced_at", | 
					
						
							|  |  |  | 				"suspended_at", | 
					
						
							|  |  |  | 				"hide_collections", | 
					
						
							|  |  |  | 				"suspension_origin", | 
					
						
							|  |  |  | 				"enable_rss", | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 			// Copy all accounts to the new table. | 
					
						
							|  |  |  | 			if _, err := tx. | 
					
						
							|  |  |  | 				NewInsert(). | 
					
						
							|  |  |  | 				Table("new_accounts"). | 
					
						
							|  |  |  | 				Table("accounts"). | 
					
						
							|  |  |  | 				Column(columns...). | 
					
						
							|  |  |  | 				Exec(ctx); err != nil { | 
					
						
							|  |  |  | 				return err | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 			// Drop the old table. | 
					
						
							|  |  |  | 			if _, err := tx. | 
					
						
							|  |  |  | 				NewDropTable(). | 
					
						
							|  |  |  | 				Table("accounts"). | 
					
						
							|  |  |  | 				Exec(ctx); err != nil { | 
					
						
							|  |  |  | 				return err | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 			// Rename new table to old table. | 
					
						
							|  |  |  | 			if _, err := tx. | 
					
						
							|  |  |  | 				ExecContext( | 
					
						
							|  |  |  | 					ctx, | 
					
						
							|  |  |  | 					"ALTER TABLE ? RENAME TO ?", | 
					
						
							|  |  |  | 					bun.Ident("new_accounts"), | 
					
						
							|  |  |  | 					bun.Ident("accounts"), | 
					
						
							|  |  |  | 				); err != nil { | 
					
						
							|  |  |  | 				return err | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 			// Add all account indexes to the new table. | 
					
						
							|  |  |  | 			for index, columns := range map[string][]string{ | 
					
						
							|  |  |  | 				// Standard indices. | 
					
						
							|  |  |  | 				"accounts_id_idx":              {"id"}, | 
					
						
							|  |  |  | 				"accounts_suspended_at_idx":    {"suspended_at"}, | 
					
						
							|  |  |  | 				"accounts_domain_idx":          {"domain"}, | 
					
						
							|  |  |  | 				"accounts_username_domain_idx": {"username", "domain"}, | 
					
						
							|  |  |  | 				// URI indices. | 
					
						
							|  |  |  | 				"accounts_uri_idx":            {"uri"}, | 
					
						
							|  |  |  | 				"accounts_url_idx":            {"url"}, | 
					
						
							|  |  |  | 				"accounts_inbox_uri_idx":      {"inbox_uri"}, | 
					
						
							|  |  |  | 				"accounts_outbox_uri_idx":     {"outbox_uri"}, | 
					
						
							|  |  |  | 				"accounts_followers_uri_idx":  {"followers_uri"}, | 
					
						
							|  |  |  | 				"accounts_following_uri_idx":  {"following_uri"}, | 
					
						
							|  |  |  | 				"accounts_public_key_uri_idx": {"public_key_uri"}, | 
					
						
							|  |  |  | 			} { | 
					
						
							|  |  |  | 				if _, err := tx. | 
					
						
							|  |  |  | 					NewCreateIndex(). | 
					
						
							|  |  |  | 					Table("accounts"). | 
					
						
							|  |  |  | 					Index(index). | 
					
						
							|  |  |  | 					Column(columns...). | 
					
						
							|  |  |  | 					Exec(ctx); err != nil { | 
					
						
							|  |  |  | 					return err | 
					
						
							|  |  |  | 				} | 
					
						
							|  |  |  | 			} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 			return nil | 
					
						
							|  |  |  | 		}) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	down := func(ctx context.Context, db *bun.DB) error { | 
					
						
							|  |  |  | 		return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { | 
					
						
							|  |  |  | 			return nil | 
					
						
							|  |  |  | 		}) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 	if err := Migrations.Register(up, down); err != nil { | 
					
						
							|  |  |  | 		panic(err) | 
					
						
							|  |  |  | 	} | 
					
						
							|  |  |  | } |