From e2d97c1ce7816c1202519baaf2b521339f57212c Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Wed, 8 Sep 2021 12:25:14 +0200 Subject: [PATCH] fix up tests, add docs --- cmd/gotosocial/admincommands.go | 18 ++++- docs/admin/cli.md | 75 +++++++++++++++++++ internal/api/s2s/user/repliesget_test.go | 6 +- .../{export/account.go => trans/export.go} | 2 +- internal/cliactions/admin/trans/import.go | 56 ++++++++++++++ internal/cliactions/server/server.go | 33 +------- internal/db/basic.go | 4 + internal/db/bundb/basic.go | 64 +++++++++------- internal/db/bundb/basic_test.go | 21 +++++- internal/db/bundb/status.go | 8 +- internal/db/bundb/status_test.go | 7 ++ internal/db/bundb/util.go | 63 ++++++++++++++++ internal/db/params.go | 6 +- internal/timeline/get_test.go | 16 ++-- internal/timeline/index_test.go | 4 +- internal/timeline/manager_test.go | 16 ++-- internal/trans/decoders.go | 16 ++-- internal/trans/export.go | 2 +- internal/trans/exporter.go | 2 + internal/trans/exportminimal.go | 10 +++ internal/trans/import.go | 24 +++--- internal/trans/import_test.go | 12 +++ internal/trans/importer.go | 2 + internal/trans/model/account.go | 36 ++++----- internal/trans/model/block.go | 13 ++-- internal/trans/model/domainblock.go | 18 +++-- internal/trans/model/follow.go | 13 ++-- internal/trans/model/followrequest.go | 13 ++-- internal/trans/model/instance.go | 11 +-- internal/trans/model/type.go | 25 ++++--- internal/trans/model/user.go | 11 +-- testrig/db.go | 6 ++ testrig/testmodels.go | 54 ++++++++++++- 33 files changed, 497 insertions(+), 170 deletions(-) rename internal/cliactions/admin/{export/account.go => trans/export.go} (99%) create mode 100644 internal/cliactions/admin/trans/import.go diff --git a/cmd/gotosocial/admincommands.go b/cmd/gotosocial/admincommands.go index 9dc4bc6e8..5d505fe77 100644 --- a/cmd/gotosocial/admincommands.go +++ b/cmd/gotosocial/admincommands.go @@ -20,7 +20,7 @@ package main import ( "github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/account" - "github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/export" + "github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/trans" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/urfave/cli/v2" ) @@ -161,7 +161,21 @@ func adminCommands() []*cli.Command { }, }, Action: func(c *cli.Context) error { - return runAction(c, export.Export) + return runAction(c, trans.Export) + }, + }, + { + Name: "import", + Usage: "import data from a file into the database", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: config.TransPathFlag, + Usage: config.TransPathUsage, + Required: true, + }, + }, + Action: func(c *cli.Context) error { + return runAction(c, trans.Import) }, }, }, diff --git a/docs/admin/cli.md b/docs/admin/cli.md index 8072a5d7a..e15daf084 100644 --- a/docs/admin/cli.md +++ b/docs/admin/cli.md @@ -213,3 +213,78 @@ Example: ```bash gotosocial admin account password --username some_username --pasword some_really_good_password ``` + +### gotosocial admin export + +This command can be used to export data from your GoToSocial instance into a file, for backup/storage. + +The file format will be a series of newline-separated JSON objects. + +`gotosocial admin export --help`: + +```text +NAME: + gotosocial admin export - export data from the database to file at the given path + +USAGE: + gotosocial admin export [command options] [arguments...] + +OPTIONS: + --path value the path of the file to import from/export to + --help, -h show help (default: false) +``` + +Example: + +```bash +gotosocial admin export --path ./example.json +``` + +`example.json`: + +```json +{"type":"account","id":"01F8MH5NBDF2MV7CTC4Q5128HF","createdAt":"2021-08-31T12:00:53.985645Z","username":"1happyturtle","locked":true,"language":"en","uri":"http://localhost:8080/users/1happyturtle","url":"http://localhost:8080/@1happyturtle","inboxURI":"http://localhost:8080/users/1happyturtle/inbox","outboxURI":"http://localhost:8080/users/1happyturtle/outbox","followingUri":"http://localhost:8080/users/1happyturtle/following","followersUri":"http://localhost:8080/users/1happyturtle/followers","featuredCollectionUri":"http://localhost:8080/users/1happyturtle/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjz\nausfsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLz\neUPxdfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFx\njUz9l0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJY\nfKhKn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq\n79WbhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQABAoIBAGF+MxHjD15VV2NY\nKKb1GjMx98i1Xx6TijgoA+zmfha4LGu35e79Lql+0LXFp0zEpa6lAQsMQQhgd0OD\nmKKmSk+pxAvskJ4FxrhIf/yBFA4RMrj5OCaAOocRtdsOJ8n5UtFBrNAF0tzMY9q/\nkgzoq97aVF1mV9iFxaeBx6zT8ozSdqBq1PK/3w1dVg89S5tfKYc7Q0lQ00SfsTnd\niTDClKyqurebo9Pt6M7gXavgg3tvBlmwwr6XHs34Leng3oiN9mW8DVzaBMPzn+rE\nxF2eqs3v9vVpj8es88OwCh5P+ff8vJYvhu7Fcr/bJ8BItBQwfb8QBDATg/MXU2BI\n2ssW6AECgYEA4wmIyYGeu9+hzDa/J3Vh8GnlVNUCohHcChQdOsWsFXUgpVlUIHrX\neKHn42vD4Rzy52/YzJts4NkZTM9sL+kEXIEcpMG/S9xIIud7U0m/hMSAlmnJK/9j\niEXws3o4jo0E77jnRcBdIjpG4K5Eekm0DSR3SFhtZfEdN2DWPvu7K98CgYEA5tER\n/qJwFMc51AobMU87ZjXON7hI2U1WY/pVF62jSl0IcSsnj2riEKWLrs+GRG+HUg+U\naFSqAHcxaVHA0h0AYR8RopAhDdVKh0kvB8biLo+IEzNjPv2vyn0yRN5YSfXdGzyJ\nUjVU6kWdQOwmzy86nHgFaqEx7eofHIaGZzJK/AECgYEAu2VNQHX63TuzQuoVUa5z\nzoq5vhGsALYZF0CO98ndRkDNV22qIL0ESQ/qZS64GYFZhWouWoQXlGfdmCbFN65v\n6SKwz9UT3rvN1vGWO6Ltr9q6AG0EnYpJT1vbV2kUcaU4Y94NFue2d9/+TMnKv91B\n/m8Q/efvNGuWH/WQIaCKV6UCgYBz89WhYMMDfS4M2mLcu5vwddk53qciGxrqMMjs\nkzsz0Va7W12NS7lzeWaZlAE0gf6t98urOdUJVNeKvBoss4sMP0phqxwf0eWV3ur0\ncjIQB+TpGGikLVdRVuGY/UXHKe9AjoHBva8B3aTpB3lbnbNJBXZbIc1uYq3sa5w7\nXWWUAQKBgH3yW73RRpQNcc9hTUssomUsnQQgHxpfWx5tNxqod36Ytd9EKBh3NqUZ\nvPcH6gdh7mcnNaVNTtQOHLHsbPfBK/pqvb3MAsdlokJcQz8MQJ9SGBBPY6PaGw8z\nq/ambaQykER6dwlXTIlU20uXY0bttOL/iYjKmgo3vA66qfzS6nsg\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzLP7oyyR+BU9ejn0CN9K+WpX3L37pxUcCgZAGH5lf3cGPZjzausf\nsFME94OjVyzw3K5M2beDkZ4E+Fak46NLtakLB1yovy9jKtj4Y4txHoMvRJLzeUPx\ndfeXtpx2d3FDj++Uq4DEE0BhbePXhTGJWaNdC9MQmWKghJnCS5mrnFkdpEFxjUz9\nl0UHl2Z4wppxPdpt7FyevcdfKqzGsAA3BxTM0dg47ZJWjtcvfCiSYpAKFNJYfKhK\nn9T3ezZgrLsF+o0IpD23KxWe1X4d5lgJRU9T4FmLmbvyJKUnfgYXbSEvLUcq79Wb\nhgRcWwxWubjmWXgPGwzULVhhpYlwd2Cv3wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/1happyturtle#main-key"} +{"type":"account","id":"01F8MH0BBE4FHXPH513MBVFHB0","createdAt":"2021-09-08T10:00:53.985634Z","username":"weed_lord420","locked":true,"language":"en","uri":"http://localhost:8080/users/weed_lord420","url":"http://localhost:8080/@weed_lord420","inboxURI":"http://localhost:8080/users/weed_lord420/inbox","outboxURI":"http://localhost:8080/users/weed_lord420/outbox","followingUri":"http://localhost:8080/users/weed_lord420/following","followersUri":"http://localhost:8080/users/weed_lord420/followers","featuredCollectionUri":"http://localhost:8080/users/weed_lord420/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0b\nMIyLRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//P\nceYpo5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4\nus6VxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+\nfNyYVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPc\nqwtx0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQABAoIBAEAA4GHNS4k+Ke4j\nx4J0XkUjV5UbuPY0pSpSDjOJHOJmUfLcg85Ds9mYYO6zxwOaqmrC42ieclI5rh84\nTWQUqX9+VAk1J9UKeE4xZ1SSBtnZ3rK9PjrERZ+dmQ0dATaCuEO5Wwgu7Trk++Bg\nIqy8WNGZL94v9tfwALp1jTXW9AvmQoNdCFBP62vcmYW4YLjnggxLCFTA8YKfdePa\nTuxxY6uLkeBbxzWpbRU2+bmlxd5OnCkiRSMHIX+6JdtCu2JdWpUTCnWrFi2n1TZz\nZQx9z5rvowK1O785jGMFum5vBWpjIU8sJcXmPjGMU25zzmrhzfmkJsTXER3CXoUo\nSqSPqgECgYEA78OR7bY5KKQQ7Lyz6dru4Fct5P/OXTQoOg5aS7TKb95LVWj+TANn\n5djwIbLmAUV30z0Id9VgiZOL0Hny8+3VV9eU088Z408pAy5WQrL3dB8tZLUJSq5c\n5k6X15/VjWOOZKppDxShzoV3mcohrnwVwkv4fhPFQQOJJBYz6xurWs0CgYEA3MDE\nsDMd9ahzO0dl62ynojkkA8ZTcn2UdyvLpGj9UxT5j9vWF3CfqitXgcpNiVSIbxqQ\nbo/pBch7c/2Xakv5zkdcrJj5/6gyr+m1/tK2o7+CjDaSE4SYwufXx+qkl03Zpyzt\nKdOi7Hz/b2tdjump7ECEDE45mG2ea8oSnPgXl0cCgYBkGGFzu/9g2B24t47ksmHH\nhp3CXIjqoDurARLxSCi7SzJoFc0ULtfRPSAC8YzUOwwrQ++lF4+V3+MexcqHy2Kl\nqXqYcn18SC/3BAE/Fzf3Yoyw3mNiqihefbEmc7PTsxxfKkVx5ksmzNGBgsFM9sCe\nvNigyeAvpCo8xogmPwbqgQKBgE34mIBTzcUzFmBdu5YH7r3RyPK8XkUWLhZZlbgg\njTmHMw6o61mkIgENBf+F4RUckoQLsfAbTIcKZPB3JcAZzcYaVpVwAv1V/3E671lu\nO6xivE2iCL50GzDcis7GBhSbHsF5kNsxMV6uV9qW5ZjQ13/m2b0u9BDuxwHzgdeH\nmW2JAoGAIUOYniuEwdygxWVnYatpr3NPjT3BOKoV5i9zkeJRu1hFpwQM6vQ4Ds5p\nGC5vbMKAv9Cwuw62e2HvqTun3+U2Y5Uived3XCpgM/50BFrFHCfuqXEnu1bEzk5z\n9mIhp8uXPxzC5N7tRQfb3/eU1IUcb6T6ksbr2P81z0j03J55erg=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAzsCcTHzwIgpWKVvut0Q/t1bFwnbj9hO6Ic6k0KXCXbf6qi0bMIyL\nRZr8DS61mD+SPSO2QKEL647xxyW2D8YGtwN6Cc6MpWETsWJkNtS8t7tDL//PceYp\no5LiqKgn0TXj0Pq8Lvb7rqpH8QJ2EVm14SK+elhKZW/Bi5ZOEwfL8pw6EHI4us6V\nxCNQ099dksu++kbdD7zxqEKnk/4zOttYt0whlVrxzkibTjlKdlSlTYpIstU+fNyY\nVE0xWvrn+yF7jVlEwZYOFGfZbpELadrdOr2k1hvAk7upkrpKmLqYfwqD/xPcqwtx\n0iS6AEnmkSiTcAvju5vLkoLFRU7Of4AZ2wIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/weed_lord420#main-key"} +{"type":"account","id":"01F8MH17FWEB39HZJ76B6VXSKF","createdAt":"2021-09-05T10:00:53.985641Z","username":"admin","locked":true,"language":"en","uri":"http://localhost:8080/users/admin","url":"http://localhost:8080/@admin","inboxURI":"http://localhost:8080/users/admin/inbox","outboxURI":"http://localhost:8080/users/admin/outbox","followingUri":"http://localhost:8080/users/admin/following","followersUri":"http://localhost:8080/users/admin/followers","featuredCollectionUri":"http://localhost:8080/users/admin/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVq\nhujDhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLR\nBI97qD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wg\nfvtEjEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G\n8kQJDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/Bk\nRhhGp2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQABAoIBAGK0aIADOU4ffJDe\n7sveiih5Fc1PATwx/QIR2QkWM1SREdx6LYclcX44V8xDanAbE44p1SkHY/CsEtYy\nXnyoXnn2FwFDQrdveY7+I6PApOPLAcKWkyLltC+hbVdj92/6YGNrm7EA/a77wruH\nmwjiivLnTG2CLecNiXSl33DA9YU4Yz+2Tza3IpTdjt8c/dz/BKKaxaWV+i9ew5VR\nioo5v51B+J8PrneCM/p8LGiLV148Njr0JqV6eFy1JuzItYMYdc3Fp+YnMzsuMZEA\n1akMcoln/ucVJyOFnCn6jx47nIoPZLl1KxX3aRDRfvrejm6W4yAkkTmR5voSRqax\njPL3rI0CgYEA9Acu4TO8xJ3uGaUad0N9JTYQVSmtAaE/g+df9LGMSzoj8X95S4xE\nQsGPqNGDm2VWADJjK4P05twZ+LfsfSKQ86wbp4/gbgnXpqB1P5Lty/B7KxiTnNwt\nwb1WGWTCukxfUSL3PRyf8uylkrg72RxKiBx4zKO3WVSLWOZWrFtn0qMCgYEA0H2p\nJs9Nv20ADOOX5tQ7+ruS6/B/Fhyj5fhflSYCAtOW7aME7+zQKJyqSQZ4b2Aub3Tp\nGIaUbRIGzjHyuTultFFWvjU3H5aI/0g1G9WKaBhNkyTIYVmMKtYyhXNvouWing8x\noraWx8TTBP8Cdnnk+QgdR2fpug8cghKupp5wvO8CgYA1JFtRL7MsHjh73TimQExA\njkWARlMmx7bNQtXis8eZmk+5h8kiaqly4DQoz3eZn7fa0x5Fm7b5j3UYdPVLSvvG\nFPTwyKRXUk1kPA1MivK+NuCbwf5jao+MYW8emJLPf1JCmRq+dD1g6aglC3n9Dewt\nOAYWipCjI4Y1FfRKFJ3HgQKBgEAb47+DTyzln3ZXJYZdDHR06SCTuwBZnixAy2NZ\nZJTp6yb3UbVU5E0Yn2QFEVNuB9lN4b8g4tMHEACnazN6G+HugPXL9z9HUqjs0yfT\n6dNIZdIxJUyJ9IfXhYFzlYhJhE+F7IVUD9kttJV8tI0pvja1QAuM8Fm9+84jYIDr\nh08RAoGAMYbjKHbtejcHBwt1kIcSss0cDmlZbBleJo8tdmdg4ndf5GE9N4/EL7tq\nm2zYSfr7OVdnOwRhoO+xF/6d1L7+TR1wz+k2fuMsI71aM5Ocp1nYTutjIkBTcldZ\nZzvjOgZWng5icuRLQQiDSKG5uqazqL/xGXkijb4kp4WW6myWY3c=\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxr2e1pqfLwwUCwHUdx56Mxnq5Kzc2EBwqN6jIPjiqVaG5eVqhujD\nhdqwMq0hnpBSPzLnvjiOtEh7Bwhx0MjuC/GRPTM9oNWPYD4PcjX5ofrubyLRBI97\nqD0SbyzUWzeyBi6R5tpW8LK1MJXNbnYlz5WouEiC4mY77ulri0EN2hCq80wgfvtE\njEvELcKBqIytKH3rutIzfAyqXD7LSQ8UDoNh9GHyIfq8Zj32gWVk2MiPI3+G8kQJ\nDmD8CKEasnrGVdSJBQUg3xDAtOibPXLP+07AIsKYMon35hVNvQNQPS7ru/BkRhhG\np2R44zqj6L9mxYbSrhFAaKDedu8oVe1aLQIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/admin#main-key"} +{"type":"account","id":"01F8MH1H7YV1Z7D2C8K2730QBF","createdAt":"2021-09-06T10:00:53.985643Z","username":"the_mighty_zork","locked":true,"language":"en","uri":"http://localhost:8080/users/the_mighty_zork","url":"http://localhost:8080/@the_mighty_zork","inboxURI":"http://localhost:8080/users/the_mighty_zork/inbox","outboxURI":"http://localhost:8080/users/the_mighty_zork/outbox","followingUri":"http://localhost:8080/users/the_mighty_zork/following","followersUri":"http://localhost:8080/users/the_mighty_zork/followers","featuredCollectionUri":"http://localhost:8080/users/the_mighty_zork/collections/featured","actorType":"Person","privateKey":"-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss\n5mEA/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvC\nC9zt/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZ\nFHptEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1\ntMhsUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlq\nefr58l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQABAoIBAFa+UypbFG1cW2Tr\nNBxPm7ngOEtXl8MicV4dIVKh0TwOo13ZxtNFBbOj7jALmPn/9HrtmbkABPQHDL1U\n/nt9aNSAeTjpwH3RaD5vFX3n0g8n2zJBOZLxxzAjNi4RBLYj5uP1AiKkdvRlsJza\nuSFDkty2zMBqN9mLPHE+RePj5Qa6tjYfIQqQzu/+YnYMlXHoC2yHNKsvz6S5FhVj\nv5zATv2JlJQH3RSmhuPOah73iQnKCLzYYEAHleawKrCg/rZ3ht37Guvabeq7MqQN\nvi9pJdAA+RMxPsboHajskePjOTYJgKQSxEAMRTMfBR40aZxklxQL0EoBd1Y3CHXh\nfMg0xWECgYEA0ORrpJ1A2WNQwKcDDeBBsaJqWF4EraoFzYrugKZrAYEeVyuGD0zq\nARUaWkZTZ1f6wQ10i1WxAuKlBEds7QsLdZzLsA4um4JlBroCZiYfPnmTtb8op1LY\nFqeYTByvAmnfWWTuOI67GX9ruLg8tEGuz38kuQVSxYs51its3tScNPUCgYEAyRst\nwRbqpOqnwoRoS6pxv0Vpc3nUcfaVYwsg/qobJkiwAdlUYeE7alvEY926VW4cvU/X\nhy3L1punAqnyLI7uuqCefXEbNxO0Cebyy4Kv2Ye1uzl0OHsJczSNdfpNqfAIKwtN\nHLCYDGCsluQhz+I/5Pd0dT+JDPPW9hKS2HG7o+kCgYBqugn1VRLo/sEnbS02TbnC\n1ESZWY/yWsgUOEObH2vUnO+vgeFAt/9nBi0sqnm6d0z6jbFZ7zI9UycUhJm2ksoM\nEUxQay6M7ZZIVYkcP6X++YbqePyAYOdey8oYOR+BkC45MkQ0SVh2so+LFTaOsnBq\nO3+7uGiN3ZBzSESbpO0acQKBgQCONrsXZeZO82XpB4tdns3LbgGRWKEkajTgEnml\nvZNvck2NMSwb/5PttbFe0ei4CyMluPV4MamJPQ9Qse+BFR67OWR63uZY/4T8z6X4\nxpUmZnLcUFfgrRlUr+AtgvEy8HxGPDquxC7x6deC6RcEFEIM3/UqCOEZGMJ1x1Ky\n31LLKQKBgGCKwVgQ8+4JyHZFkie3YdHhxJDokgY+Opb0HNnoBY/lZ54UMCCJQPS2\n0XPSu651j/3adr3RQneU04gF6U2/D5JzFEV0kUsqZ4Zy2EEU0LU4ibus0gyomSpK\niWhU4QrC/M4ELxYZinlNu3ThPWNQ/PMNteVWfdgOcV7uUWl0ViFp\n-----END RSA PRIVATE KEY-----\n","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEApBmF8U+or+E0mgUMH3LE4uRIWzeV9rhYnvSMm9OpOsxwJiss5mEA\n/NtPHvQlq2UwrqXX89Wvu94K9EzZ4VyWYQGdxaiPpt17vRqUfsHUnXkY0pvCC9zt\n/aNlJtdt2xm+7PTC0YQd4+E1FX3aaoUPJL8MXzNlpJzaUtuwLZe1iBmFfatZFHpt\nEgc4nlf6TNLTzj3Yw1/7zIGVS8Vi7VquHc0Xo8dRiL2RxCGzLWnwL6GlrxY1tMhs\nUg467XeoiwegFCpcIhAhPFREKoTnCEksL/N0rpXl7m6CAy5uqBGs5mMXnXlqefr5\n8l0j2dU6zc60LCHH9TJC+roXsKJhy9sx/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://localhost:8080/users/the_mighty_zork#main-key"} +{"type":"block","id":"01FEXXET6XXMF7G2V3ASZP3YQW","createdAt":"2021-09-08T09:00:53.965362Z","uri":"http://localhost:8080/users/1happyturtle/blocks/01FEXXET6XXMF7G2V3ASZP3YQW","accountId":"01F8MH5NBDF2MV7CTC4Q5128HF","targetAccountId":"01F8MH5ZK5VRH73AKHQM6Y9VNX"} +{"type":"account","id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","createdAt":"2021-08-31T12:00:53.985646Z","username":"foss_satan","domain":"fossbros-anonymous.io","locked":true,"language":"en","uri":"http://fossbros-anonymous.io/users/foss_satan","url":"http://fossbros-anonymous.io/@foss_satan","inboxURI":"http://fossbros-anonymous.io/users/foss_satan/inbox","outboxURI":"http://fossbros-anonymous.io/users/foss_satan/outbox","followingUri":"http://fossbros-anonymous.io/users/foss_satan/following","followersUri":"http://fossbros-anonymous.io/users/foss_satan/followers","featuredCollectionUri":"http://fossbros-anonymous.io/users/foss_satan/collections/featured","actorType":"Person","publicKey":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEA2OyVgkaIL9VohXKYTh319j4OouHRX/8QC7piXj71k7q5RDzEyvis\nVZBc5/C1/crCpxt895i0Ai2CiXQx+dISV7s/JBhAGl8s7TQ8jLlMuptrI0+sdkBC\nlu8pU0qQmoeXVnlquOzNmqGufUxIDtLXLZDN17qf/7vWA23q4d0tG5KQhGGGKiVM\n61Ufvr9MmgPBSpyUvYMAulFlz1264L49aGWeVgOz3qUQzqtxjrP0kaIbeyt56miP\nKr5AqkRgSsXci+FAo6suxR5gzo9NgleNkbZWF9MQyKlawukPwZUDSh396vtNQMee\n/4mto7mAXw8iio0IacrYO3F7iyewXnmI/QIDAQAB\n-----END RSA PUBLIC KEY-----\n","publicKeyUri":"http://fossbros-anonymous.io/users/foss_satan/main-key"} +{"type":"follow","id":"01F8PYDCE8XE23GRE5DPZJDZDP","createdAt":"2021-09-08T09:00:54.749465Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PYDCE8XE23GRE5DPZJDZDP","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH5NBDF2MV7CTC4Q5128HF"} +{"type":"follow","id":"01F8PY8RHWRQZV038T4E8T9YK8","createdAt":"2021-09-06T12:00:54.749459Z","uri":"http://localhost:8080/users/the_mighty_zork/follow/01F8PY8RHWRQZV038T4E8T9YK8","accountId":"01F8MH1H7YV1Z7D2C8K2730QBF","targetAccountId":"01F8MH17FWEB39HZJ76B6VXSKF"} +{"type":"domainBlock","id":"01FF22EQM7X8E3RX1XGPN7S87D","createdAt":"2021-09-08T10:00:53.968971Z","domain":"replyguys.com","createdByAccountID":"01F8MH17FWEB39HZJ76B6VXSKF","privateComment":"i blocked this domain because they keep replying with pushy + unwarranted linux advice","publicComment":"reply-guying to tech posts","obfuscate":false} +{"type":"user","id":"01F8MGYG9E893WRHW0TAEXR8GJ","createdAt":"2021-09-08T10:00:53.97247Z","accountID":"01F8MH0BBE4FHXPH513MBVFHB0","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","locale":"en","lastEmailedAt":"0001-01-01T00:00:00Z","confirmationToken":"a5a280bd-34be-44a3-8330-a57eaf61b8dd","confirmationTokenSentAt":"2021-09-08T10:00:53.972472Z","unconfirmedEmail":"weed_lord420@example.org","moderator":false,"admin":false,"disabled":false,"approved":false} +{"type":"user","id":"01F8MGWYWKVKS3VS8DV1AMYPGE","createdAt":"2021-09-05T10:00:53.972475Z","email":"admin@example.org","accountID":"01F8MH17FWEB39HZJ76B6VXSKF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:50:53.972477Z","lastSignInAt":"2021-09-08T08:00:53.972477Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:30:53.972478Z","confirmedAt":"2021-09-05T10:00:53.972478Z","moderator":true,"admin":true,"disabled":false,"approved":true} +{"type":"user","id":"01F8MGVGPHQ2D3P3X0454H54Z5","createdAt":"2021-09-06T22:00:53.97248Z","email":"zork@example.org","accountID":"01F8MH1H7YV1Z7D2C8K2730QBF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972481Z","lastSignInAt":"2021-09-08T08:00:53.972481Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972482Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972483Z","confirmedAt":"2021-09-07T00:00:53.972482Z","moderator":false,"admin":false,"disabled":false,"approved":true} +{"type":"user","id":"01F8MH1VYJAE00TVVGMM5JNJ8X","createdAt":"2021-09-06T22:00:53.972485Z","email":"tortle.dude@example.org","accountID":"01F8MH5NBDF2MV7CTC4Q5128HF","encryptedPassword":"$2y$10$ggWz5QWwnx6kzb9g0tnIJurFtE0dhr5Zfeaqs9iFuUIXzafQlJVZS","currentSignInAt":"2021-09-08T09:30:53.972485Z","lastSignInAt":"2021-09-08T08:00:53.972486Z","chosenLanguages":["en"],"locale":"en","lastEmailedAt":"2021-09-08T09:05:53.972487Z","confirmationTokenSentAt":"2021-09-06T22:00:53.972487Z","confirmedAt":"2021-09-07T00:00:53.972487Z","moderator":false,"admin":false,"disabled":false,"approved":true} +{"type":"instance","id":"01BZDDRPAB8J645ABY31HHF68Y","createdAt":"2021-09-08T10:00:54.763912Z","domain":"localhost:8080","title":"localhost:8080","uri":"http://localhost:8080","reputation":0} +``` + +### gotosocial admin import + +This command can be used to import data from a file into your GoToSocial database. + +If GoToSocial tables don't yet exist in the database, they will be created. + +If any conflicts occur while importing (an already exists while attempting to import a specific account, for example), then the process will be aborted. + +The file format should be a series of newline-separated JSON objects (see above). + +`gotosocial admin import --help`: + +```text +NAME: + gotosocial admin import - import data from a file into the database + +USAGE: + gotosocial admin import [command options] [arguments...] + +OPTIONS: + --path value the path of the file to import from/export to + --help, -h show help (default: false) +``` + +Example: + +```bash +gotosocial admin import --path ./example.json +``` diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go index 75edbc882..a785b2cff 100644 --- a/internal/api/s2s/user/repliesget_test.go +++ b/internal/api/s2s/user/repliesget_test.go @@ -157,7 +157,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() { b, err := ioutil.ReadAll(result.Body) assert.NoError(suite.T(), err) - assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) // should be a Collection m := make(map[string]interface{}) @@ -188,7 +188,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { // setup request recorder := httptest.NewRecorder() ctx, _ := gin.CreateTestContext(recorder) - ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting + ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0", nil) // the endpoint we're hitting ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader) ctx.Request.Header.Set("Date", signedRequest.DateHeader) @@ -220,7 +220,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { assert.NoError(suite.T(), err) fmt.Println(string(b)) - assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) + assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b)) // should be a Collection m := make(map[string]interface{}) diff --git a/internal/cliactions/admin/export/account.go b/internal/cliactions/admin/trans/export.go similarity index 99% rename from internal/cliactions/admin/export/account.go rename to internal/cliactions/admin/trans/export.go index f680b1c0a..3d9607ea6 100644 --- a/internal/cliactions/admin/export/account.go +++ b/internal/cliactions/admin/trans/export.go @@ -16,7 +16,7 @@ along with this program. If not, see . */ -package export +package trans import ( "context" diff --git a/internal/cliactions/admin/trans/import.go b/internal/cliactions/admin/trans/import.go new file mode 100644 index 000000000..7b137eccc --- /dev/null +++ b/internal/cliactions/admin/trans/import.go @@ -0,0 +1,56 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + "errors" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/cliactions" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db/bundb" + "github.com/superseriousbusiness/gotosocial/internal/trans" +) + +// Import imports info from a file into the database +var Import cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { + dbConn, err := bundb.NewBunDBService(ctx, c, log) + if err != nil { + return fmt.Errorf("error creating dbservice: %s", err) + } + + importer := trans.NewImporter(dbConn, log) + + path, ok := c.ExportCLIFlags[config.TransPathFlag] + if !ok { + return errors.New("no path set") + } + + if err := dbConn.CreateAllTables(ctx); err != nil { + return err + } + + if err := importer.Import(ctx, path); err != nil { + return err + } + + return dbConn.Stop(ctx) +} diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go index 0769ade82..3ef714fb0 100644 --- a/internal/cliactions/server/server.go +++ b/internal/cliactions/server/server.go @@ -39,7 +39,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gotosocial" - "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/oidc" @@ -51,32 +50,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/web" ) -var models []interface{} = []interface{}{ - >smodel.Account{}, - >smodel.Application{}, - >smodel.Block{}, - >smodel.DomainBlock{}, - >smodel.EmailDomainBlock{}, - >smodel.Follow{}, - >smodel.FollowRequest{}, - >smodel.MediaAttachment{}, - >smodel.Mention{}, - >smodel.Status{}, - >smodel.StatusToEmoji{}, - >smodel.StatusToTag{}, - >smodel.StatusFave{}, - >smodel.StatusBookmark{}, - >smodel.StatusMute{}, - >smodel.Tag{}, - >smodel.User{}, - >smodel.Emoji{}, - >smodel.Instance{}, - >smodel.Notification{}, - >smodel.RouterSession{}, - >smodel.Token{}, - >smodel.Client{}, -} - // Start creates and starts a gotosocial server var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { dbService, err := bundb.NewBunDBService(ctx, c, log) @@ -84,10 +57,8 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log return fmt.Errorf("error creating dbservice: %s", err) } - for _, m := range models { - if err := dbService.CreateTable(ctx, m); err != nil { - return fmt.Errorf("table creation error: %s", err) - } + if err := dbService.CreateAllTables(ctx); err != nil { + return fmt.Errorf("error creating database tables: %s", err) } if err := dbService.CreateInstanceAccount(ctx); err != nil { diff --git a/internal/db/basic.go b/internal/db/basic.go index cf65ddc09..2a1141c8d 100644 --- a/internal/db/basic.go +++ b/internal/db/basic.go @@ -26,6 +26,10 @@ type Basic interface { // For implementations that don't use tables, this can just return nil. CreateTable(ctx context.Context, i interface{}) Error + // CreateAllTables creates *all* tables necessary for the running of GoToSocial. + // Because it uses the 'if not exists' parameter it is safe to run against a GtS that's already been initialized. + CreateAllTables(ctx context.Context) Error + // DropTable drops the table for the given interface. // For implementations that don't use tables, this can just return nil. DropTable(ctx context.Context, i interface{}) Error diff --git a/internal/db/bundb/basic.go b/internal/db/bundb/basic.go index a3a8d0ae9..d4de5bb0b 100644 --- a/internal/db/bundb/basic.go +++ b/internal/db/bundb/basic.go @@ -24,6 +24,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/uptrace/bun" ) @@ -53,17 +54,8 @@ func (b *basicDB) GetWhere(ctx context.Context, where []db.Where, i interface{}) } q := b.conn.NewSelect().Model(i) - for _, w := range where { - if w.Value == nil { - q = q.Where("? IS NULL", bun.Ident(w.Key)) - } else { - if w.CaseInsensitive { - q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value) - } else { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } - } - } + + selectWhere(q, where) err := q.Scan(ctx) return b.conn.ProcessError(err) @@ -97,9 +89,7 @@ func (b *basicDB) DeleteWhere(ctx context.Context, where []db.Where, i interface NewDelete(). Model(i) - for _, w := range where { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } + deleteWhere(q, where) _, err := q.Exec(ctx) return b.conn.ProcessError(err) @@ -128,17 +118,7 @@ func (b *basicDB) UpdateOneByID(ctx context.Context, id string, key string, valu func (b *basicDB) UpdateWhere(ctx context.Context, where []db.Where, key string, value interface{}, i interface{}) db.Error { q := b.conn.NewUpdate().Model(i) - for _, w := range where { - if w.Value == nil { - q = q.Where("? IS NULL", bun.Ident(w.Key)) - } else { - if w.CaseInsensitive { - q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value) - } else { - q = q.Where("? = ?", bun.Safe(w.Key), w.Value) - } - } - } + updateWhere(q, where) q = q.Set("? = ?", bun.Safe(key), value) @@ -151,6 +131,40 @@ func (b *basicDB) CreateTable(ctx context.Context, i interface{}) db.Error { return err } +func (b *basicDB) CreateAllTables(ctx context.Context) db.Error { + models := []interface{}{ + >smodel.Account{}, + >smodel.Application{}, + >smodel.Block{}, + >smodel.DomainBlock{}, + >smodel.EmailDomainBlock{}, + >smodel.Follow{}, + >smodel.FollowRequest{}, + >smodel.MediaAttachment{}, + >smodel.Mention{}, + >smodel.Status{}, + >smodel.StatusToEmoji{}, + >smodel.StatusToTag{}, + >smodel.StatusFave{}, + >smodel.StatusBookmark{}, + >smodel.StatusMute{}, + >smodel.Tag{}, + >smodel.User{}, + >smodel.Emoji{}, + >smodel.Instance{}, + >smodel.Notification{}, + >smodel.RouterSession{}, + >smodel.Token{}, + >smodel.Client{}, + } + for _, i := range models { + if err := b.CreateTable(ctx, i); err != nil { + return err + } + } + return nil +} + func (b *basicDB) DropTable(ctx context.Context, i interface{}) db.Error { _, err := b.conn.NewDropTable().Model(i).IfExists().Exec(ctx) return b.conn.ProcessError(err) diff --git a/internal/db/bundb/basic_test.go b/internal/db/bundb/basic_test.go index d8067fb9d..e5f7e159a 100644 --- a/internal/db/bundb/basic_test.go +++ b/internal/db/bundb/basic_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -42,7 +43,25 @@ func (suite *BasicTestSuite) TestGetAllStatuses() { s := []*gtsmodel.Status{} err := suite.db.GetAll(context.Background(), &s) suite.NoError(err) - suite.Len(s, 12) + suite.Len(s, 13) +} + +func (suite *BasicTestSuite) TestGetAllNotNull() { + where := []db.Where{{ + Key: "domain", + Value: nil, + Not: true, + }} + + a := []*gtsmodel.Account{} + + err := suite.db.GetWhere(context.Background(), where, &a) + suite.NoError(err) + suite.NotEmpty(a) + + for _, acct := range a { + suite.NotEmpty(acct.Domain) + } } func TestBasicTestSuite(t *testing.T) { diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go index 9464cfadf..2c26a7df9 100644 --- a/internal/db/bundb/status.go +++ b/internal/db/bundb/status.go @@ -240,11 +240,11 @@ func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status, } } - // only do one loop if we only want direct children - if onlyDirect { - return + // if we're not only looking for direct children of status, then do the same children-finding + // operation for the found child status too. + if !onlyDirect { + s.statusChildren(ctx, child, foundStatuses, false, minID) } - s.statusChildren(ctx, child, foundStatuses, false, minID) } } diff --git a/internal/db/bundb/status_test.go b/internal/db/bundb/status_test.go index 4b4a5aca4..ff86390fe 100644 --- a/internal/db/bundb/status_test.go +++ b/internal/db/bundb/status_test.go @@ -104,6 +104,13 @@ func (suite *StatusTestSuite) TestGetStatusTwice() { suite.Less(duration2, duration1) } +func (suite *StatusTestSuite) TestGetStatusChildren() { + targetStatus := suite.testStatuses["local_account_1_status_1"] + children, err := suite.db.GetStatusChildren(context.Background(), targetStatus, true, "") + suite.NoError(err) + suite.Len(children, 2) +} + func TestStatusTestSuite(t *testing.T) { suite.Run(t, new(StatusTestSuite)) } diff --git a/internal/db/bundb/util.go b/internal/db/bundb/util.go index 9e1afb87e..459f65d8c 100644 --- a/internal/db/bundb/util.go +++ b/internal/db/bundb/util.go @@ -19,6 +19,7 @@ package bundb import ( + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/uptrace/bun" ) @@ -35,3 +36,65 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery { WhereOr("? = ''", bun.Ident(column)) } } + +// updateWhere parses []db.Where and adds it to the given update query. +func updateWhere(q *bun.UpdateQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// selectWhere parses []db.Where and adds it to the given select query. +func selectWhere(q *bun.SelectQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// deleteWhere parses []db.Where and adds it to the given where query. +func deleteWhere(q *bun.DeleteQuery, where []db.Where) { + for _, w := range where { + query, args := parseWhere(w) + q = q.Where(query, args...) + } +} + +// parseWhere looks through the options on a single db.Where entry, and +// returns the appropriate query string and arguments. +func parseWhere(w db.Where) (query string, args []interface{}) { + if w.Not { + if w.Value == nil { + query = "? IS NOT NULL" + args = []interface{}{bun.Ident(w.Key)} + return + } + + if w.CaseInsensitive { + query = "LOWER(?) != LOWER(?)" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + query = "? != ?" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + if w.Value == nil { + query = "? IS NULL" + args = []interface{}{bun.Ident(w.Key)} + return + } + + if w.CaseInsensitive { + query = "LOWER(?) = LOWER(?)" + args = []interface{}{bun.Safe(w.Key), w.Value} + return + } + + query = "? = ?" + args = []interface{}{bun.Safe(w.Key), w.Value} + return +} diff --git a/internal/db/params.go b/internal/db/params.go index f0c384435..dbbf734a1 100644 --- a/internal/db/params.go +++ b/internal/db/params.go @@ -22,9 +22,13 @@ package db type Where struct { // The table to search on. Key string - // The value that must be set. + // The value to match. Value interface{} // Whether the value (if a string) should be case sensitive or not. // Defaults to false. CaseInsensitive bool + // If set, reverse the where. + // `WHERE k = v` becomes `WHERE k != v`. + // `WHERE k IS NULL` becomes `WHERE k IS NOT NULL` + Not bool } diff --git a/internal/timeline/get_test.go b/internal/timeline/get_test.go index 96c333c5f..6c4a58c76 100644 --- a/internal/timeline/get_test.go +++ b/internal/timeline/get_test.go @@ -73,8 +73,8 @@ func (suite *GetTestSuite) TestGetDefault() { suite.FailNow(err.Error()) } - // we only have 12 statuses in the test suite - suite.Len(statuses, 12) + // we only have 13 statuses in the test suite + suite.Len(statuses, 13) // statuses should be sorted highest to lowest ID var highest string @@ -166,8 +166,8 @@ func (suite *GetTestSuite) TestGetMinID() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a min ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a min ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string @@ -188,8 +188,8 @@ func (suite *GetTestSuite) TestGetSinceID() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string @@ -210,8 +210,8 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() { suite.FailNow(err.Error()) } - // we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries - suite.Len(statuses, 5) + // we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries + suite.Len(statuses, 6) // statuses should be sorted highest to lowest ID var highest string diff --git a/internal/timeline/index_test.go b/internal/timeline/index_test.go index 25565a1de..2a6429b3e 100644 --- a/internal/timeline/index_test.go +++ b/internal/timeline/index_test.go @@ -66,7 +66,7 @@ func (suite *IndexTestSuite) TestIndexBeforeLowID() { // the oldest indexed post should be the lowest one we have in our testrig postID, err := suite.timeline.OldestIndexedPostID(context.Background()) suite.NoError(err) - suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID) + suite.Equal("01F8MHAMCHF6Y650WCRSCP4WMY", postID) indexLength := suite.timeline.PostIndexLength(context.Background()) suite.Equal(10, indexLength) @@ -95,7 +95,7 @@ func (suite *IndexTestSuite) TestIndexBehindHighID() { // the newest indexed post should be the highest one we have in our testrig postID, err := suite.timeline.NewestIndexedPostID(context.Background()) suite.NoError(err) - suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID) + suite.Equal("01FF25D5Q0DH7CHD57CTRS6WK0", postID) // indexLength should be 10 because that's all this user has hometimelineable indexLength := suite.timeline.PostIndexLength(context.Background()) diff --git a/internal/timeline/manager_test.go b/internal/timeline/manager_test.go index ea4dc4c12..a67a8ae5a 100644 --- a/internal/timeline/manager_test.go +++ b/internal/timeline/manager_test.go @@ -67,9 +67,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { err = suite.manager.PrepareXFromTop(context.Background(), testAccount.ID, 20) suite.NoError(err) - // local_account_1 can see 12 statuses out of the testrig statuses in its home timeline + // local_account_1 can see 13 statuses out of the testrig statuses in its home timeline indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(12, indexedLen) + suite.Equal(13, indexedLen) // oldest should now be set oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -79,7 +79,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // get hometimeline statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false) suite.NoError(err) - suite.Len(statuses, 12) + suite.Len(statuses, 13) // now wipe the last status from all timelines, as though it had been deleted by the owner err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R") @@ -87,7 +87,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(11, indexedLen) + suite.Equal(12, indexedLen) // oldest should now be different oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -101,7 +101,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(10, indexedLen) + suite.Equal(11, indexedLen) // oldest should now be different oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID) @@ -112,9 +112,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID) suite.NoError(err) - // timeline should be empty now + // timeline should be shorter indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(5, indexedLen) + suite.Equal(6, indexedLen) // ingest 1 into the timeline status1 := suite.testStatuses["admin_account_status_1"] @@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() { // timeline should be longer now indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID) - suite.Equal(7, indexedLen) + suite.Equal(8, indexedLen) // try to ingest status 2 again ingested, err = suite.manager.IngestAndPrepare(context.Background(), status2, testAccount.ID) diff --git a/internal/trans/decoders.go b/internal/trans/decoders.go index 6e73881d6..b4f146023 100644 --- a/internal/trans/decoders.go +++ b/internal/trans/decoders.go @@ -37,7 +37,7 @@ func newDecoder(target interface{}) (*mapstructure.Decoder, error) { return mapstructure.NewDecoder(decoderConfig) } -func (i *importer) accountDecode(e transmodel.TransEntry) (*transmodel.Account, error) { +func (i *importer) accountDecode(e transmodel.Entry) (*transmodel.Account, error) { a := &transmodel.Account{} if err := i.simpleDecode(e, a); err != nil { return nil, err @@ -70,7 +70,7 @@ func (i *importer) accountDecode(e transmodel.TransEntry) (*transmodel.Account, return a, nil } -func (i *importer) blockDecode(e transmodel.TransEntry) (*transmodel.Block, error) { +func (i *importer) blockDecode(e transmodel.Entry) (*transmodel.Block, error) { b := &transmodel.Block{} if err := i.simpleDecode(e, b); err != nil { return nil, err @@ -79,7 +79,7 @@ func (i *importer) blockDecode(e transmodel.TransEntry) (*transmodel.Block, erro return b, nil } -func (i *importer) domainBlockDecode(e transmodel.TransEntry) (*transmodel.DomainBlock, error) { +func (i *importer) domainBlockDecode(e transmodel.Entry) (*transmodel.DomainBlock, error) { b := &transmodel.DomainBlock{} if err := i.simpleDecode(e, b); err != nil { return nil, err @@ -88,7 +88,7 @@ func (i *importer) domainBlockDecode(e transmodel.TransEntry) (*transmodel.Domai return b, nil } -func (i *importer) followDecode(e transmodel.TransEntry) (*transmodel.Follow, error) { +func (i *importer) followDecode(e transmodel.Entry) (*transmodel.Follow, error) { f := &transmodel.Follow{} if err := i.simpleDecode(e, f); err != nil { return nil, err @@ -97,7 +97,7 @@ func (i *importer) followDecode(e transmodel.TransEntry) (*transmodel.Follow, er return f, nil } -func (i *importer) followRequestDecode(e transmodel.TransEntry) (*transmodel.FollowRequest, error) { +func (i *importer) followRequestDecode(e transmodel.Entry) (*transmodel.FollowRequest, error) { f := &transmodel.FollowRequest{} if err := i.simpleDecode(e, f); err != nil { return nil, err @@ -106,7 +106,7 @@ func (i *importer) followRequestDecode(e transmodel.TransEntry) (*transmodel.Fol return f, nil } -func (i *importer) instanceDecode(e transmodel.TransEntry) (*transmodel.Instance, error) { +func (i *importer) instanceDecode(e transmodel.Entry) (*transmodel.Instance, error) { inst := &transmodel.Instance{} if err := i.simpleDecode(e, inst); err != nil { return nil, err @@ -115,7 +115,7 @@ func (i *importer) instanceDecode(e transmodel.TransEntry) (*transmodel.Instance return inst, nil } -func (i *importer) userDecode(e transmodel.TransEntry) (*transmodel.User, error) { +func (i *importer) userDecode(e transmodel.Entry) (*transmodel.User, error) { u := &transmodel.User{} if err := i.simpleDecode(e, u); err != nil { return nil, err @@ -124,7 +124,7 @@ func (i *importer) userDecode(e transmodel.TransEntry) (*transmodel.User, error) return u, nil } -func (i *importer) simpleDecode(entry transmodel.TransEntry, target interface{}) error { +func (i *importer) simpleDecode(entry transmodel.Entry, target interface{}) error { decoder, err := newDecoder(target) if err != nil { return fmt.Errorf("simpleDecode: error creating decoder: %s", err) diff --git a/internal/trans/export.go b/internal/trans/export.go index f6c807d20..bfedd791a 100644 --- a/internal/trans/export.go +++ b/internal/trans/export.go @@ -93,7 +93,7 @@ func (e *exporter) exportDomainBlocks(ctx context.Context, f *os.File) ([]*trans } for _, b := range domainBlocks { - b.Type = transmodel.TransBlock + b.Type = transmodel.TransDomainBlock if err := e.simpleEncode(ctx, f, b, b.ID); err != nil { return nil, fmt.Errorf("exportBlocks: error encoding domain block: %s", err) } diff --git a/internal/trans/exporter.go b/internal/trans/exporter.go index 1cd1b38ff..3dc0558f5 100644 --- a/internal/trans/exporter.go +++ b/internal/trans/exporter.go @@ -25,6 +25,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" ) +// Exporter wraps functionality for exporting entries from the database to a file. type Exporter interface { ExportMinimal(ctx context.Context, path string) error } @@ -35,6 +36,7 @@ type exporter struct { writtenIDs map[string]bool } +// NewExporter returns a new Exporter that will use the given db and logger. func NewExporter(db db.DB, log *logrus.Logger) Exporter { return &exporter{ db: db, diff --git a/internal/trans/exportminimal.go b/internal/trans/exportminimal.go index 5660d5ab6..c2073eb99 100644 --- a/internal/trans/exportminimal.go +++ b/internal/trans/exportminimal.go @@ -136,5 +136,15 @@ func (e *exporter) ExportMinimal(ctx context.Context, path string) error { return fmt.Errorf("ExportMinimal: error exporting instances: %s", err) } + // export all SUSPENDED accounts to make sure the suspension sticks across db migration etc + whereSuspended := []db.Where{{ + Key: "suspended_at", + Not: true, + Value: nil, + }} + if _, err := e.exportAccounts(ctx, whereSuspended, f); err != nil { + return fmt.Errorf("ExportMinimal: error exporting suspended accounts: %s", err) + } + return neatClose(f) } diff --git a/internal/trans/import.go b/internal/trans/import.go index d798aed86..36c735aa8 100644 --- a/internal/trans/import.go +++ b/internal/trans/import.go @@ -30,37 +30,41 @@ import ( ) func (i *importer) Import(ctx context.Context, path string) error { + if path == "" { + return errors.New("Export: path empty") + } + f, err := os.Open(path) if err != nil { - return fmt.Errorf("ImportMinimal: error opening file %s: %s", path, err) + return fmt.Errorf("Import: couldn't export to %s: %s", path, err) } decoder := json.NewDecoder(f) decoder.UseNumber() for { - entry := transmodel.TransEntry{} + entry := transmodel.Entry{} err := decoder.Decode(&entry) if err != nil { if err == io.EOF { - i.log.Infof("ImportMinimal: reached end of file") + i.log.Infof("Import: reached end of file") return neatClose(f) } - return fmt.Errorf("ImportMinimal: error decoding in readLoop: %s", err) + return fmt.Errorf("Import: error decoding in readLoop: %s", err) } if err := i.inputEntry(ctx, entry); err != nil { - return fmt.Errorf("ImportMinimal: error inputting entry: %s", err) + return fmt.Errorf("Import: error inputting entry: %s", err) } } } -func (i *importer) inputEntry(ctx context.Context, entry transmodel.TransEntry) error { +func (i *importer) inputEntry(ctx context.Context, entry transmodel.Entry) error { t, ok := entry[transmodel.TypeKey].(string) if !ok { return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json") } - switch transmodel.TransType(t) { + switch transmodel.Type(t) { case transmodel.TransAccount: account, err := i.accountDecode(entry) if err != nil { @@ -84,12 +88,12 @@ func (i *importer) inputEntry(ctx context.Context, entry transmodel.TransEntry) case transmodel.TransDomainBlock: block, err := i.domainBlockDecode(entry) if err != nil { - return fmt.Errorf("inputEntry: error decoding entry into block: %s", err) + return fmt.Errorf("inputEntry: error decoding entry into domain block: %s", err) } if err := i.putInDB(ctx, block); err != nil { - return fmt.Errorf("inputEntry: error adding block to database: %s", err) + return fmt.Errorf("inputEntry: error adding domain block to database: %s", err) } - i.log.Infof("inputEntry: added block with id %s", block.ID) + i.log.Infof("inputEntry: added domain block with id %s", block.ID) return nil case transmodel.TransFollow: follow, err := i.followDecode(entry) diff --git a/internal/trans/import_test.go b/internal/trans/import_test.go index 0e8791b8c..137a5fae1 100644 --- a/internal/trans/import_test.go +++ b/internal/trans/import_test.go @@ -72,6 +72,18 @@ func (suite *ImportMinimalTestSuite) TestImportMinimalOK() { err = newDB.GetAll(ctx, &blocks) suite.NoError(err) suite.NotEmpty(blocks) + + // we should have some follows in the database + follows := []*gtsmodel.Follow{} + err = newDB.GetAll(ctx, &follows) + suite.NoError(err) + suite.NotEmpty(follows) + + // we should have some domain blocks in the database + domainBlocks := []*gtsmodel.DomainBlock{} + err = newDB.GetAll(ctx, &domainBlocks) + suite.NoError(err) + suite.NotEmpty(domainBlocks) } func TestImportMinimalTestSuite(t *testing.T) { diff --git a/internal/trans/importer.go b/internal/trans/importer.go index 4917cf3d3..ea8866f53 100644 --- a/internal/trans/importer.go +++ b/internal/trans/importer.go @@ -25,6 +25,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" ) +// Importer wraps functionality for importing entries from a file into the database. type Importer interface { Import(ctx context.Context, path string) error } @@ -34,6 +35,7 @@ type importer struct { log *logrus.Logger } +// NewImporter returns a new Importer interface that uses the given db and logger. func NewImporter(db db.DB, log *logrus.Logger) Importer { return &importer{ db: db, diff --git a/internal/trans/model/account.go b/internal/trans/model/account.go index 59aac81a1..011a0ca12 100644 --- a/internal/trans/model/account.go +++ b/internal/trans/model/account.go @@ -25,26 +25,28 @@ import ( // Account represents the minimum viable representation of an account for export/import. type Account struct { - Type TransType `json:"type" bun:"-"` - ID string `json:"id"` - CreatedAt *time.Time `json:"createdAt"` - Username string `json:"username"` + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Username string `json:"username" bun:",nullzero"` Domain string `json:"domain,omitempty" bun:",nullzero"` + HeaderRemoteURL string `json:"headerRemoteURL,omitempty" bun:",nullzero"` + AvatarRemoteURL string `json:"avatarRemoteURL,omitempty" bun:",nullzero"` Locked bool `json:"locked"` - Language string `json:"language,omitempty"` - URI string `json:"uri"` - URL string `json:"url"` - InboxURI string `json:"inboxURI"` - OutboxURI string `json:"outboxURI"` - FollowingURI string `json:"followingUri"` - FollowersURI string `json:"followersUri"` - FeaturedCollectionURI string `json:"featuredCollectionUri"` - ActorType string `json:"actorType"` + Language string `json:"language,omitempty" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + URL string `json:"url" bun:",nullzero"` + InboxURI string `json:"inboxURI" bun:",nullzero"` + OutboxURI string `json:"outboxURI" bun:",nullzero"` + FollowingURI string `json:"followingUri" bun:",nullzero"` + FollowersURI string `json:"followersUri" bun:",nullzero"` + FeaturedCollectionURI string `json:"featuredCollectionUri" bun:",nullzero"` + ActorType string `json:"actorType" bun:",nullzero"` PrivateKey *rsa.PrivateKey `json:"-" mapstructure:"-"` - PrivateKeyString string `json:"privateKey,omitempty" bun:"-" mapstructure:"privateKey"` + PrivateKeyString string `json:"privateKey,omitempty" bun:"-" mapstructure:"privateKey" bun:",nullzero"` PublicKey *rsa.PublicKey `json:"-" mapstructure:"-"` - PublicKeyString string `json:"publicKey,omitempty" bun:"-" mapstructure:"publicKey"` - PublicKeyURI string `json:"publicKeyUri"` - SuspendedAt *time.Time `json:"suspendedAt,omitempty"` + PublicKeyString string `json:"publicKey,omitempty" bun:"-" mapstructure:"publicKey" bun:",nullzero"` + PublicKeyURI string `json:"publicKeyUri" bun:",nullzero"` + SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"` SuspensionOrigin string `json:"suspensionOrigin,omitempty" bun:",nullzero"` } diff --git a/internal/trans/model/block.go b/internal/trans/model/block.go index 34b6cabc8..313e6a7cd 100644 --- a/internal/trans/model/block.go +++ b/internal/trans/model/block.go @@ -20,11 +20,12 @@ package trans import "time" +// Block represents an account block as serialized in an exported file. type Block struct { - Type TransType `json:"type" bun:"-"` - ID string `json:"id"` - CreatedAt *time.Time `json:"createdAt"` - URI string `json:"uri"` - AccountID string `json:"accountId"` - TargetAccountID string `json:"targetAccountId"` + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` } diff --git a/internal/trans/model/domainblock.go b/internal/trans/model/domainblock.go index 5e9d1bc3d..de2bcd00a 100644 --- a/internal/trans/model/domainblock.go +++ b/internal/trans/model/domainblock.go @@ -20,13 +20,15 @@ package trans import "time" +// DomainBlock represents a domain block as serialized in an exported file. type DomainBlock struct { - Type TransType `json:"type" bun:"-"` - ID string `json:"id"` - CreatedAt *time.Time `json:"createdAt"` - CreatedByAccountID string `json:"createdByAccountID"` - PrivateComment string `json:"privateComment,omitempty"` - PublicComment string `json:"publicComment,omitempty"` - Obfuscate bool `json:"obfuscate"` - SubscriptionID string `json:"subscriptionID,omitempty"` + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Domain string `json:"domain" bun:",nullzero"` + CreatedByAccountID string `json:"createdByAccountID" bun:",nullzero"` + PrivateComment string `json:"privateComment,omitempty" bun:",nullzero"` + PublicComment string `json:"publicComment,omitempty" bun:",nullzero"` + Obfuscate bool `json:"obfuscate" bun:",nullzero"` + SubscriptionID string `json:"subscriptionID,omitempty" bun:",nullzero"` } diff --git a/internal/trans/model/follow.go b/internal/trans/model/follow.go index 84379c335..b94f2600d 100644 --- a/internal/trans/model/follow.go +++ b/internal/trans/model/follow.go @@ -20,11 +20,12 @@ package trans import "time" +// Follow represents an account follow as serialized in an export file. type Follow struct { - Type TransType `json:"type" bun:"-"` - ID string `json:"id"` - CreatedAt *time.Time `json:"createdAt"` - URI string `json:"uri"` - AccountID string `json:"accountId"` - TargetAccountID string `json:"targetAccountId"` + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` } diff --git a/internal/trans/model/followrequest.go b/internal/trans/model/followrequest.go index f394bdacc..844bcb7af 100644 --- a/internal/trans/model/followrequest.go +++ b/internal/trans/model/followrequest.go @@ -20,11 +20,12 @@ package trans import "time" +// FollowRequest represents an account follow request as serialized in an export file. type FollowRequest struct { - Type TransType `json:"type" bun:"-"` - ID string `json:"id"` - CreatedAt *time.Time `json:"createdAt"` - URI string `json:"uri"` - AccountID string `json:"accountId"` - TargetAccountID string `json:"targetAccountId"` + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + URI string `json:"uri" bun:",nullzero"` + AccountID string `json:"accountId" bun:",nullzero"` + TargetAccountID string `json:"targetAccountId" bun:",nullzero"` } diff --git a/internal/trans/model/instance.go b/internal/trans/model/instance.go index a099161bd..a75aa65bf 100644 --- a/internal/trans/model/instance.go +++ b/internal/trans/model/instance.go @@ -22,13 +22,14 @@ import ( "time" ) +// Instance represents an instance entry as serialized in an export file. type Instance struct { - Type TransType `json:"type" bun:"-"` - ID string `json:"id"` - CreatedAt *time.Time `json:"createdAt"` - Domain string `json:"domain"` + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` + Domain string `json:"domain" bun:",nullzero"` Title string `json:"title,omitempty" bun:",nullzero"` - URI string `json:"uri"` + URI string `json:"uri" bun:",nullzero"` SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"` DomainBlockID string `json:"domainBlockID,omitempty" bun:",nullzero"` ShortDescription string `json:"shortDescription,omitempty" bun:",nullzero"` diff --git a/internal/trans/model/type.go b/internal/trans/model/type.go index ae040a5a2..76f57c843 100644 --- a/internal/trans/model/type.go +++ b/internal/trans/model/type.go @@ -18,21 +18,24 @@ package trans +// TypeKey should be set on a TransEntry to indicate the type of entry it is. const TypeKey = "type" -// TransType describes the type of a trans entry, and how it should be read/serialized. -type TransType string +// Type describes the type of a trans entry, and how it should be read/serialized. +type Type string // Type of the trans entry. Describes how it should be read from file. const ( - TransAccount TransType = "account" - TransBlock TransType = "block" - TransDomainBlock TransType = "domainBlock" - TransEmailDomainBlock TransType = "emailDomainBlock" - TransFollow TransType = "follow" - TransFollowRequest TransType = "followRequest" - TransInstance TransType = "instance" - TransUser TransType = "user" + TransAccount Type = "account" + TransBlock Type = "block" + TransDomainBlock Type = "domainBlock" + TransEmailDomainBlock Type = "emailDomainBlock" + TransFollow Type = "follow" + TransFollowRequest Type = "followRequest" + TransInstance Type = "instance" + TransUser Type = "user" ) -type TransEntry map[string]interface{} +// Entry is used for deserializing trans entries into a rough interface so that +// the TypeKey can be fetched, before continuing with full parsing. +type Entry map[string]interface{} diff --git a/internal/trans/model/user.go b/internal/trans/model/user.go index c360f872d..293b124a2 100644 --- a/internal/trans/model/user.go +++ b/internal/trans/model/user.go @@ -22,13 +22,14 @@ import ( "time" ) +// User represents a local instance user as serialized to an export file. type User struct { - Type TransType `json:"type" bun:"-"` - ID string `json:"id"` - CreatedAt *time.Time `json:"createdAt"` + Type Type `json:"type" bun:"-"` + ID string `json:"id" bun:",nullzero"` + CreatedAt *time.Time `json:"createdAt" bun:",nullzero"` Email string `json:"email,omitempty" bun:",nullzero"` - AccountID string `json:"accountID"` - EncryptedPassword string `json:"encryptedPassword"` + AccountID string `json:"accountID" bun:",nullzero"` + EncryptedPassword string `json:"encryptedPassword" bun:",nullzero"` CurrentSignInAt *time.Time `json:"currentSignInAt,omitempty" bun:",nullzero"` LastSignInAt *time.Time `json:"lastSignInAt,omitempty" bun:",nullzero"` InviteID string `json:"inviteID,omitempty" bun:",nullzero"` diff --git a/testrig/db.go b/testrig/db.go index 5798af5ce..268ba16b7 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -123,6 +123,12 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) { } } + for _, v := range NewTestDomainBlocks() { + if err := db.Put(ctx, v); err != nil { + logrus.Panic(err) + } + } + for _, v := range NewTestUsers() { if err := db.Put(ctx, v); err != nil { logrus.Panic(err) diff --git a/testrig/testmodels.go b/testrig/testmodels.go index 45f47f46a..311b89e4b 100644 --- a/testrig/testmodels.go +++ b/testrig/testmodels.go @@ -736,6 +736,19 @@ func NewTestEmojis() map[string]*gtsmodel.Emoji { } } +func NewTestDomainBlocks() map[string]*gtsmodel.DomainBlock { + return map[string]*gtsmodel.DomainBlock{ + "replyguys.com": { + ID: "01FF22EQM7X8E3RX1XGPN7S87D", + Domain: "replyguys.com", + CreatedByAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", + PrivateComment: "i blocked this domain because they keep replying with pushy + unwarranted linux advice", + PublicComment: "reply-guying to tech posts", + Obfuscate: false, + }, + } +} + type filenames struct { Original string Small string @@ -836,6 +849,33 @@ func NewTestStatuses() map[string]*gtsmodel.Status { }, ActivityStreamsType: ap.ObjectNote, }, + "admin_account_status_3": { + ID: "01FF25D5Q0DH7CHD57CTRS6WK0", + URI: "http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0", + URL: "http://localhost:8080/@admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0", + Content: "hi @the_mighty_zork welcome to the instance!", + CreatedAt: time.Now().Add(-46 * time.Hour), + UpdatedAt: time.Now().Add(-46 * time.Hour), + Local: true, + AccountURI: "http://localhost:8080/users/admin", + MentionIDs: []string{"01FF26A6BGEKCZFWNEHXB2ZZ6M"}, + AccountID: "01F8MH17FWEB39HZJ76B6VXSKF", + InReplyToID: "01F8MHAMCHF6Y650WCRSCP4WMY", + InReplyToAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + InReplyToURI: "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY", + BoostOfID: "", + Visibility: gtsmodel.VisibilityPublic, + Sensitive: false, + Language: "en", + CreatedWithApplicationID: "01F8MGXQRHYF5QPMTMXP78QC2F", + VisibilityAdvanced: gtsmodel.VisibilityAdvanced{ + Federated: true, + Boostable: true, + Replyable: true, + Likeable: true, + }, + ActivityStreamsType: ap.ObjectNote, + }, "local_account_1_status_1": { ID: "01F8MHAMCHF6Y650WCRSCP4WMY", URI: "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY", @@ -1149,6 +1189,18 @@ func NewTestMentions() map[string]*gtsmodel.Mention { TargetAccountURI: "http://localhost:8080/users/the_mighty_zork", TargetAccountURL: "http://localhost:8080/@the_mighty_zork", }, + "admin_account_mention_zork": { + ID: "01FF26A6BGEKCZFWNEHXB2ZZ6M", + StatusID: "01FF25D5Q0DH7CHD57CTRS6WK0", + CreatedAt: time.Now().Add(-46 * time.Hour), + UpdatedAt: time.Now().Add(-46 * time.Hour), + OriginAccountID: "01F8MH17FWEB39HZJ76B6VXSKF", + OriginAccountURI: "http://localhost:8080/users/admin", + TargetAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + NameString: "@the_mighty_zork", + TargetAccountURI: "http://localhost:8080/users/the_mighty_zork", + TargetAccountURL: "http://localhost:8080/@the_mighty_zork", + }, } } @@ -1387,7 +1439,7 @@ func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[strin DateHeader: date, } - target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5") + target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0") sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target) fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{ SignatureHeader: sig,