diff --git a/cmd/gotosocial/admincommands.go b/cmd/gotosocial/admincommands.go index a777ee525..5b60ad050 100644 --- a/cmd/gotosocial/admincommands.go +++ b/cmd/gotosocial/admincommands.go @@ -20,6 +20,7 @@ package main import ( "github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/account" + "github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/export" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/urfave/cli/v2" ) @@ -41,14 +42,17 @@ func adminCommands() []*cli.Command { &cli.StringFlag{ Name: config.UsernameFlag, Usage: config.UsernameUsage, + Required: true, }, &cli.StringFlag{ Name: config.EmailFlag, Usage: config.EmailUsage, + Required: true, }, &cli.StringFlag{ Name: config.PasswordFlag, Usage: config.PasswordUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -62,6 +66,7 @@ func adminCommands() []*cli.Command { &cli.StringFlag{ Name: config.UsernameFlag, Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -75,6 +80,7 @@ func adminCommands() []*cli.Command { &cli.StringFlag{ Name: config.UsernameFlag, Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -88,6 +94,7 @@ func adminCommands() []*cli.Command { &cli.StringFlag{ Name: config.UsernameFlag, Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -101,6 +108,7 @@ func adminCommands() []*cli.Command { &cli.StringFlag{ Name: config.UsernameFlag, Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -114,6 +122,7 @@ func adminCommands() []*cli.Command { &cli.StringFlag{ Name: config.UsernameFlag, Usage: config.UsernameUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -127,10 +136,12 @@ func adminCommands() []*cli.Command { &cli.StringFlag{ Name: config.UsernameFlag, Usage: config.UsernameUsage, + Required: true, }, &cli.StringFlag{ Name: config.PasswordFlag, Usage: config.PasswordUsage, + Required: true, }, }, Action: func(c *cli.Context) error { @@ -139,6 +150,20 @@ func adminCommands() []*cli.Command { }, }, }, + { + Name: "export", + Usage: "export data from the database to file at the given path", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: config.TransPathFlag, + Usage: config.TransPathUsage, + Required: true, + }, + }, + Action: func(c *cli.Context) error { + return runAction(c, export.Export) + }, + }, }, }, } diff --git a/internal/cliactions/admin/export/account.go b/internal/cliactions/admin/export/account.go new file mode 100644 index 000000000..f680b1c0a --- /dev/null +++ b/internal/cliactions/admin/export/account.go @@ -0,0 +1,52 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package export + +import ( + "context" + "errors" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/cliactions" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db/bundb" + "github.com/superseriousbusiness/gotosocial/internal/trans" +) + +// Export exports info from the database into a file +var Export cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error { + dbConn, err := bundb.NewBunDBService(ctx, c, log) + if err != nil { + return fmt.Errorf("error creating dbservice: %s", err) + } + + exporter := trans.NewExporter(dbConn, log) + + path, ok := c.ExportCLIFlags[config.TransPathFlag] + if !ok { + return errors.New("no path set") + } + + if err := exporter.ExportMinimal(ctx, path); err != nil { + return err + } + + return dbConn.Stop(ctx) +} diff --git a/internal/config/config.go b/internal/config/config.go index 68e958995..a6768c5b9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,6 +36,9 @@ const ( PasswordFlag = "password" PasswordUsage = "the password to set for this account" + + TransPathFlag = "path" + TransPathUsage = "the path of the file to import from/export to" ) // Config pulls together all the configuration needed to run gotosocial @@ -65,6 +68,7 @@ type Config struct { Not parsed from .yaml configuration file. */ AccountCLIFlags map[string]string + ExportCLIFlags map[string]string SoftwareVersion string } @@ -92,6 +96,7 @@ func Empty() *Config { LetsEncryptConfig: &LetsEncryptConfig{}, OIDCConfig: &OIDCConfig{}, AccountCLIFlags: make(map[string]string), + ExportCLIFlags: make(map[string]string), } } @@ -320,6 +325,9 @@ func (c *Config) ParseCLIFlags(f KeyedFlags, version string) error { c.AccountCLIFlags[EmailFlag] = f.String(EmailFlag) c.AccountCLIFlags[PasswordFlag] = f.String(PasswordFlag) + // export CLI flags + c.ExportCLIFlags[TransPathFlag] = f.String(TransPathFlag) + c.SoftwareVersion = version return nil } diff --git a/internal/trans/decoders.go b/internal/trans/decoders.go index 33557b2b5..6e73881d6 100644 --- a/internal/trans/decoders.go +++ b/internal/trans/decoders.go @@ -79,6 +79,51 @@ func (i *importer) blockDecode(e transmodel.TransEntry) (*transmodel.Block, erro return b, nil } +func (i *importer) domainBlockDecode(e transmodel.TransEntry) (*transmodel.DomainBlock, error) { + b := &transmodel.DomainBlock{} + if err := i.simpleDecode(e, b); err != nil { + return nil, err + } + + return b, nil +} + +func (i *importer) followDecode(e transmodel.TransEntry) (*transmodel.Follow, error) { + f := &transmodel.Follow{} + if err := i.simpleDecode(e, f); err != nil { + return nil, err + } + + return f, nil +} + +func (i *importer) followRequestDecode(e transmodel.TransEntry) (*transmodel.FollowRequest, error) { + f := &transmodel.FollowRequest{} + if err := i.simpleDecode(e, f); err != nil { + return nil, err + } + + return f, nil +} + +func (i *importer) instanceDecode(e transmodel.TransEntry) (*transmodel.Instance, error) { + inst := &transmodel.Instance{} + if err := i.simpleDecode(e, inst); err != nil { + return nil, err + } + + return inst, nil +} + +func (i *importer) userDecode(e transmodel.TransEntry) (*transmodel.User, error) { + u := &transmodel.User{} + if err := i.simpleDecode(e, u); err != nil { + return nil, err + } + + return u, nil +} + func (i *importer) simpleDecode(entry transmodel.TransEntry, target interface{}) error { decoder, err := newDecoder(target) if err != nil { diff --git a/internal/trans/encoders.go b/internal/trans/encoders.go index 89dba6eaa..62ffa34d9 100644 --- a/internal/trans/encoders.go +++ b/internal/trans/encoders.go @@ -64,8 +64,8 @@ func (e *exporter) accountEncode(ctx context.Context, f *os.File, a *transmodel. // simpleEncode can be used for any type that doesn't have special keys which need handling differently, // or for types where special keys have already been handled. // -// The 'type' key on the passed interface should already have been set, since simpleEncode won't know -// what type it is! +// Beware, the 'type' key on the passed interface should already have been set, since simpleEncode won't know +// what type it is! If you try to decode stuff you've encoded with a missing type key, you're going to have a bad time. func (e *exporter) simpleEncode(ctx context.Context, f *os.File, i interface{}, id string) error { _, alreadyWritten := e.writtenIDs[id] if alreadyWritten { diff --git a/internal/trans/export.go b/internal/trans/export.go new file mode 100644 index 000000000..f6c807d20 --- /dev/null +++ b/internal/trans/export.go @@ -0,0 +1,223 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/package trans + +import ( + "context" + "fmt" + "os" + + "github.com/superseriousbusiness/gotosocial/internal/db" + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, f *os.File) ([]*transmodel.Account, error) { + // select using the 'where' we've been provided + accounts := []*transmodel.Account{} + if err := e.db.GetWhere(ctx, where, &accounts); err != nil { + return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err) + } + + // write any accounts found to file + for _, a := range accounts { + if err := e.accountEncode(ctx, f, a); err != nil { + return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err) + } + } + + return accounts, nil +} + +func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.Block, error) { + blocksUnique := make(map[string]*transmodel.Block) + + // for each account we want to export both where it's blocking and where it's blocked + for _, a := range accounts { + // 1. export blocks owned by given account + whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}} + blocking := []*transmodel.Block{} + if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err) + } + for _, b := range blocking { + b.Type = transmodel.TransBlock + if err := e.simpleEncode(ctx, f, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err) + } + blocksUnique[b.ID] = b + } + + // 2. export blocks that target given account + whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}} + blocked := []*transmodel.Block{} + if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err) + } + for _, b := range blocked { + b.Type = transmodel.TransBlock + if err := e.simpleEncode(ctx, f, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err) + } + blocksUnique[b.ID] = b + } + } + + // now return all the blocks we found + blocks := []*transmodel.Block{} + for _, b := range blocksUnique { + blocks = append(blocks, b) + } + + return blocks, nil +} + +func (e *exporter) exportDomainBlocks(ctx context.Context, f *os.File) ([]*transmodel.DomainBlock, error) { + domainBlocks := []*transmodel.DomainBlock{} + + if err := e.db.GetAll(ctx, &domainBlocks); err != nil { + return nil, fmt.Errorf("exportBlocks: error selecting domain blocks: %s", err) + } + + for _, b := range domainBlocks { + b.Type = transmodel.TransBlock + if err := e.simpleEncode(ctx, f, b, b.ID); err != nil { + return nil, fmt.Errorf("exportBlocks: error encoding domain block: %s", err) + } + } + + return domainBlocks, nil +} + +func (e *exporter) exportFollows(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.Follow, error) { + followsUnique := make(map[string]*transmodel.Follow) + + // for each account we want to export both where it's following and where it's followed + for _, a := range accounts { + // 1. export follows owned by given account + whereFollowing := []db.Where{{Key: "account_id", Value: a.ID}} + following := []*transmodel.Follow{} + if err := e.db.GetWhere(ctx, whereFollowing, &following); err != nil { + return nil, fmt.Errorf("exportFollows: error selecting follows owned by account %s: %s", a.ID, err) + } + for _, follow := range following { + follow.Type = transmodel.TransFollow + if err := e.simpleEncode(ctx, f, follow, follow.ID); err != nil { + return nil, fmt.Errorf("exportFollows: error encoding follow owned by account %s: %s", a.ID, err) + } + followsUnique[follow.ID] = follow + } + + // 2. export follows that target given account + whereFollowed := []db.Where{{Key: "target_account_id", Value: a.ID}} + followed := []*transmodel.Follow{} + if err := e.db.GetWhere(ctx, whereFollowed, &followed); err != nil { + return nil, fmt.Errorf("exportFollows: error selecting follows targeting account %s: %s", a.ID, err) + } + for _, follow := range followed { + follow.Type = transmodel.TransFollow + if err := e.simpleEncode(ctx, f, follow, follow.ID); err != nil { + return nil, fmt.Errorf("exportFollows: error encoding follow targeting account %s: %s", a.ID, err) + } + followsUnique[follow.ID] = follow + } + } + + // now return all the follows we found + follows := []*transmodel.Follow{} + for _, follow := range followsUnique { + follows = append(follows, follow) + } + + return follows, nil +} + +func (e *exporter) exportFollowRequests(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.FollowRequest, error) { + frsUnique := make(map[string]*transmodel.FollowRequest) + + // for each account we want to export both where it's following and where it's followed + for _, a := range accounts { + // 1. export follow requests owned by given account + whereRequesting := []db.Where{{Key: "account_id", Value: a.ID}} + requesting := []*transmodel.FollowRequest{} + if err := e.db.GetWhere(ctx, whereRequesting, &requesting); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests owned by account %s: %s", a.ID, err) + } + for _, fr := range requesting { + fr.Type = transmodel.TransFollowRequest + if err := e.simpleEncode(ctx, f, fr, fr.ID); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error encoding follow request owned by account %s: %s", a.ID, err) + } + frsUnique[fr.ID] = fr + } + + // 2. export follow requests that target given account + whereRequested := []db.Where{{Key: "target_account_id", Value: a.ID}} + requested := []*transmodel.FollowRequest{} + if err := e.db.GetWhere(ctx, whereRequested, &requested); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests targeting account %s: %s", a.ID, err) + } + for _, fr := range requested { + fr.Type = transmodel.TransFollowRequest + if err := e.simpleEncode(ctx, f, fr, fr.ID); err != nil { + return nil, fmt.Errorf("exportFollowRequests: error encoding follow request targeting account %s: %s", a.ID, err) + } + frsUnique[fr.ID] = fr + } + } + + // now return all the followRequests we found + followRequests := []*transmodel.FollowRequest{} + for _, fr := range frsUnique { + followRequests = append(followRequests, fr) + } + + return followRequests, nil +} + +func (e *exporter) exportInstances(ctx context.Context, f *os.File) ([]*transmodel.Instance, error) { + instances := []*transmodel.Instance{} + + if err := e.db.GetAll(ctx, &instances); err != nil { + return nil, fmt.Errorf("exportInstances: error selecting instance: %s", err) + } + + for _, u := range instances { + u.Type = transmodel.TransInstance + if err := e.simpleEncode(ctx, f, u, u.ID); err != nil { + return nil, fmt.Errorf("exportInstances: error encoding instance: %s", err) + } + } + + return instances, nil +} + +func (e *exporter) exportUsers(ctx context.Context, f *os.File) ([]*transmodel.User, error) { + users := []*transmodel.User{} + + if err := e.db.GetAll(ctx, &users); err != nil { + return nil, fmt.Errorf("exportUsers: error selecting users: %s", err) + } + + for _, u := range users { + u.Type = transmodel.TransUser + if err := e.simpleEncode(ctx, f, u, u.ID); err != nil { + return nil, fmt.Errorf("exportUsers: error encoding user: %s", err) + } + } + + return users, nil +} diff --git a/internal/trans/exportminimal.go b/internal/trans/exportminimal.go index dd5a9995c..5660d5ab6 100644 --- a/internal/trans/exportminimal.go +++ b/internal/trans/exportminimal.go @@ -20,17 +20,21 @@ package trans import ( "context" + "errors" "fmt" "os" "github.com/superseriousbusiness/gotosocial/internal/db" - transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" ) func (e *exporter) ExportMinimal(ctx context.Context, path string) error { + if path == "" { + return errors.New("ExportMinimal: path empty") + } + f, err := os.Create(path) if err != nil { - return err + return fmt.Errorf("ExportMinimal: couldn't export to %s: %s", path, err) } // export all local accounts we have in the database @@ -39,13 +43,14 @@ func (e *exporter) ExportMinimal(ctx context.Context, path string) error { return fmt.Errorf("ExportMinimal: error exporting accounts: %s", err) } - // export all blocks that relate to those accounts + // export all blocks that relate to local accounts blocks, err := e.exportBlocks(ctx, localAccounts, f) if err != nil { return fmt.Errorf("ExportMinimal: error exporting blocks: %s", err) } - // for each block, make sure we've written out the account owning it, or targeted by it + // for each block, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything for _, b := range blocks { _, alreadyWritten := e.writtenIDs[b.AccountID] if !alreadyWritten { @@ -64,65 +69,72 @@ func (e *exporter) ExportMinimal(ctx context.Context, path string) error { } } + // export all follows that relate to local accounts + follows, err := e.exportFollows(ctx, localAccounts, f) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follows: %s", err) + } + + // for each follow, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything + for _, follow := range follows { + _, alreadyWritten := e.writtenIDs[follow.AccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.AccountID}}, f) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow owner account: %s", err) + } + } + + _, alreadyWritten = e.writtenIDs[follow.TargetAccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.TargetAccountID}}, f) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow target account: %s", err) + } + } + } + + // export all follow requests that relate to local accounts + frs, err := e.exportFollowRequests(ctx, localAccounts, f) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow requests: %s", err) + } + + // for each follow request, make sure we've written out the account owning it, or targeted by it -- + // this might include non-local accounts, but we need these so we don't lose anything + for _, fr := range frs { + _, alreadyWritten := e.writtenIDs[fr.AccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.AccountID}}, f) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow request owner account: %s", err) + } + } + + _, alreadyWritten = e.writtenIDs[fr.TargetAccountID] + if !alreadyWritten { + _, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.TargetAccountID}}, f) + if err != nil { + return fmt.Errorf("ExportMinimal: error exporting follow request target account: %s", err) + } + } + } + + // export all domain blocks + if _, err := e.exportDomainBlocks(ctx, f); err != nil { + return fmt.Errorf("ExportMinimal: error exporting domain blocks: %s", err) + } + + // export all users + if _, err := e.exportUsers(ctx, f); err != nil { + return fmt.Errorf("ExportMinimal: error exporting users: %s", err) + } + + // export all instances + if _, err := e.exportInstances(ctx, f); err != nil { + return fmt.Errorf("ExportMinimal: error exporting instances: %s", err) + } + return neatClose(f) } - -func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, f *os.File) ([]*transmodel.Account, error) { - // select using the 'where' we've been provided - accounts := []*transmodel.Account{} - if err := e.db.GetWhere(ctx, where, &accounts); err != nil { - return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err) - } - - // write any accounts found to file - for _, a := range accounts { - if err := e.accountEncode(ctx, f, a); err != nil { - return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err) - } - } - - return accounts, nil -} - -func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.Block, error) { - blocksUnique := make(map[string]*transmodel.Block) - - // for each account we want to export both where it's blocking and where it's blocked - for _, a := range accounts { - // 1. export blocks owned by given account - whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}} - blocking := []*transmodel.Block{} - if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil { - return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err) - } - for _, b := range blocking { - b.Type = transmodel.TransBlock - if err := e.simpleEncode(ctx, f, b, b.ID); err != nil { - return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err) - } - blocksUnique[b.ID] = b - } - - // 2. export blocks that target given account - whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}} - blocked := []*transmodel.Block{} - if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil { - return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err) - } - for _, b := range blocked { - b.Type = transmodel.TransBlock - if err := e.simpleEncode(ctx, f, b, b.ID); err != nil { - return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err) - } - blocksUnique[b.ID] = b - } - } - - // now return all the blocks we found - blocks := []*transmodel.Block{} - for _, b := range blocksUnique { - blocks = append(blocks, b) - } - - return blocks, nil -} diff --git a/internal/trans/import.go b/internal/trans/import.go new file mode 100644 index 000000000..d798aed86 --- /dev/null +++ b/internal/trans/import.go @@ -0,0 +1,142 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" +) + +func (i *importer) Import(ctx context.Context, path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("ImportMinimal: error opening file %s: %s", path, err) + } + + decoder := json.NewDecoder(f) + decoder.UseNumber() + + for { + entry := transmodel.TransEntry{} + err := decoder.Decode(&entry) + if err != nil { + if err == io.EOF { + i.log.Infof("ImportMinimal: reached end of file") + return neatClose(f) + } + return fmt.Errorf("ImportMinimal: error decoding in readLoop: %s", err) + } + if err := i.inputEntry(ctx, entry); err != nil { + return fmt.Errorf("ImportMinimal: error inputting entry: %s", err) + } + } +} + +func (i *importer) inputEntry(ctx context.Context, entry transmodel.TransEntry) error { + t, ok := entry[transmodel.TypeKey].(string) + if !ok { + return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json") + } + + switch transmodel.TransType(t) { + case transmodel.TransAccount: + account, err := i.accountDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into account: %s", err) + } + if err := i.putInDB(ctx, account); err != nil { + return fmt.Errorf("inputEntry: error adding account to database: %s", err) + } + i.log.Infof("inputEntry: added account with id %s", account.ID) + return nil + case transmodel.TransBlock: + block, err := i.blockDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into block: %s", err) + } + if err := i.putInDB(ctx, block); err != nil { + return fmt.Errorf("inputEntry: error adding block to database: %s", err) + } + i.log.Infof("inputEntry: added block with id %s", block.ID) + return nil + case transmodel.TransDomainBlock: + block, err := i.domainBlockDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into block: %s", err) + } + if err := i.putInDB(ctx, block); err != nil { + return fmt.Errorf("inputEntry: error adding block to database: %s", err) + } + i.log.Infof("inputEntry: added block with id %s", block.ID) + return nil + case transmodel.TransFollow: + follow, err := i.followDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into follow: %s", err) + } + if err := i.putInDB(ctx, follow); err != nil { + return fmt.Errorf("inputEntry: error adding follow to database: %s", err) + } + i.log.Infof("inputEntry: added follow with id %s", follow.ID) + return nil + case transmodel.TransFollowRequest: + fr, err := i.followRequestDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into follow request: %s", err) + } + if err := i.putInDB(ctx, fr); err != nil { + return fmt.Errorf("inputEntry: error adding follow request to database: %s", err) + } + i.log.Infof("inputEntry: added follow request with id %s", fr.ID) + return nil + case transmodel.TransInstance: + inst, err := i.instanceDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into instance: %s", err) + } + if err := i.putInDB(ctx, inst); err != nil { + return fmt.Errorf("inputEntry: error adding instance to database: %s", err) + } + i.log.Infof("inputEntry: added instance with id %s", inst.ID) + return nil + case transmodel.TransUser: + user, err := i.userDecode(entry) + if err != nil { + return fmt.Errorf("inputEntry: error decoding entry into user: %s", err) + } + if err := i.putInDB(ctx, user); err != nil { + return fmt.Errorf("inputEntry: error adding user to database: %s", err) + } + i.log.Infof("inputEntry: added user with id %s", user.ID) + return nil + } + + i.log.Errorf("inputEntry: didn't recognize transtype '%s', skipping it", t) + return nil +} + +func (i *importer) putInDB(ctx context.Context, entry interface{}) error { + return i.db.Put(ctx, entry) +} diff --git a/internal/trans/importminimal_test.go b/internal/trans/import_test.go similarity index 86% rename from internal/trans/importminimal_test.go rename to internal/trans/import_test.go index af6e25a34..0e8791b8c 100644 --- a/internal/trans/importminimal_test.go +++ b/internal/trans/import_test.go @@ -26,7 +26,6 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/trans" "github.com/superseriousbusiness/gotosocial/testrig" @@ -51,7 +50,7 @@ func (suite *ImportMinimalTestSuite) TestImportMinimalOK() { b, err := os.ReadFile(tempFilePath) suite.NoError(err) suite.NotEmpty(b) - suite.T().Log(string(b)) + fmt.Println(string(b)) // create a new database with just the tables created, no entries testrig.StandardDBTeardown(suite.db) @@ -59,14 +58,20 @@ func (suite *ImportMinimalTestSuite) TestImportMinimalOK() { testrig.CreateTestTables(newDB) importer := trans.NewImporter(newDB, suite.log) - err = importer.ImportMinimal(ctx, tempFilePath) + err = importer.Import(ctx, tempFilePath) suite.NoError(err) - // we should now have some accounts in the database + // we should have some accounts in the database accounts := []*gtsmodel.Account{} - err = newDB.GetWhere(ctx, []db.Where{{Key: "domain", Value: nil}}, &accounts) + err = newDB.GetAll(ctx, &accounts) suite.NoError(err) suite.NotEmpty(accounts) + + // we should have some blocks in the database + blocks := []*gtsmodel.Block{} + err = newDB.GetAll(ctx, &blocks) + suite.NoError(err) + suite.NotEmpty(blocks) } func TestImportMinimalTestSuite(t *testing.T) { diff --git a/internal/trans/importer.go b/internal/trans/importer.go index 812ec0d86..c37d76adc 100644 --- a/internal/trans/importer.go +++ b/internal/trans/importer.go @@ -26,7 +26,7 @@ import ( ) type Importer interface { - ImportMinimal(ctx context.Context, path string) error + Import(ctx context.Context, path string) error } type importer struct { diff --git a/internal/trans/importminimal.go b/internal/trans/importminimal.go deleted file mode 100644 index 7bd182f59..000000000 --- a/internal/trans/importminimal.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package trans - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - - transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" -) - -func (i *importer) ImportMinimal(ctx context.Context, path string) error { - f, err := os.Open(path) - if err != nil { - return fmt.Errorf("ImportMinimal: error opening file %s: %s", path, err) - } - - decoder := json.NewDecoder(f) - decoder.UseNumber() - - for { - entry := transmodel.TransEntry{} - err := decoder.Decode(&entry) - if err != nil { - if err == io.EOF { - i.log.Infof("ImportMinimal: reached end of file") - return neatClose(f) - } - return fmt.Errorf("ImportMinimal: error decoding in readLoop: %s", err) - } - if err := i.inputEntry(ctx, entry); err != nil { - return fmt.Errorf("ImportMinimal: error inputting entry: %s", err) - } - } -} diff --git a/internal/trans/input.go b/internal/trans/input.go deleted file mode 100644 index 70ece0ac8..000000000 --- a/internal/trans/input.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -*/ - -package trans - -import ( - "context" - "errors" - "fmt" - - transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model" -) - -func (i *importer) inputEntry(ctx context.Context, entry transmodel.TransEntry) error { - t, ok := entry[transmodel.TypeKey].(string) - if !ok { - return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json") - } - - switch transmodel.TransType(t) { - case transmodel.TransAccount: - account, err := i.accountDecode(entry) - if err != nil { - return fmt.Errorf("inputEntry: error decoding entry into account: %s", err) - } - if err := i.putInDB(ctx, account); err != nil { - return fmt.Errorf("inputEntry: error adding account to database: %s", err) - } - i.log.Infof("inputEntry: added account with id %s", account.ID) - return nil - } - - return fmt.Errorf("inputEntry: didn't recognize transtype %s", t) -} - -func (i *importer) putInDB(ctx context.Context, entry interface{}) error { - return i.db.Put(ctx, entry) -} diff --git a/internal/trans/model/account.go b/internal/trans/model/account.go index 64cd3b1a3..59aac81a1 100644 --- a/internal/trans/model/account.go +++ b/internal/trans/model/account.go @@ -28,7 +28,6 @@ type Account struct { Type TransType `json:"type" bun:"-"` ID string `json:"id"` CreatedAt *time.Time `json:"createdAt"` - UpdatedAt *time.Time `json:"updatedAt"` Username string `json:"username"` Domain string `json:"domain,omitempty" bun:",nullzero"` Locked bool `json:"locked"` diff --git a/internal/trans/model/block.go b/internal/trans/model/block.go index 1fcd9f20b..781eaa2a2 100644 --- a/internal/trans/model/block.go +++ b/internal/trans/model/block.go @@ -23,8 +23,7 @@ import "time" type Block struct { Type TransType `json:"type" bun:"-"` ID string `json:"id"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + CreatedAt *time.Time `json:"createdAt"` URI string `json:"uri"` AccountID string `json:"accountId"` TargetAccountID string `json:"targetAccountId"` diff --git a/internal/trans/model/domainblock.go b/internal/trans/model/domainblock.go index cc3e8924b..5e9d1bc3d 100644 --- a/internal/trans/model/domainblock.go +++ b/internal/trans/model/domainblock.go @@ -1,3 +1,32 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + package trans +import "time" +type DomainBlock struct { + Type TransType `json:"type" bun:"-"` + ID string `json:"id"` + CreatedAt *time.Time `json:"createdAt"` + CreatedByAccountID string `json:"createdByAccountID"` + PrivateComment string `json:"privateComment,omitempty"` + PublicComment string `json:"publicComment,omitempty"` + Obfuscate bool `json:"obfuscate"` + SubscriptionID string `json:"subscriptionID,omitempty"` +} diff --git a/internal/trans/model/follow.go b/internal/trans/model/follow.go index 854cb4372..7f61dc8da 100644 --- a/internal/trans/model/follow.go +++ b/internal/trans/model/follow.go @@ -23,8 +23,7 @@ import "time" type Follow struct { Type TransType `json:"type" bun:"-"` ID string `json:"id"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + CreatedAt *time.Time `json:"createdAt"` URI string `json:"uri"` AccountID string `json:"accountId"` TargetAccountID string `json:"targetAccountId"` diff --git a/internal/trans/model/followrequest.go b/internal/trans/model/followrequest.go new file mode 100644 index 000000000..3686d7012 --- /dev/null +++ b/internal/trans/model/followrequest.go @@ -0,0 +1,30 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import "time" + +type FollowRequest struct { + Type TransType `json:"type" bun:"-"` + ID string `json:"id"` + CreatedAt *time.Time `json:"createdAt"` + URI string `json:"uri"` + AccountID string `json:"accountId"` + TargetAccountID string `json:"targetAccountId"` +} diff --git a/internal/trans/model/instance.go b/internal/trans/model/instance.go new file mode 100644 index 000000000..15c4da081 --- /dev/null +++ b/internal/trans/model/instance.go @@ -0,0 +1,42 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "time" +) + +type Instance struct { + Type TransType `json:"type" bun:"-"` + ID string `json:"id"` + CreatedAt *time.Time `json:"createdAt"` + Domain string `json:"domain"` + Title string `json:"title,omitempty" bun:",nullzero"` + URI string `json:"uri"` + SuspendedAt *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"` + DomainBlockID string `json:"domainBlockID,omitempty" bun:",nullzero"` + ShortDescription string `json:"shortDescription,omitempty" bun:",nullzero"` + Description string `json:"description,omitempty" bun:",nullzero"` + Terms string `json:"terms,omitempty" bun:",nullzero"` + ContactEmail string `json:"contactEmail,omitempty" bun:",nullzero"` + ContactAccountUsername string `json:"contactAccountUsername,omitempty" bun:",nullzero"` + ContactAccountID string `json:"contactAccountID,omitempty" bun:",nullzero"` + Reputation int64 `json:"reputation"` + Version string `json:"version,omitempty" bun:",nullzero"` +} diff --git a/internal/trans/model/type.go b/internal/trans/model/type.go index 2372203e8..ae040a5a2 100644 --- a/internal/trans/model/type.go +++ b/internal/trans/model/type.go @@ -25,9 +25,14 @@ type TransType string // Type of the trans entry. Describes how it should be read from file. const ( - TransAccount TransType = "account" - TransBlock TransType = "block" - TransFollow TransType = "follow" + TransAccount TransType = "account" + TransBlock TransType = "block" + TransDomainBlock TransType = "domainBlock" + TransEmailDomainBlock TransType = "emailDomainBlock" + TransFollow TransType = "follow" + TransFollowRequest TransType = "followRequest" + TransInstance TransType = "instance" + TransUser TransType = "user" ) type TransEntry map[string]interface{} diff --git a/internal/trans/model/user.go b/internal/trans/model/user.go new file mode 100644 index 000000000..2a39d0d88 --- /dev/null +++ b/internal/trans/model/user.go @@ -0,0 +1,49 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +package trans + +import ( + "time" +) + +type User struct { + Type TransType `json:"type" bun:"-"` + ID string `json:"id"` + CreatedAt *time.Time `json:"createdAt"` + Email string `json:"email,omitempty" bun:",nullzero"` + AccountID string `json:"accountID"` + EncryptedPassword string `json:"encryptedPassword"` + CurrentSignInAt *time.Time `json:"currentSignInAt,omitempty" bun:",nullzero"` + LastSignInAt *time.Time `json:"lastSignInAt,omitempty" bun:",nullzero"` + InviteID string `json:"inviteID,omitempty" bun:",nullzero"` + ChosenLanguages []string `json:"chosenLanguages,omitempty" bun:",nullzero"` + FilteredLanguages []string `json:"filteredLanguage,omitempty" bun:",nullzero"` + Locale string `json:"locale" bun:",nullzero"` + LastEmailedAt time.Time `json:"lastEmailedAt,omitempty" bun:",nullzero"` + ConfirmationToken string `json:"confirmationToken,omitempty" bun:",nullzero"` + ConfirmationSentAt *time.Time `json:"confirmationTokenSentAt,omitempty" bun:",nullzero"` + ConfirmedAt *time.Time `json:"confirmedAt,omitempty" bun:",nullzero"` + UnconfirmedEmail string `json:"unconfirmedEmail,omitempty" bun:",nullzero"` + Moderator bool `json:"moderator"` + Admin bool `json:"admin"` + Disabled bool `json:"disabled"` + Approved bool `json:"approved"` + ResetPasswordToken string `json:"resetPasswordToken,omitempty" bun:",nullzero"` + ResetPasswordSentAt *time.Time `json:"resetPasswordSentAt,omitempty" bun:",nullzero"` +}