working pretty alright!

cmd/gotosocial/admincommands.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/account"
+	"github.com/superseriousbusiness/gotosocial/internal/cliactions/admin/export"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/urfave/cli/v2"
 )
@@ -41,14 +42,17 @@ func adminCommands() []*cli.Command {
 								&cli.StringFlag{
 									Name:  config.UsernameFlag,
 									Usage: config.UsernameUsage,
+									Required: true,
 								},
 								&cli.StringFlag{
 									Name:  config.EmailFlag,
 									Usage: config.EmailUsage,
+									Required: true,
 								},
 								&cli.StringFlag{
 									Name:  config.PasswordFlag,
 									Usage: config.PasswordUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -62,6 +66,7 @@ func adminCommands() []*cli.Command {
 								&cli.StringFlag{
 									Name:  config.UsernameFlag,
 									Usage: config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -75,6 +80,7 @@ func adminCommands() []*cli.Command {
 								&cli.StringFlag{
 									Name:  config.UsernameFlag,
 									Usage: config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -88,6 +94,7 @@ func adminCommands() []*cli.Command {
 								&cli.StringFlag{
 									Name:  config.UsernameFlag,
 									Usage: config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -101,6 +108,7 @@ func adminCommands() []*cli.Command {
 								&cli.StringFlag{
 									Name:  config.UsernameFlag,
 									Usage: config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -114,6 +122,7 @@ func adminCommands() []*cli.Command {
 								&cli.StringFlag{
 									Name:  config.UsernameFlag,
 									Usage: config.UsernameUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -127,10 +136,12 @@ func adminCommands() []*cli.Command {
 								&cli.StringFlag{
 									Name:  config.UsernameFlag,
 									Usage: config.UsernameUsage,
+									Required: true,
 								},
 								&cli.StringFlag{
 									Name:  config.PasswordFlag,
 									Usage: config.PasswordUsage,
+									Required: true,
 								},
 							},
 							Action: func(c *cli.Context) error {
@@ -139,6 +150,20 @@ func adminCommands() []*cli.Command {
 						},
 					},
 				},
+				{
+					Name:  "export",
+					Usage: "export data from the database to file at the given path",
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:  config.TransPathFlag,
+							Usage: config.TransPathUsage,
+							Required: true,
+						},
+					},
+					Action: func(c *cli.Context) error {
+						return runAction(c, export.Export)
+					},
+				},
 			},
 		},
 	}

internal/cliactions/admin/export/account.go

@@ -0,0 +1,52 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package export
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/cliactions"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+)
+
+// Export exports info from the database into a file
+var Export cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
+	dbConn, err := bundb.NewBunDBService(ctx, c, log)
+	if err != nil {
+		return fmt.Errorf("error creating dbservice: %s", err)
+	}
+
+	exporter := trans.NewExporter(dbConn, log)
+
+	path, ok := c.ExportCLIFlags[config.TransPathFlag]
+	if !ok {
+		return errors.New("no path set")
+	}
+
+	if err := exporter.ExportMinimal(ctx, path); err != nil {
+		return err
+	}
+
+	return dbConn.Stop(ctx)
+}

internal/config/config.go

@@ -36,6 +36,9 @@ const (
 
 	PasswordFlag  = "password"
 	PasswordUsage = "the password to set for this account"
+
+	TransPathFlag = "path"
+	TransPathUsage = "the path of the file to import from/export to"
 )
 
 // Config pulls together all the configuration needed to run gotosocial
@@ -65,6 +68,7 @@ type Config struct {
 		Not parsed from .yaml configuration file.
 	*/
 	AccountCLIFlags map[string]string
+	ExportCLIFlags  map[string]string
 	SoftwareVersion string
 }
 
@@ -92,6 +96,7 @@ func Empty() *Config {
 		LetsEncryptConfig: &LetsEncryptConfig{},
 		OIDCConfig:        &OIDCConfig{},
 		AccountCLIFlags:   make(map[string]string),
+		ExportCLIFlags:    make(map[string]string),
 	}
 }
 
@@ -320,6 +325,9 @@ func (c *Config) ParseCLIFlags(f KeyedFlags, version string) error {
 	c.AccountCLIFlags[EmailFlag] = f.String(EmailFlag)
 	c.AccountCLIFlags[PasswordFlag] = f.String(PasswordFlag)
 
+	// export CLI flags
+	c.ExportCLIFlags[TransPathFlag] = f.String(TransPathFlag)
+
 	c.SoftwareVersion = version
 	return nil
 }

internal/trans/decoders.go

@@ -79,6 +79,51 @@ func (i *importer) blockDecode(e transmodel.TransEntry) (*transmodel.Block, erro
 	return b, nil
 }
 
+func (i *importer) domainBlockDecode(e transmodel.TransEntry) (*transmodel.DomainBlock, error) {
+	b := &transmodel.DomainBlock{}
+	if err := i.simpleDecode(e, b); err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (i *importer) followDecode(e transmodel.TransEntry) (*transmodel.Follow, error) {
+	f := &transmodel.Follow{}
+	if err := i.simpleDecode(e, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func (i *importer) followRequestDecode(e transmodel.TransEntry) (*transmodel.FollowRequest, error) {
+	f := &transmodel.FollowRequest{}
+	if err := i.simpleDecode(e, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func (i *importer) instanceDecode(e transmodel.TransEntry) (*transmodel.Instance, error) {
+	inst := &transmodel.Instance{}
+	if err := i.simpleDecode(e, inst); err != nil {
+		return nil, err
+	}
+
+	return inst, nil
+}
+
+func (i *importer) userDecode(e transmodel.TransEntry) (*transmodel.User, error) {
+	u := &transmodel.User{}
+	if err := i.simpleDecode(e, u); err != nil {
+		return nil, err
+	}
+
+	return u, nil
+}
+
 func (i *importer) simpleDecode(entry transmodel.TransEntry, target interface{}) error {
 	decoder, err := newDecoder(target)
 	if err != nil {

internal/trans/encoders.go

@@ -64,8 +64,8 @@ func (e *exporter) accountEncode(ctx context.Context, f *os.File, a *transmodel.
 // simpleEncode can be used for any type that doesn't have special keys which need handling differently,
 // or for types where special keys have already been handled.
 //
-// The 'type' key on the passed interface should already have been set, since simpleEncode won't know
+// Beware, the 'type' key on the passed interface should already have been set, since simpleEncode won't know
-// what type it is!
+// what type it is! If you try to decode stuff you've encoded with a missing type key, you're going to have a bad time.
 func (e *exporter) simpleEncode(ctx context.Context, f *os.File, i interface{}, id string) error {
 	_, alreadyWritten := e.writtenIDs[id]
 	if alreadyWritten {

internal/trans/export.go

@@ -0,0 +1,223 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/package trans
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, f *os.File) ([]*transmodel.Account, error) {
+	// select using the 'where' we've been provided
+	accounts := []*transmodel.Account{}
+	if err := e.db.GetWhere(ctx, where, &accounts); err != nil {
+		return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err)
+	}
+
+	// write any accounts found to file
+	for _, a := range accounts {
+		if err := e.accountEncode(ctx, f, a); err != nil {
+			return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err)
+		}
+	}
+
+	return accounts, nil
+}
+
+func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.Block, error) {
+	blocksUnique := make(map[string]*transmodel.Block)
+
+	// for each account we want to export both where it's blocking and where it's blocked
+	for _, a := range accounts {
+		// 1. export blocks owned by given account
+		whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}}
+		blocking := []*transmodel.Block{}
+		if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err)
+		}
+		for _, b := range blocking {
+			b.Type = transmodel.TransBlock
+			if err := e.simpleEncode(ctx, f, b, b.ID); err != nil {
+				return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err)
+			}
+			blocksUnique[b.ID] = b
+		}
+
+		// 2. export blocks that target given account
+		whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		blocked := []*transmodel.Block{}
+		if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err)
+		}
+		for _, b := range blocked {
+			b.Type = transmodel.TransBlock
+			if err := e.simpleEncode(ctx, f, b, b.ID); err != nil {
+				return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err)
+			}
+			blocksUnique[b.ID] = b
+		}
+	}
+
+	// now return all the blocks we found
+	blocks := []*transmodel.Block{}
+	for _, b := range blocksUnique {
+		blocks = append(blocks, b)
+	}
+
+	return blocks, nil
+}
+
+func (e *exporter) exportDomainBlocks(ctx context.Context, f *os.File) ([]*transmodel.DomainBlock, error) {
+	domainBlocks := []*transmodel.DomainBlock{}
+
+	if err := e.db.GetAll(ctx, &domainBlocks); err != nil {
+		return nil, fmt.Errorf("exportBlocks: error selecting domain blocks: %s", err)
+	}
+
+	for _, b := range domainBlocks {
+		b.Type = transmodel.TransBlock
+		if err := e.simpleEncode(ctx, f, b, b.ID); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error encoding domain block: %s", err)
+		}
+	}
+
+	return domainBlocks, nil
+}
+
+func (e *exporter) exportFollows(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.Follow, error) {
+	followsUnique := make(map[string]*transmodel.Follow)
+
+	// for each account we want to export both where it's following and where it's followed
+	for _, a := range accounts {
+		// 1. export follows owned by given account
+		whereFollowing := []db.Where{{Key: "account_id", Value: a.ID}}
+		following := []*transmodel.Follow{}
+		if err := e.db.GetWhere(ctx, whereFollowing, &following); err != nil {
+			return nil, fmt.Errorf("exportFollows: error selecting follows owned by account %s: %s", a.ID, err)
+		}
+		for _, follow := range following {
+			follow.Type = transmodel.TransFollow
+			if err := e.simpleEncode(ctx, f, follow, follow.ID); err != nil {
+				return nil, fmt.Errorf("exportFollows: error encoding follow owned by account %s: %s", a.ID, err)
+			}
+			followsUnique[follow.ID] = follow
+		}
+
+		// 2. export follows that target given account
+		whereFollowed := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		followed := []*transmodel.Follow{}
+		if err := e.db.GetWhere(ctx, whereFollowed, &followed); err != nil {
+			return nil, fmt.Errorf("exportFollows: error selecting follows targeting account %s: %s", a.ID, err)
+		}
+		for _, follow := range followed {
+			follow.Type = transmodel.TransFollow
+			if err := e.simpleEncode(ctx, f, follow, follow.ID); err != nil {
+				return nil, fmt.Errorf("exportFollows: error encoding follow targeting account %s: %s", a.ID, err)
+			}
+			followsUnique[follow.ID] = follow
+		}
+	}
+
+	// now return all the follows we found
+	follows := []*transmodel.Follow{}
+	for _, follow := range followsUnique {
+		follows = append(follows, follow)
+	}
+
+	return follows, nil
+}
+
+func (e *exporter) exportFollowRequests(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.FollowRequest, error) {
+	frsUnique := make(map[string]*transmodel.FollowRequest)
+
+	// for each account we want to export both where it's following and where it's followed
+	for _, a := range accounts {
+		// 1. export follow requests owned by given account
+		whereRequesting := []db.Where{{Key: "account_id", Value: a.ID}}
+		requesting := []*transmodel.FollowRequest{}
+		if err := e.db.GetWhere(ctx, whereRequesting, &requesting); err != nil {
+			return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests owned by account %s: %s", a.ID, err)
+		}
+		for _, fr := range requesting {
+			fr.Type = transmodel.TransFollowRequest
+			if err := e.simpleEncode(ctx, f, fr, fr.ID); err != nil {
+				return nil, fmt.Errorf("exportFollowRequests: error encoding follow request owned by account %s: %s", a.ID, err)
+			}
+			frsUnique[fr.ID] = fr
+		}
+
+		// 2. export follow requests that target given account
+		whereRequested := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		requested := []*transmodel.FollowRequest{}
+		if err := e.db.GetWhere(ctx, whereRequested, &requested); err != nil {
+			return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests targeting account %s: %s", a.ID, err)
+		}
+		for _, fr := range requested {
+			fr.Type = transmodel.TransFollowRequest
+			if err := e.simpleEncode(ctx, f, fr, fr.ID); err != nil {
+				return nil, fmt.Errorf("exportFollowRequests: error encoding follow request targeting account %s: %s", a.ID, err)
+			}
+			frsUnique[fr.ID] = fr
+		}
+	}
+
+	// now return all the followRequests we found
+	followRequests := []*transmodel.FollowRequest{}
+	for _, fr := range frsUnique {
+		followRequests = append(followRequests, fr)
+	}
+
+	return followRequests, nil
+}
+
+func (e *exporter) exportInstances(ctx context.Context, f *os.File) ([]*transmodel.Instance, error) {
+	instances := []*transmodel.Instance{}
+
+	if err := e.db.GetAll(ctx, &instances); err != nil {
+		return nil, fmt.Errorf("exportInstances: error selecting instance: %s", err)
+	}
+
+	for _, u := range instances {
+		u.Type = transmodel.TransInstance
+		if err := e.simpleEncode(ctx, f, u, u.ID); err != nil {
+			return nil, fmt.Errorf("exportInstances: error encoding instance: %s", err)
+		}
+	}
+
+	return instances, nil
+}
+
+func (e *exporter) exportUsers(ctx context.Context, f *os.File) ([]*transmodel.User, error) {
+	users := []*transmodel.User{}
+
+	if err := e.db.GetAll(ctx, &users); err != nil {
+		return nil, fmt.Errorf("exportUsers: error selecting users: %s", err)
+	}
+
+	for _, u := range users {
+		u.Type = transmodel.TransUser
+		if err := e.simpleEncode(ctx, f, u, u.ID); err != nil {
+			return nil, fmt.Errorf("exportUsers: error encoding user: %s", err)
+		}
+	}
+
+	return users, nil
+}

internal/trans/exportminimal.go

@@ -20,17 +20,21 @@ package trans
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 
 	"github.com/superseriousbusiness/gotosocial/internal/db"
-	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
 )
 
 func (e *exporter) ExportMinimal(ctx context.Context, path string) error {
+	if path == "" {
+		return errors.New("ExportMinimal: path empty")
+	}
+
 	f, err := os.Create(path)
 	if err != nil {
-		return err
+		return fmt.Errorf("ExportMinimal: couldn't export to %s: %s", path, err)
 	}
 
 	// export all local accounts we have in the database
@@ -39,13 +43,14 @@ func (e *exporter) ExportMinimal(ctx context.Context, path string) error {
 		return fmt.Errorf("ExportMinimal: error exporting accounts: %s", err)
 	}
 
-	// export all blocks that relate to those accounts
+	// export all blocks that relate to local accounts
 	blocks, err := e.exportBlocks(ctx, localAccounts, f)
 	if err != nil {
 		return fmt.Errorf("ExportMinimal: error exporting blocks: %s", err)
 	}
 
-	// for each block, make sure we've written out the account owning it, or targeted by it
+	// for each block, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
 	for _, b := range blocks {
 		_, alreadyWritten := e.writtenIDs[b.AccountID]
 		if !alreadyWritten {
@@ -64,65 +69,72 @@ func (e *exporter) ExportMinimal(ctx context.Context, path string) error {
 		}
 	}
 
+	// export all follows that relate to local accounts
+	follows, err := e.exportFollows(ctx, localAccounts, f)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting follows: %s", err)
+	}
+
+	// for each follow, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
+	for _, follow := range follows {
+		_, alreadyWritten := e.writtenIDs[follow.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.AccountID}}, f)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[follow.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.TargetAccountID}}, f)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow target account: %s", err)
+			}
+		}
+	}
+
+	// export all follow requests that relate to local accounts
+	frs, err := e.exportFollowRequests(ctx, localAccounts, f)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting follow requests: %s", err)
+	}
+
+	// for each follow request, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
+	for _, fr := range frs {
+		_, alreadyWritten := e.writtenIDs[fr.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.AccountID}}, f)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow request owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[fr.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.TargetAccountID}}, f)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow request target account: %s", err)
+			}
+		}
+	}
+
+	// export all domain blocks
+	if _, err := e.exportDomainBlocks(ctx, f); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting domain blocks: %s", err)
+	}
+
+	// export all users
+	if _, err := e.exportUsers(ctx, f); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting users: %s", err)
+	}
+
+	// export all instances
+	if _, err := e.exportInstances(ctx, f); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting instances: %s", err)
+	}
+
 	return neatClose(f)
 }
-
-func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, f *os.File) ([]*transmodel.Account, error) {
-	// select using the 'where' we've been provided
-	accounts := []*transmodel.Account{}
-	if err := e.db.GetWhere(ctx, where, &accounts); err != nil {
-		return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err)
-	}
-
-	// write any accounts found to file
-	for _, a := range accounts {
-		if err := e.accountEncode(ctx, f, a); err != nil {
-			return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err)
-		}
-	}
-
-	return accounts, nil
-}
-
-func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.Block, error) {
-	blocksUnique := make(map[string]*transmodel.Block)
-
-	// for each account we want to export both where it's blocking and where it's blocked
-	for _, a := range accounts {
-		// 1. export blocks owned by given account
-		whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}}
-		blocking := []*transmodel.Block{}
-		if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil {
-			return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err)
-		}
-		for _, b := range blocking {
-			b.Type = transmodel.TransBlock
-			if err := e.simpleEncode(ctx, f, b, b.ID); err != nil {
-				return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err)
-			}
-			blocksUnique[b.ID] = b
-		}
-
-		// 2. export blocks that target given account
-		whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}}
-		blocked := []*transmodel.Block{}
-		if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil {
-			return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err)
-		}
-		for _, b := range blocked {
-			b.Type = transmodel.TransBlock
-			if err := e.simpleEncode(ctx, f, b, b.ID); err != nil {
-				return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err)
-			}
-			blocksUnique[b.ID] = b
-		}
-	}
-
-	// now return all the blocks we found
-	blocks := []*transmodel.Block{}
-	for _, b := range blocksUnique {
-		blocks = append(blocks, b)
-	}
-
-	return blocks, nil
-}

internal/trans/import.go

@@ -0,0 +1,142 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func (i *importer) Import(ctx context.Context, path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("ImportMinimal: error opening file %s: %s", path, err)
+	}
+
+	decoder := json.NewDecoder(f)
+	decoder.UseNumber()
+
+	for {
+		entry := transmodel.TransEntry{}
+		err := decoder.Decode(&entry)
+		if err != nil {
+			if err == io.EOF {
+				i.log.Infof("ImportMinimal: reached end of file")
+				return neatClose(f)
+			}
+			return fmt.Errorf("ImportMinimal: error decoding in readLoop: %s", err)
+		}
+		if err := i.inputEntry(ctx, entry); err != nil {
+			return fmt.Errorf("ImportMinimal: error inputting entry: %s", err)
+		}
+	}
+}
+
+func (i *importer) inputEntry(ctx context.Context, entry transmodel.TransEntry) error {
+	t, ok := entry[transmodel.TypeKey].(string)
+	if !ok {
+		return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json")
+	}
+
+	switch transmodel.TransType(t) {
+	case transmodel.TransAccount:
+		account, err := i.accountDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into account: %s", err)
+		}
+		if err := i.putInDB(ctx, account); err != nil {
+			return fmt.Errorf("inputEntry: error adding account to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added account with id %s", account.ID)
+		return nil
+	case transmodel.TransBlock:
+		block, err := i.blockDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into block: %s", err)
+		}
+		if err := i.putInDB(ctx, block); err != nil {
+			return fmt.Errorf("inputEntry: error adding block to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added block with id %s", block.ID)
+		return nil
+	case transmodel.TransDomainBlock:
+		block, err := i.domainBlockDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into block: %s", err)
+		}
+		if err := i.putInDB(ctx, block); err != nil {
+			return fmt.Errorf("inputEntry: error adding block to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added block with id %s", block.ID)
+		return nil
+	case transmodel.TransFollow:
+		follow, err := i.followDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into follow: %s", err)
+		}
+		if err := i.putInDB(ctx, follow); err != nil {
+			return fmt.Errorf("inputEntry: error adding follow to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added follow with id %s", follow.ID)
+		return nil
+	case transmodel.TransFollowRequest:
+		fr, err := i.followRequestDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into follow request: %s", err)
+		}
+		if err := i.putInDB(ctx, fr); err != nil {
+			return fmt.Errorf("inputEntry: error adding follow request to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added follow request with id %s", fr.ID)
+		return nil
+	case transmodel.TransInstance:
+		inst, err := i.instanceDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into instance: %s", err)
+		}
+		if err := i.putInDB(ctx, inst); err != nil {
+			return fmt.Errorf("inputEntry: error adding instance to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added instance with id %s", inst.ID)
+		return nil
+	case transmodel.TransUser:
+		user, err := i.userDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into user: %s", err)
+		}
+		if err := i.putInDB(ctx, user); err != nil {
+			return fmt.Errorf("inputEntry: error adding user to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added user with id %s", user.ID)
+		return nil
+	}
+
+	i.log.Errorf("inputEntry: didn't recognize transtype '%s', skipping it", t)
+	return nil
+}
+
+func (i *importer) putInDB(ctx context.Context, entry interface{}) error {
+	return i.db.Put(ctx, entry)
+}

internal/trans/import_test.go

@@ -26,7 +26,6 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/suite"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/trans"
 	"github.com/superseriousbusiness/gotosocial/testrig"
@@ -51,7 +50,7 @@ func (suite *ImportMinimalTestSuite) TestImportMinimalOK() {
 	b, err := os.ReadFile(tempFilePath)
 	suite.NoError(err)
 	suite.NotEmpty(b)
-	suite.T().Log(string(b))
+	fmt.Println(string(b))
 
 	// create a new database with just the tables created, no entries
 	testrig.StandardDBTeardown(suite.db)
@@ -59,14 +58,20 @@ func (suite *ImportMinimalTestSuite) TestImportMinimalOK() {
 	testrig.CreateTestTables(newDB)
 
 	importer := trans.NewImporter(newDB, suite.log)
-	err = importer.ImportMinimal(ctx, tempFilePath)
+	err = importer.Import(ctx, tempFilePath)
 	suite.NoError(err)
 
-	// we should now have some accounts in the database
+	// we should have some accounts in the database
 	accounts := []*gtsmodel.Account{}
-	err = newDB.GetWhere(ctx, []db.Where{{Key: "domain", Value: nil}}, &accounts)
+	err = newDB.GetAll(ctx, &accounts)
 	suite.NoError(err)
 	suite.NotEmpty(accounts)
+
+	// we should have some blocks in the database
+	blocks := []*gtsmodel.Block{}
+	err = newDB.GetAll(ctx, &blocks)
+	suite.NoError(err)
+	suite.NotEmpty(blocks)
 }
 
 func TestImportMinimalTestSuite(t *testing.T) {

internal/trans/importer.go

@@ -26,7 +26,7 @@ import (
 )
 
 type Importer interface {
-	ImportMinimal(ctx context.Context, path string) error
+	Import(ctx context.Context, path string) error
 }
 
 type importer struct {

internal/trans/importminimal.go

@@ -1,54 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package trans
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os"
-
-	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
-)
-
-func (i *importer) ImportMinimal(ctx context.Context, path string) error {
-	f, err := os.Open(path)
-	if err != nil {
-		return fmt.Errorf("ImportMinimal: error opening file %s: %s", path, err)
-	}
-
-	decoder := json.NewDecoder(f)
-	decoder.UseNumber()
-
-	for {
-		entry := transmodel.TransEntry{}
-		err := decoder.Decode(&entry)
-		if err != nil {
-			if err == io.EOF {
-				i.log.Infof("ImportMinimal: reached end of file")
-				return neatClose(f)
-			}
-			return fmt.Errorf("ImportMinimal: error decoding in readLoop: %s", err)
-		}
-		if err := i.inputEntry(ctx, entry); err != nil {
-			return fmt.Errorf("ImportMinimal: error inputting entry: %s", err)
-		}
-	}
-}

internal/trans/input.go

@@ -1,53 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package trans
-
-import (
-	"context"
-	"errors"
-	"fmt"
-
-	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
-)
-
-func (i *importer) inputEntry(ctx context.Context, entry transmodel.TransEntry) error {
-	t, ok := entry[transmodel.TypeKey].(string)
-	if !ok {
-		return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json")
-	}
-
-	switch transmodel.TransType(t) {
-	case transmodel.TransAccount:
-		account, err := i.accountDecode(entry)
-		if err != nil {
-			return fmt.Errorf("inputEntry: error decoding entry into account: %s", err)
-		}
-		if err := i.putInDB(ctx, account); err != nil {
-			return fmt.Errorf("inputEntry: error adding account to database: %s", err)
-		}
-		i.log.Infof("inputEntry: added account with id %s", account.ID)
-		return nil
-	}
-
-	return fmt.Errorf("inputEntry: didn't recognize transtype %s", t)
-}
-
-func (i *importer) putInDB(ctx context.Context, entry interface{}) error {
-	return i.db.Put(ctx, entry)
-}

internal/trans/model/account.go

@@ -28,7 +28,6 @@ type Account struct {
 	Type                  TransType       `json:"type" bun:"-"`
 	ID                    string          `json:"id"`
 	CreatedAt             *time.Time      `json:"createdAt"`
-	UpdatedAt             *time.Time      `json:"updatedAt"`
 	Username              string          `json:"username"`
 	Domain                string          `json:"domain,omitempty" bun:",nullzero"`
 	Locked                bool            `json:"locked"`

internal/trans/model/block.go

@@ -23,8 +23,7 @@ import "time"
 type Block struct {
 	Type            TransType `json:"type" bun:"-"`
 	ID              string    `json:"id"`
-	CreatedAt       time.Time `json:"createdAt"`
+	CreatedAt       *time.Time `json:"createdAt"`
-	UpdatedAt       time.Time `json:"updatedAt"`
 	URI             string    `json:"uri"`
 	AccountID       string    `json:"accountId"`
 	TargetAccountID string    `json:"targetAccountId"`

internal/trans/model/domainblock.go

@@ -1,3 +1,32 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package trans
 
+import "time"
 
+type DomainBlock struct {
+	Type               TransType  `json:"type" bun:"-"`
+	ID                 string     `json:"id"`
+	CreatedAt          *time.Time `json:"createdAt"`
+	CreatedByAccountID string     `json:"createdByAccountID"`
+	PrivateComment     string     `json:"privateComment,omitempty"`
+	PublicComment      string     `json:"publicComment,omitempty"`
+	Obfuscate          bool       `json:"obfuscate"`
+	SubscriptionID     string     `json:"subscriptionID,omitempty"`
+}

internal/trans/model/follow.go

@@ -23,8 +23,7 @@ import "time"
 type Follow struct {
 	Type            TransType `json:"type" bun:"-"`
 	ID              string    `json:"id"`
-	CreatedAt       time.Time `json:"createdAt"`
+	CreatedAt       *time.Time `json:"createdAt"`
-	UpdatedAt       time.Time `json:"updatedAt"`
 	URI             string    `json:"uri"`
 	AccountID       string    `json:"accountId"`
 	TargetAccountID string    `json:"targetAccountId"`

internal/trans/model/followrequest.go

@@ -0,0 +1,30 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+type FollowRequest struct {
+	Type            TransType `json:"type" bun:"-"`
+	ID              string    `json:"id"`
+	CreatedAt       *time.Time `json:"createdAt"`
+	URI             string    `json:"uri"`
+	AccountID       string    `json:"accountId"`
+	TargetAccountID string    `json:"targetAccountId"`
+}

internal/trans/model/instance.go

@@ -0,0 +1,42 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"time"
+)
+
+type Instance struct {
+	Type                   TransType  `json:"type" bun:"-"`
+	ID                     string     `json:"id"`
+	CreatedAt              *time.Time `json:"createdAt"`
+	Domain                 string     `json:"domain"`
+	Title                  string     `json:"title,omitempty" bun:",nullzero"`
+	URI                    string     `json:"uri"`
+	SuspendedAt            *time.Time  `json:"suspendedAt,omitempty" bun:",nullzero"`
+	DomainBlockID          string     `json:"domainBlockID,omitempty" bun:",nullzero"`
+	ShortDescription       string     `json:"shortDescription,omitempty" bun:",nullzero"`
+	Description            string     `json:"description,omitempty" bun:",nullzero"`
+	Terms                  string     `json:"terms,omitempty" bun:",nullzero"`
+	ContactEmail           string     `json:"contactEmail,omitempty" bun:",nullzero"`
+	ContactAccountUsername string     `json:"contactAccountUsername,omitempty" bun:",nullzero"`
+	ContactAccountID       string     `json:"contactAccountID,omitempty" bun:",nullzero"`
+	Reputation             int64      `json:"reputation"`
+	Version                string     `json:"version,omitempty" bun:",nullzero"`
+}

internal/trans/model/type.go

@@ -25,9 +25,14 @@ type TransType string
 
 // Type of the trans entry. Describes how it should be read from file.
 const (
-	TransAccount TransType = "account"
+	TransAccount          TransType = "account"
-	TransBlock   TransType = "block"
+	TransBlock            TransType = "block"
-	TransFollow  TransType = "follow"
+	TransDomainBlock      TransType = "domainBlock"
+	TransEmailDomainBlock TransType = "emailDomainBlock"
+	TransFollow           TransType = "follow"
+	TransFollowRequest    TransType = "followRequest"
+	TransInstance         TransType = "instance"
+	TransUser             TransType = "user"
 )
 
 type TransEntry map[string]interface{}

internal/trans/model/user.go

@@ -0,0 +1,49 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"time"
+)
+
+type User struct {
+	Type                TransType  `json:"type" bun:"-"`
+	ID                  string     `json:"id"`
+	CreatedAt           *time.Time `json:"createdAt"`
+	Email               string     `json:"email,omitempty" bun:",nullzero"`
+	AccountID           string     `json:"accountID"`
+	EncryptedPassword   string     `json:"encryptedPassword"`
+	CurrentSignInAt     *time.Time  `json:"currentSignInAt,omitempty" bun:",nullzero"`
+	LastSignInAt        *time.Time  `json:"lastSignInAt,omitempty" bun:",nullzero"`
+	InviteID            string     `json:"inviteID,omitempty" bun:",nullzero"`
+	ChosenLanguages     []string   `json:"chosenLanguages,omitempty" bun:",nullzero"`
+	FilteredLanguages   []string   `json:"filteredLanguage,omitempty" bun:",nullzero"`
+	Locale              string     `json:"locale" bun:",nullzero"`
+	LastEmailedAt       time.Time  `json:"lastEmailedAt,omitempty" bun:",nullzero"`
+	ConfirmationToken   string     `json:"confirmationToken,omitempty" bun:",nullzero"`
+	ConfirmationSentAt  *time.Time  `json:"confirmationTokenSentAt,omitempty" bun:",nullzero"`
+	ConfirmedAt         *time.Time  `json:"confirmedAt,omitempty" bun:",nullzero"`
+	UnconfirmedEmail    string     `json:"unconfirmedEmail,omitempty" bun:",nullzero"`
+	Moderator           bool       `json:"moderator"`
+	Admin               bool       `json:"admin"`
+	Disabled            bool       `json:"disabled"`
+	Approved            bool       `json:"approved"`
+	ResetPasswordToken  string     `json:"resetPasswordToken,omitempty" bun:",nullzero"`
+	ResetPasswordSentAt *time.Time  `json:"resetPasswordSentAt,omitempty" bun:",nullzero"`
+}