stuff is WORKING

internal/trans/decoders.go

@@ -19,53 +19,75 @@
 package trans
 
 import (
-	"crypto/rsa"
+	"crypto/x509"
-	"encoding"
+	"encoding/pem"
+	"errors"
 	"fmt"
-	"reflect"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
-	"github.com/sirupsen/logrus"
 	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
 )
 
+func newDecoder(target interface{}) (*mapstructure.Decoder, error) {
+	decoderConfig := &mapstructure.DecoderConfig{
+		DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), // this is needed to decode time.Time entries serialized as string
+		Result:     target,
+	}
+	return mapstructure.NewDecoder(decoderConfig)
+}
+
 func (i *importer) accountDecode(e transmodel.TransEntry) (*transmodel.Account, error) {
 	a := &transmodel.Account{}
-
+	if err := i.simpleDecode(e, a); err != nil {
-	decoderConfig := &mapstructure.DecoderConfig{
+		return nil, err
-		DecodeHook: mapstructure.ComposeDecodeHookFunc(
-			mapstructure.StringToTimeHookFunc(time.RFC3339),
-			keyHookFunc(i.log),
-		),
-		Result: a,
 	}
-	decoder, err := mapstructure.NewDecoder(decoderConfig)
+
+	// extract public key
+	publicKeyBlock, _ := pem.Decode([]byte(a.PublicKeyString))
+	if publicKeyBlock == nil {
+		return nil, errors.New("accountDecode: error decoding account public key")
+	}
+	publicKey, err := x509.ParsePKCS1PublicKey(publicKeyBlock.Bytes)
 	if err != nil {
-		return nil, fmt.Errorf("accountDecode: error creating decoder: %s", err)
+		return nil, fmt.Errorf("accountDecode: error parsing account public key: %s", err)
 	}
+	a.PublicKey = publicKey
 
-	if err := decoder.Decode(&e); err != nil {
+	if a.Domain == "" {
-		return nil, fmt.Errorf("accountDecode: error decoding account: %s", err)
+		// extract private key (local account)
+		privateKeyBlock, _ := pem.Decode([]byte(a.PrivateKeyString))
+		if privateKeyBlock == nil {
+			return nil, errors.New("accountDecode: error decoding account private key")
+		}
+		privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyBlock.Bytes)
+		if err != nil {
+			return nil, fmt.Errorf("accountDecode: error parsing account private key: %s", err)
+		}
+		a.PrivateKey = privateKey
 	}
 
 	return a, nil
 }
 
-func keyHookFunc(log *logrus.Logger) mapstructure.DecodeHookFunc {
+func (i *importer) blockDecode(e transmodel.TransEntry) (*transmodel.Block, error) {
-	return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
+	b := &transmodel.Block{}
-		if t != reflect.TypeOf(rsa.PrivateKey{}) {
+	if err := i.simpleDecode(e, b); err != nil {
-			return data, nil
+		return nil, err
-		}
-
-		result := reflect.New(t).Interface()
-		unmarshaller, ok := result.(encoding.BinaryUnmarshaler)
-		if !ok {
-			return data, nil
-		}
-		if err := unmarshaller.UnmarshalBinary([]byte(data.(string))); err != nil {
-			return nil, err
-		}
-		return result, nil
 	}
+
+	return b, nil
+}
+
+func (i *importer) simpleDecode(entry transmodel.TransEntry, target interface{}) error {
+	decoder, err := newDecoder(target)
+	if err != nil {
+		return fmt.Errorf("simpleDecode: error creating decoder: %s", err)
+	}
+
+	if err := decoder.Decode(&entry); err != nil {
+		return fmt.Errorf("simpleDecode: error decoding: %s", err)
+	}
+
+	return nil
 }

internal/trans/encoders.go

@@ -0,0 +1,83 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"os"
+
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+// accountEncode handles special fields like private + public keys on accounts
+func (e *exporter) accountEncode(ctx context.Context, f *os.File, a *transmodel.Account) error {
+	a.Type = transmodel.TransAccount
+
+	// marshal public key
+	encodedPublicKey := x509.MarshalPKCS1PublicKey(a.PublicKey)
+	if encodedPublicKey == nil {
+		return errors.New("could not MarshalPKCS1PublicKey")
+	}
+	publicKeyBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PUBLIC KEY",
+		Bytes: encodedPublicKey,
+	})
+	a.PublicKeyString = string(publicKeyBytes)
+
+	if a.Domain == "" {
+		// marshal private key for local account
+		encodedPrivateKey := x509.MarshalPKCS1PrivateKey(a.PrivateKey)
+		if encodedPrivateKey == nil {
+			return errors.New("could not MarshalPKCS1PrivateKey")
+		}
+		privateKeyBytes := pem.EncodeToMemory(&pem.Block{
+			Type:  "RSA PRIVATE KEY",
+			Bytes: encodedPrivateKey,
+		})
+		a.PrivateKeyString = string(privateKeyBytes)
+	}
+
+	return e.simpleEncode(ctx, f, a, a.ID)
+}
+
+// simpleEncode can be used for any type that doesn't have special keys which need handling differently,
+// or for types where special keys have already been handled.
+//
+// The 'type' key on the passed interface should already have been set, since simpleEncode won't know
+// what type it is!
+func (e *exporter) simpleEncode(ctx context.Context, f *os.File, i interface{}, id string) error {
+	_, alreadyWritten := e.writtenIDs[id]
+	if alreadyWritten {
+		// this exporter has already exported an entry with this ID, no need to do it twice
+		return nil
+	}
+
+	err := json.NewEncoder(f).Encode(i)
+	if err != nil {
+		return fmt.Errorf("simpleEncode: error encoding entry with id %s: %s", id, err)
+	}
+
+	e.writtenIDs[id] = true
+	return nil
+}

internal/trans/exporter.go

@@ -30,13 +30,15 @@ type Exporter interface {
 }
 
 type exporter struct {
-	db  db.DB
+	db         db.DB
-	log *logrus.Logger
+	log        *logrus.Logger
+	writtenIDs map[string]bool
 }
 
 func NewExporter(db db.DB, log *logrus.Logger) Exporter {
 	return &exporter{
-		db: db,
+		db:         db,
-      log: log,
+		log:        log,
+		writtenIDs: make(map[string]bool),
 	}
 }

internal/trans/exportminimal.go

@@ -20,7 +20,6 @@ package trans
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
 
@@ -34,28 +33,96 @@ func (e *exporter) ExportMinimal(ctx context.Context, path string) error {
 		return err
 	}
 
-	encoder := json.NewEncoder(f)
+	// export all local accounts we have in the database
-
+	localAccounts, err := e.exportAccounts(ctx, []db.Where{{Key: "domain", Value: nil}}, f)
-	accounts := []*transmodel.Account{}
+	if err != nil {
-	if err := e.db.GetWhere(ctx, []db.Where{{Key: "domain", Value: nil}}, &accounts); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting accounts: %s", err)
-		return fmt.Errorf("ExportMinimal: error selecting accounts: %s", err)
 	}
 
-	for _, a := range accounts {
+	// export all blocks that relate to those accounts
-		a.Type = transmodel.TransAccount
+	blocks, err := e.exportBlocks(ctx, localAccounts, f)
-		if err := encoder.Encode(a); err != nil {
+	if err != nil {
-			return fmt.Errorf("ExportMinimal: error encoding account: %s", err)
+		return fmt.Errorf("ExportMinimal: error exporting blocks: %s", err)
+	}
+
+	// for each block, make sure we've written out the account owning it, or targeted by it
+	for _, b := range blocks {
+		_, alreadyWritten := e.writtenIDs[b.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.AccountID}}, f)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting block owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[b.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.TargetAccountID}}, f)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting block target account: %s", err)
+			}
 		}
-		e.log.Infof("ExportMinimal: exported account %s to %s", a.ID, path)
 	}
 
 	return neatClose(f)
 }
 
-func neatClose(f *os.File) error {
+func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, f *os.File) ([]*transmodel.Account, error) {
-	if err := f.Close(); err != nil {
+	// select using the 'where' we've been provided
-		return fmt.Errorf("error closing file: %s", err)
+	accounts := []*transmodel.Account{}
+	if err := e.db.GetWhere(ctx, where, &accounts); err != nil {
+		return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err)
 	}
 
-	return nil
+	// write any accounts found to file
+	for _, a := range accounts {
+		if err := e.accountEncode(ctx, f, a); err != nil {
+			return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err)
+		}
+	}
+
+	return accounts, nil
+}
+
+func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, f *os.File) ([]*transmodel.Block, error) {
+	blocksUnique := make(map[string]*transmodel.Block)
+
+	// for each account we want to export both where it's blocking and where it's blocked
+	for _, a := range accounts {
+		// 1. export blocks owned by given account
+		whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}}
+		blocking := []*transmodel.Block{}
+		if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err)
+		}
+		for _, b := range blocking {
+			b.Type = transmodel.TransBlock
+			if err := e.simpleEncode(ctx, f, b, b.ID); err != nil {
+				return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err)
+			}
+			blocksUnique[b.ID] = b
+		}
+
+		// 2. export blocks that target given account
+		whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		blocked := []*transmodel.Block{}
+		if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err)
+		}
+		for _, b := range blocked {
+			b.Type = transmodel.TransBlock
+			if err := e.simpleEncode(ctx, f, b, b.ID); err != nil {
+				return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err)
+			}
+			blocksUnique[b.ID] = b
+		}
+	}
+
+	// now return all the blocks we found
+	blocks := []*transmodel.Block{}
+	for _, b := range blocksUnique {
+		blocks = append(blocks, b)
+	}
+
+	return blocks, nil
 }

internal/trans/exportminimal_test.go

@@ -46,7 +46,7 @@ func (suite *ExportMinimalTestSuite) TestExportMinimalOK() {
 	b, err := os.ReadFile(tempFilePath)
 	suite.NoError(err)
 	suite.NotEmpty(b)
-	suite.T().Log(string(b))
+	fmt.Println(string(b))
 }
 
 func TestExportMinimalTestSuite(t *testing.T) {

internal/trans/importminimal_test.go

@@ -40,7 +40,7 @@ func (suite *ImportMinimalTestSuite) TestImportMinimalOK() {
 	ctx := context.Background()
 
 	// use a temporary file path
-	tempFilePath := fmt.Sprintf("%s/%s", os.TempDir(), uuid.NewString())
+	tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString())
 
 	// export to the tempFilePath
 	exporter := trans.NewExporter(suite.db, suite.log)
@@ -53,18 +53,18 @@ func (suite *ImportMinimalTestSuite) TestImportMinimalOK() {
 	suite.NotEmpty(b)
 	suite.T().Log(string(b))
 
-	// now that the file is stored, tear down the database...
+	// create a new database with just the tables created, no entries
 	testrig.StandardDBTeardown(suite.db)
-	// and create just the tables -- no entries!
+	newDB := testrig.NewTestDB()
-	testrig.CreateTestTables(suite.db)
+	testrig.CreateTestTables(newDB)
 
-	importer := trans.NewImporter(suite.db, suite.log)
+	importer := trans.NewImporter(newDB, suite.log)
 	err = importer.ImportMinimal(ctx, tempFilePath)
 	suite.NoError(err)
 
 	// we should now have some accounts in the database
 	accounts := []*gtsmodel.Account{}
-	err = suite.db.GetWhere(ctx, []db.Where{{Key: "domain", Value: nil}}, &accounts)
+	err = newDB.GetWhere(ctx, []db.Where{{Key: "domain", Value: nil}}, &accounts)
 	suite.NoError(err)
 	suite.NotEmpty(accounts)
 }

internal/trans/model/account.go

@@ -30,7 +30,7 @@ type Account struct {
 	CreatedAt             *time.Time      `json:"createdAt"`
 	UpdatedAt             *time.Time      `json:"updatedAt"`
 	Username              string          `json:"username"`
-	Domain                string          `json:"domain,omitempty"`
+	Domain                string          `json:"domain,omitempty" bun:",nullzero"`
 	Locked                bool            `json:"locked"`
 	Language              string          `json:"language,omitempty"`
 	URI                   string          `json:"uri"`
@@ -41,9 +41,11 @@ type Account struct {
 	FollowersURI          string          `json:"followersUri"`
 	FeaturedCollectionURI string          `json:"featuredCollectionUri"`
 	ActorType             string          `json:"actorType"`
-	PrivateKey            *rsa.PrivateKey `json:"privateKey,omitempty"`
+	PrivateKey            *rsa.PrivateKey `json:"-" mapstructure:"-"`
-	PublicKey             *rsa.PublicKey  `json:"publicKey"`
+	PrivateKeyString      string          `json:"privateKey,omitempty" bun:"-" mapstructure:"privateKey"`
+	PublicKey             *rsa.PublicKey  `json:"-" mapstructure:"-"`
+	PublicKeyString       string          `json:"publicKey,omitempty" bun:"-" mapstructure:"publicKey"`
 	PublicKeyURI          string          `json:"publicKeyUri"`
 	SuspendedAt           *time.Time      `json:"suspendedAt,omitempty"`
-	SuspensionOrigin      string          `json:"suspensionOrigin,omitempty"`
+	SuspensionOrigin      string          `json:"suspensionOrigin,omitempty" bun:",nullzero"`
 }

internal/trans/model/account_test.go

@@ -1,57 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package trans_test
-
-import (
-	"context"
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/suite"
-	trans "github.com/superseriousbusiness/gotosocial/internal/trans/model"
-)
-
-type AccountTestSuite struct {
-	ModelTestSuite
-}
-
-func (suite *AccountTestSuite) TestAccountsIdempotent() {
-	// we should be able to get all accounts with the simple trans.Account struct
-	accounts := []*trans.Account{}
-	err := suite.db.GetAll(context.Background(), &accounts)
-	suite.NoError(err)
-	suite.NotEmpty(accounts)
-
-	// we should be able to marshal the accounts to json with no problems
-	b, err := json.Marshal(&accounts)
-	suite.NoError(err)
-	suite.NotNil(b)
-	suite.T().Log(string(b))
-
-	// the json should be idempotent
-	mAccounts := []*trans.Account{}
-	err = json.Unmarshal(b, &mAccounts)
-	suite.NoError(err)
-	suite.NotEmpty(mAccounts)
-	suite.EqualValues(accounts, mAccounts)
-}
-
-func TestAccountTestSuite(t *testing.T) {
-	suite.Run(t, &AccountTestSuite{})
-}

internal/trans/model/block_test.go

@@ -1,57 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package trans_test
-
-import (
-	"context"
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/suite"
-	trans "github.com/superseriousbusiness/gotosocial/internal/trans/model"
-)
-
-type BlockTestSuite struct {
-	ModelTestSuite
-}
-
-func (suite *AccountTestSuite) TestBlocksIdempotent() {
-	// we should be able to get all blocks with the simple trans.Block struct
-	blocks := []*trans.Block{}
-	err := suite.db.GetAll(context.Background(), &blocks)
-	suite.NoError(err)
-	suite.NotEmpty(blocks)
-
-	// we should be able to marshal the blocks to json with no problems
-	b, err := json.Marshal(&blocks)
-	suite.NoError(err)
-	suite.NotNil(b)
-	suite.T().Log(string(b))
-
-	// the json should be idempotent
-	mBlocks := []*trans.Block{}
-	err = json.Unmarshal(b, &mBlocks)
-	suite.NoError(err)
-	suite.NotEmpty(mBlocks)
-	suite.EqualValues(blocks, mBlocks)
-}
-
-func TestBlockTestSuite(t *testing.T) {
-	suite.Run(t, &BlockTestSuite{})
-}

internal/trans/model/follow.go

@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+type Follow struct {
+	Type            TransType `json:"type" bun:"-"`
+	ID              string    `json:"id"`
+	CreatedAt       time.Time `json:"createdAt"`
+	UpdatedAt       time.Time `json:"updatedAt"`
+	URI             string    `json:"uri"`
+	AccountID       string    `json:"accountId"`
+	TargetAccountID string    `json:"targetAccountId"`
+}

internal/trans/model/type.go

@@ -27,6 +27,7 @@ type TransType string
 const (
 	TransAccount TransType = "account"
 	TransBlock   TransType = "block"
+	TransFollow  TransType = "follow"
 )
 
 type TransEntry map[string]interface{}

internal/trans/util.go

@@ -16,24 +16,17 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-package trans_test
+package trans
 
 import (
-	"github.com/stretchr/testify/suite"
+	"fmt"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"os"
-	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
-type ModelTestSuite struct {
+func neatClose(f *os.File) error {
-	suite.Suite
+	if err := f.Close(); err != nil {
-	db db.DB
+		return fmt.Errorf("error closing file: %s", err)
-}
+	}
 
-func (suite *ModelTestSuite) SetupTest() {
+	return nil
-	suite.db = testrig.NewTestDB()
-	testrig.StandardDBSetup(suite.db, nil)
-}
-
-func (suite *ModelTestSuite) TearDownTest() {
-	testrig.StandardDBTeardown(suite.db)
 }