messing about with decoding/encoding

internal/trans/decoders.go

@@ -0,0 +1,66 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"crypto/rsa"
+	"fmt"
+	"net"
+	"reflect"
+	"time"
+
+	"github.com/mitchellh/mapstructure"
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func accountDecode(e transmodel.TransEntry) (*transmodel.Account, error) {
+	a := &transmodel.Account{}
+
+	decoderConfig := &mapstructure.DecoderConfig{
+		DecodeHook: mapstructure.ComposeDecodeHookFunc(
+			mapstructure.StringToTimeHookFunc(time.RFC3339),
+			PrivateKeyHookFunc(),
+		),
+		Result: a,
+	}
+	decoder, err := mapstructure.NewDecoder(decoderConfig)
+	if err != nil {
+		return nil, fmt.Errorf("accountDecode: error creating decoder: %s", err)
+	}
+
+	if err := decoder.Decode(&e); err != nil {
+		return nil, fmt.Errorf("accountDecode: error decoding account: %s", err)
+	}
+
+	return a, nil
+}
+
+var PrivateKeyHookFunc mapstructure.DecodeHookFunc = func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
+	if t != reflect.TypeOf(rsa.PrivateKey{}) {
+		return data, nil
+	}
+
+
+	rsa.
+
+	// Convert it by parsing
+	_, net, err := net.ParseCIDR(data.(string))
+	return net, err
+
+}

internal/trans/exporter.go

@@ -21,6 +21,7 @@ package trans
 import (
 	"context"
 
+	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 )
 
@@ -29,11 +30,13 @@ type Exporter interface {
 }
 
 type exporter struct {
-	db db.DB
+	db  db.DB
+	log *logrus.Logger
 }
 
-func NewExporter(db db.DB) Exporter {
+func NewExporter(db db.DB, log *logrus.Logger) Exporter {
 	return &exporter{
 		db: db,
+      log: log,
 	}
 }

internal/trans/exportminimal.go

@@ -19,7 +19,6 @@
 package trans
 
 import (
-	"bufio"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -35,26 +34,25 @@ func (e *exporter) ExportMinimal(ctx context.Context, path string) error {
 		return err
 	}
 
-	w := bufio.NewWriter(f)
-	encoder := json.NewEncoder(w)
+	encoder := json.NewEncoder(f)
 
 	accounts := []*transmodel.Account{}
 	if err := e.db.GetWhere(ctx, []db.Where{{Key: "domain", Value: nil}}, &accounts); err != nil {
-		return fmt.Errorf("error selecting accounts: %s", err)
+		return fmt.Errorf("ExportMinimal: error selecting accounts: %s", err)
 	}
 
 	for _, a := range accounts {
-		encoder.Encode(a)
+		a.Type = transmodel.TransAccount
+		if err := encoder.Encode(a); err != nil {
+			return fmt.Errorf("ExportMinimal: error encoding account: %s", err)
+		}
+		e.log.Infof("ExportMinimal: exported account %s to %s", a.ID, path)
 	}
 
-	return neatClose(w, f)
+	return neatClose(f)
 }
 
-func neatClose(w *bufio.Writer, f *os.File) error {
-	if err := w.Flush(); err != nil {
-		return fmt.Errorf("error flushing writer: %s", err)
-	}
-
+func neatClose(f *os.File) error {
 	if err := f.Close(); err != nil {
 		return fmt.Errorf("error closing file: %s", err)
 	}

internal/trans/exportminimal_test.go

@@ -38,7 +38,7 @@ func (suite *ExportMinimalTestSuite) TestExportMinimalOK() {
 	tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString())
 
 	// export to the tempFilePath
-	exporter := trans.NewExporter(suite.db)
+	exporter := trans.NewExporter(suite.db, suite.log)
 	err := exporter.ExportMinimal(context.Background(), tempFilePath)
 	suite.NoError(err)
 

internal/trans/importer.go

@@ -0,0 +1,42 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+type Importer interface {
+	ImportMinimal(ctx context.Context, path string) error
+}
+
+type importer struct {
+	db  db.DB
+	log *logrus.Logger
+}
+
+func NewImporter(db db.DB, log *logrus.Logger) Importer {
+	return &importer{
+		db: db,
+      log: log,
+	}
+}

internal/trans/importminimal.go

@@ -0,0 +1,54 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func (i *importer) ImportMinimal(ctx context.Context, path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("ImportMinimal: error opening file %s: %s", path, err)
+	}
+
+	decoder := json.NewDecoder(f)
+	decoder.UseNumber()
+
+	for {
+		entry := transmodel.TransEntry{}
+		err := decoder.Decode(&entry)
+		if err != nil {
+			if err == io.EOF {
+				i.log.Infof("ImportMinimal: reached end of file")
+				return neatClose(f)
+			}
+			return fmt.Errorf("ImportMinimal: error decoding in readLoop: %s", err)
+		}
+		if err := i.inputEntry(ctx, entry); err != nil {
+			return fmt.Errorf("ImportMinimal: error inputting entry: %s", err)
+		}
+	}
+}

internal/trans/importminimal_test.go

@@ -0,0 +1,74 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type ImportMinimalTestSuite struct {
+	TransTestSuite
+}
+
+func (suite *ImportMinimalTestSuite) TestImportMinimalOK() {
+	ctx := context.Background()
+
+	// use a temporary file path
+	tempFilePath := fmt.Sprintf("%s/%s", os.TempDir(), uuid.NewString())
+
+	// export to the tempFilePath
+	exporter := trans.NewExporter(suite.db, suite.log)
+	err := exporter.ExportMinimal(ctx, tempFilePath)
+	suite.NoError(err)
+
+	// we should have some bytes in that file now
+	b, err := os.ReadFile(tempFilePath)
+	suite.NoError(err)
+	suite.NotEmpty(b)
+	suite.T().Log(string(b))
+
+	// now that the file is stored, tear down the database...
+	testrig.StandardDBTeardown(suite.db)
+	// and create just the tables -- no entries!
+	testrig.CreateTestTables(suite.db)
+
+	importer := trans.NewImporter(suite.db, suite.log)
+	err = importer.ImportMinimal(ctx, tempFilePath)
+	suite.NoError(err)
+
+	// we should now have some accounts in the database
+	accounts := []*gtsmodel.Account{}
+	err = suite.db.GetWhere(ctx, []db.Where{{Key: "domain", Value: nil}}, &accounts)
+	suite.NoError(err)
+	suite.NotEmpty(accounts)
+}
+
+func TestImportMinimalTestSuite(t *testing.T) {
+	suite.Run(t, &ImportMinimalTestSuite{})
+}

internal/trans/input.go

@@ -0,0 +1,53 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func (i *importer) inputEntry(ctx context.Context, entry transmodel.TransEntry) error {
+	t, ok := entry[transmodel.TypeKey].(string)
+	if !ok {
+		return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json")
+	}
+
+	switch transmodel.TransType(t) {
+	case transmodel.TransAccount:
+		account, err := accountDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into account: %s", err)
+		}
+		if err := i.putInDB(ctx, account); err != nil {
+			return fmt.Errorf("inputEntry: error adding account to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added account with id %s", account.ID)
+		return nil
+	}
+
+	return fmt.Errorf("inputEntry: didn't recognize transtype %s", t)
+}
+
+func (i *importer) putInDB(ctx context.Context, entry interface{}) error {
+	return i.db.Put(ctx, entry)
+}

internal/trans/model/account.go

@@ -25,24 +25,25 @@ import (
 
 // Account represents the minimum viable representation of an account for export/import.
 type Account struct {
+	Type                  TransType       `json:"type" bun:"-"`
 	ID                    string          `json:"id"`
-	CreatedAt             *time.Time      `json:"created_at"`
-	UpdatedAt             *time.Time      `json:"updated_at"`
+	CreatedAt             *time.Time      `json:"createdAt"`
+	UpdatedAt             *time.Time      `json:"updatedAt"`
 	Username              string          `json:"username"`
 	Domain                string          `json:"domain,omitempty"`
 	Locked                bool            `json:"locked"`
 	Language              string          `json:"language,omitempty"`
 	URI                   string          `json:"uri"`
 	URL                   string          `json:"url"`
-	InboxURI              string          `json:"inbox_uri"`
-	OutboxURI             string          `json:"outbox_uri"`
-	FollowingURI          string          `json:"following_uri"`
-	FollowersURI          string          `json:"followers_uri"`
-	FeaturedCollectionURI string          `json:"featured_collection_uri"`
-	ActorType             string          `json:"actor_type"`
-	PrivateKey            *rsa.PrivateKey `json:"private_key,omitempty"`
-	PublicKey             *rsa.PublicKey  `json:"public_key"`
-	PublicKeyURI          string          `json:"public_key_uri"`
-	SuspendedAt           *time.Time      `json:"suspended_at,omitempty"`
-	SuspensionOrigin      string          `json:"suspension_origin,omitempty"`
+	InboxURI              string          `json:"inboxURI"`
+	OutboxURI             string          `json:"outboxURI"`
+	FollowingURI          string          `json:"followingUri"`
+	FollowersURI          string          `json:"followersUri"`
+	FeaturedCollectionURI string          `json:"featuredCollectionUri"`
+	ActorType             string          `json:"actorType"`
+	PrivateKey            *rsa.PrivateKey `json:"privateKey,omitempty"`
+	PublicKey             *rsa.PublicKey  `json:"publicKey"`
+	PublicKeyURI          string          `json:"publicKeyUri"`
+	SuspendedAt           *time.Time      `json:"suspendedAt,omitempty"`
+	SuspensionOrigin      string          `json:"suspensionOrigin,omitempty"`
 }

internal/trans/model/block.go

@@ -21,10 +21,11 @@ package trans
 import "time"
 
 type Block struct {
+	Type            TransType `json:"type" bun:"-"`
 	ID              string    `json:"id"`
-	CreatedAt       time.Time `json:"created_at"`
-	UpdatedAt       time.Time `json:"updated_at"`
+	CreatedAt       time.Time `json:"createdAt"`
+	UpdatedAt       time.Time `json:"updatedAt"`
 	URI             string    `json:"uri"`
-	AccountID       string    `json:"account_id"`
-	TargetAccountID string    `json:"target_account_id"`
+	AccountID       string    `json:"accountId"`
+	TargetAccountID string    `json:"targetAccountId"`
 }

internal/trans/model/type.go

@@ -0,0 +1,32 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+const TypeKey = "type"
+
+// TransType describes the type of a trans entry, and how it should be read/serialized.
+type TransType string
+
+// Type of the trans entry. Describes how it should be read from file.
+const (
+	TransAccount TransType = "account"
+	TransBlock   TransType = "block"
+)
+
+type TransEntry map[string]interface{}

internal/trans/trans_test.go

@@ -19,6 +19,7 @@
 package trans_test
 
 import (
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/testrig"
@@ -26,11 +27,13 @@ import (
 
 type TransTestSuite struct {
 	suite.Suite
-	db db.DB
+	db  db.DB
+	log *logrus.Logger
 }
 
 func (suite *TransTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
+   suite.log = testrig.NewTestLog()
 	testrig.StandardDBSetup(suite.db, nil)
 }