Import export (#194)

* start with export/import code

* messing about with decoding/encoding

* some more fiddling

* stuff is WORKING

* working pretty alright!

* go fmt

* fix up tests, add docs

* start backup/restore doc

* tweaks

* credits

* update advancedVisibility settings

* update bun library -> v1.0.4

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update oauth library -> v4.3.1-SSB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* handle oauth token scope, fix user.SigninCount + token.UserID

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update oauth library --> v4.3.2-SSB

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* update sqlite library -> v1.13.0

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

* review changes

* start with export/import code

* messing about with decoding/encoding

* some more fiddling

* stuff is WORKING

* working pretty alright!

* go fmt

* fix up tests, add docs

* start backup/restore doc

* tweaks

* credits

* update advancedVisibility settings

* review changes

Co-authored-by: kim (grufwub) <grufwub@gmail.com>
Co-authored-by: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>

internal/api/s2s/user/repliesget_test.go

@@ -157,7 +157,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() {
 	b, err := ioutil.ReadAll(result.Body)
 	assert.NoError(suite.T(), err)
 
-	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
+	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/admin/statuses/01FF25D5Q0DH7CHD57CTRS6WK0","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
 
 	// should be a Collection
 	m := make(map[string]interface{})
@@ -188,7 +188,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
 	// setup request
 	recorder := httptest.NewRecorder()
 	ctx, _ := gin.CreateTestContext(recorder)
-	ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FF25D5Q0DH7CHD57CTRS6WK0", nil) // the endpoint we're hitting
 	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
 	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
 
@@ -220,7 +220,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
 	assert.NoError(suite.T(), err)
 
 	fmt.Println(string(b))
-	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
+	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FF25D5Q0DH7CHD57CTRS6WK0","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
 
 	// should be a Collection
 	m := make(map[string]interface{})

internal/cache/status.go

@@ -144,7 +144,10 @@ func copyStatus(status *gtsmodel.Status) *gtsmodel.Status {
 		Sensitive:                status.Sensitive,
 		Language:                 status.Language,
 		CreatedWithApplicationID: status.CreatedWithApplicationID,
-		VisibilityAdvanced:       status.VisibilityAdvanced,
+		Federated:                status.Federated,
+		Boostable:                status.Boostable,
+		Replyable:                status.Replyable,
+		Likeable:                 status.Likeable,
 		ActivityStreamsType:      status.ActivityStreamsType,
 		Text:                     status.Text,
 		Pinned:                   status.Pinned,

internal/cliactions/admin/trans/export.go

@@ -0,0 +1,52 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/cliactions"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+)
+
+// Export exports info from the database into a file
+var Export cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
+	dbConn, err := bundb.NewBunDBService(ctx, c, log)
+	if err != nil {
+		return fmt.Errorf("error creating dbservice: %s", err)
+	}
+
+	exporter := trans.NewExporter(dbConn, log)
+
+	path, ok := c.ExportCLIFlags[config.TransPathFlag]
+	if !ok {
+		return errors.New("no path set")
+	}
+
+	if err := exporter.ExportMinimal(ctx, path); err != nil {
+		return err
+	}
+
+	return dbConn.Stop(ctx)
+}

internal/cliactions/admin/trans/import.go

@@ -0,0 +1,56 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/cliactions"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+)
+
+// Import imports info from a file into the database
+var Import cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
+	dbConn, err := bundb.NewBunDBService(ctx, c, log)
+	if err != nil {
+		return fmt.Errorf("error creating dbservice: %s", err)
+	}
+
+	importer := trans.NewImporter(dbConn, log)
+
+	path, ok := c.ExportCLIFlags[config.TransPathFlag]
+	if !ok {
+		return errors.New("no path set")
+	}
+
+	if err := dbConn.CreateAllTables(ctx); err != nil {
+		return err
+	}
+
+	if err := importer.Import(ctx, path); err != nil {
+		return err
+	}
+
+	return dbConn.Stop(ctx)
+}

internal/cliactions/server/server.go

@@ -39,7 +39,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/gotosocial"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/oidc"
@@ -51,32 +50,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/web"
 )
 
-var models []interface{} = []interface{}{
-	&gtsmodel.Account{},
-	&gtsmodel.Application{},
-	&gtsmodel.Block{},
-	&gtsmodel.DomainBlock{},
-	&gtsmodel.EmailDomainBlock{},
-	&gtsmodel.Follow{},
-	&gtsmodel.FollowRequest{},
-	&gtsmodel.MediaAttachment{},
-	&gtsmodel.Mention{},
-	&gtsmodel.Status{},
-	&gtsmodel.StatusToEmoji{},
-	&gtsmodel.StatusToTag{},
-	&gtsmodel.StatusFave{},
-	&gtsmodel.StatusBookmark{},
-	&gtsmodel.StatusMute{},
-	&gtsmodel.Tag{},
-	&gtsmodel.User{},
-	&gtsmodel.Emoji{},
-	&gtsmodel.Instance{},
-	&gtsmodel.Notification{},
-	&gtsmodel.RouterSession{},
-	&gtsmodel.Token{},
-	&gtsmodel.Client{},
-}
-
 // Start creates and starts a gotosocial server
 var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log *logrus.Logger) error {
 	dbService, err := bundb.NewBunDBService(ctx, c, log)
@@ -84,10 +57,8 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log
 		return fmt.Errorf("error creating dbservice: %s", err)
 	}
 
-	for _, m := range models {
-		if err := dbService.CreateTable(ctx, m); err != nil {
-			return fmt.Errorf("table creation error: %s", err)
-		}
+	if err := dbService.CreateAllTables(ctx); err != nil {
+		return fmt.Errorf("error creating database tables: %s", err)
 	}
 
 	if err := dbService.CreateInstanceAccount(ctx); err != nil {

internal/config/config.go

@@ -36,6 +36,9 @@ const (
 
 	PasswordFlag  = "password"
 	PasswordUsage = "the password to set for this account"
+
+	TransPathFlag  = "path"
+	TransPathUsage = "the path of the file to import from/export to"
 )
 
 // Config pulls together all the configuration needed to run gotosocial
@@ -65,6 +68,7 @@ type Config struct {
 		Not parsed from .yaml configuration file.
 	*/
 	AccountCLIFlags map[string]string
+	ExportCLIFlags  map[string]string
 	SoftwareVersion string
 }
 
@@ -92,6 +96,7 @@ func Empty() *Config {
 		LetsEncryptConfig: &LetsEncryptConfig{},
 		OIDCConfig:        &OIDCConfig{},
 		AccountCLIFlags:   make(map[string]string),
+		ExportCLIFlags:    make(map[string]string),
 	}
 }
 
@@ -320,6 +325,9 @@ func (c *Config) ParseCLIFlags(f KeyedFlags, version string) error {
 	c.AccountCLIFlags[EmailFlag] = f.String(EmailFlag)
 	c.AccountCLIFlags[PasswordFlag] = f.String(PasswordFlag)
 
+	// export CLI flags
+	c.ExportCLIFlags[TransPathFlag] = f.String(TransPathFlag)
+
 	c.SoftwareVersion = version
 	return nil
 }

internal/db/basic.go

@@ -26,6 +26,10 @@ type Basic interface {
 	// For implementations that don't use tables, this can just return nil.
 	CreateTable(ctx context.Context, i interface{}) Error
 
+	// CreateAllTables creates *all* tables necessary for the running of GoToSocial.
+	// Because it uses the 'if not exists' parameter it is safe to run against a GtS that's already been initialized.
+	CreateAllTables(ctx context.Context) Error
+
 	// DropTable drops the table for the given interface.
 	// For implementations that don't use tables, this can just return nil.
 	DropTable(ctx context.Context, i interface{}) Error

internal/db/bundb/basic.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/uptrace/bun"
 )
 
@@ -53,17 +54,8 @@ func (b *basicDB) GetWhere(ctx context.Context, where []db.Where, i interface{})
 	}
 
 	q := b.conn.NewSelect().Model(i)
-	for _, w := range where {
-		if w.Value == nil {
-			q = q.Where("? IS NULL", bun.Ident(w.Key))
-		} else {
-			if w.CaseInsensitive {
-				q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value)
-			} else {
-				q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
-			}
-		}
-	}
+
+	selectWhere(q, where)
 
 	err := q.Scan(ctx)
 	return b.conn.ProcessError(err)
@@ -97,9 +89,7 @@ func (b *basicDB) DeleteWhere(ctx context.Context, where []db.Where, i interface
 		NewDelete().
 		Model(i)
 
-	for _, w := range where {
-		q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
-	}
+	deleteWhere(q, where)
 
 	_, err := q.Exec(ctx)
 	return b.conn.ProcessError(err)
@@ -128,17 +118,7 @@ func (b *basicDB) UpdateOneByID(ctx context.Context, id string, key string, valu
 func (b *basicDB) UpdateWhere(ctx context.Context, where []db.Where, key string, value interface{}, i interface{}) db.Error {
 	q := b.conn.NewUpdate().Model(i)
 
-	for _, w := range where {
-		if w.Value == nil {
-			q = q.Where("? IS NULL", bun.Ident(w.Key))
-		} else {
-			if w.CaseInsensitive {
-				q = q.Where("LOWER(?) = LOWER(?)", bun.Safe(w.Key), w.Value)
-			} else {
-				q = q.Where("? = ?", bun.Safe(w.Key), w.Value)
-			}
-		}
-	}
+	updateWhere(q, where)
 
 	q = q.Set("? = ?", bun.Safe(key), value)
 
@@ -151,6 +131,40 @@ func (b *basicDB) CreateTable(ctx context.Context, i interface{}) db.Error {
 	return err
 }
 
+func (b *basicDB) CreateAllTables(ctx context.Context) db.Error {
+	models := []interface{}{
+		&gtsmodel.Account{},
+		&gtsmodel.Application{},
+		&gtsmodel.Block{},
+		&gtsmodel.DomainBlock{},
+		&gtsmodel.EmailDomainBlock{},
+		&gtsmodel.Follow{},
+		&gtsmodel.FollowRequest{},
+		&gtsmodel.MediaAttachment{},
+		&gtsmodel.Mention{},
+		&gtsmodel.Status{},
+		&gtsmodel.StatusToEmoji{},
+		&gtsmodel.StatusToTag{},
+		&gtsmodel.StatusFave{},
+		&gtsmodel.StatusBookmark{},
+		&gtsmodel.StatusMute{},
+		&gtsmodel.Tag{},
+		&gtsmodel.User{},
+		&gtsmodel.Emoji{},
+		&gtsmodel.Instance{},
+		&gtsmodel.Notification{},
+		&gtsmodel.RouterSession{},
+		&gtsmodel.Token{},
+		&gtsmodel.Client{},
+	}
+	for _, i := range models {
+		if err := b.CreateTable(ctx, i); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (b *basicDB) DropTable(ctx context.Context, i interface{}) db.Error {
 	_, err := b.conn.NewDropTable().Model(i).IfExists().Exec(ctx)
 	return b.conn.ProcessError(err)

internal/db/bundb/basic_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
@@ -42,7 +43,25 @@ func (suite *BasicTestSuite) TestGetAllStatuses() {
 	s := []*gtsmodel.Status{}
 	err := suite.db.GetAll(context.Background(), &s)
 	suite.NoError(err)
-	suite.Len(s, 12)
+	suite.Len(s, 13)
+}
+
+func (suite *BasicTestSuite) TestGetAllNotNull() {
+	where := []db.Where{{
+		Key:   "domain",
+		Value: nil,
+		Not:   true,
+	}}
+
+	a := []*gtsmodel.Account{}
+
+	err := suite.db.GetWhere(context.Background(), where, &a)
+	suite.NoError(err)
+	suite.NotEmpty(a)
+
+	for _, acct := range a {
+		suite.NotEmpty(acct.Domain)
+	}
 }
 
 func TestBasicTestSuite(t *testing.T) {

internal/db/bundb/status.go

@@ -240,11 +240,11 @@ func (s *statusDB) statusChildren(ctx context.Context, status *gtsmodel.Status,
 			}
 		}
 
-		// only do one loop if we only want direct children
-		if onlyDirect {
-			return
+		// if we're not only looking for direct children of status, then do the same children-finding
+		// operation for the found child status too.
+		if !onlyDirect {
+			s.statusChildren(ctx, child, foundStatuses, false, minID)
 		}
-		s.statusChildren(ctx, child, foundStatuses, false, minID)
 	}
 }
 

internal/db/bundb/status_test.go

@@ -43,10 +43,14 @@ func (suite *StatusTestSuite) TestGetStatusByID() {
 	suite.Nil(status.BoostOfAccount)
 	suite.Nil(status.InReplyTo)
 	suite.Nil(status.InReplyToAccount)
+	suite.True(status.Federated)
+	suite.True(status.Boostable)
+	suite.True(status.Replyable)
+	suite.True(status.Likeable)
 }
 
 func (suite *StatusTestSuite) TestGetStatusByURI() {
-	status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_1_status_1"].URI)
+	status, err := suite.db.GetStatusByURI(context.Background(), suite.testStatuses["local_account_2_status_3"].URI)
 	if err != nil {
 		suite.FailNow(err.Error())
 	}
@@ -57,6 +61,10 @@ func (suite *StatusTestSuite) TestGetStatusByURI() {
 	suite.Nil(status.BoostOfAccount)
 	suite.Nil(status.InReplyTo)
 	suite.Nil(status.InReplyToAccount)
+	suite.True(status.Federated)
+	suite.True(status.Boostable)
+	suite.False(status.Replyable)
+	suite.False(status.Likeable)
 }
 
 func (suite *StatusTestSuite) TestGetStatusWithExtras() {
@@ -70,6 +78,10 @@ func (suite *StatusTestSuite) TestGetStatusWithExtras() {
 	suite.NotEmpty(status.Tags)
 	suite.NotEmpty(status.Attachments)
 	suite.NotEmpty(status.Emojis)
+	suite.True(status.Federated)
+	suite.True(status.Boostable)
+	suite.True(status.Replyable)
+	suite.True(status.Likeable)
 }
 
 func (suite *StatusTestSuite) TestGetStatusWithMention() {
@@ -83,6 +95,10 @@ func (suite *StatusTestSuite) TestGetStatusWithMention() {
 	suite.NotEmpty(status.MentionIDs)
 	suite.NotEmpty(status.InReplyToID)
 	suite.NotEmpty(status.InReplyToAccountID)
+	suite.True(status.Federated)
+	suite.True(status.Boostable)
+	suite.True(status.Replyable)
+	suite.True(status.Likeable)
 }
 
 func (suite *StatusTestSuite) TestGetStatusTwice() {
@@ -104,6 +120,18 @@ func (suite *StatusTestSuite) TestGetStatusTwice() {
 	suite.Less(duration2, duration1)
 }
 
+func (suite *StatusTestSuite) TestGetStatusChildren() {
+	targetStatus := suite.testStatuses["local_account_1_status_1"]
+	children, err := suite.db.GetStatusChildren(context.Background(), targetStatus, true, "")
+	suite.NoError(err)
+	suite.Len(children, 2)
+	for _, c := range children {
+		suite.Equal(targetStatus.URI, c.InReplyToURI)
+		suite.Equal(targetStatus.AccountID, c.InReplyToAccountID)
+		suite.Equal(targetStatus.ID, c.InReplyToID)
+	}
+}
+
 func TestStatusTestSuite(t *testing.T) {
 	suite.Run(t, new(StatusTestSuite))
 }

internal/db/bundb/util.go

@@ -19,6 +19,7 @@
 package bundb
 
 import (
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/uptrace/bun"
 )
 
@@ -35,3 +36,65 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
 			WhereOr("? = ''", bun.Ident(column))
 	}
 }
+
+// updateWhere parses []db.Where and adds it to the given update query.
+func updateWhere(q *bun.UpdateQuery, where []db.Where) {
+	for _, w := range where {
+		query, args := parseWhere(w)
+		q = q.Where(query, args...)
+	}
+}
+
+// selectWhere parses []db.Where and adds it to the given select query.
+func selectWhere(q *bun.SelectQuery, where []db.Where) {
+	for _, w := range where {
+		query, args := parseWhere(w)
+		q = q.Where(query, args...)
+	}
+}
+
+// deleteWhere parses []db.Where and adds it to the given where query.
+func deleteWhere(q *bun.DeleteQuery, where []db.Where) {
+	for _, w := range where {
+		query, args := parseWhere(w)
+		q = q.Where(query, args...)
+	}
+}
+
+// parseWhere looks through the options on a single db.Where entry, and
+// returns the appropriate query string and arguments.
+func parseWhere(w db.Where) (query string, args []interface{}) {
+	if w.Not {
+		if w.Value == nil {
+			query = "? IS NOT NULL"
+			args = []interface{}{bun.Ident(w.Key)}
+			return
+		}
+
+		if w.CaseInsensitive {
+			query = "LOWER(?) != LOWER(?)"
+			args = []interface{}{bun.Safe(w.Key), w.Value}
+			return
+		}
+
+		query = "? != ?"
+		args = []interface{}{bun.Safe(w.Key), w.Value}
+		return
+	}
+
+	if w.Value == nil {
+		query = "? IS NULL"
+		args = []interface{}{bun.Ident(w.Key)}
+		return
+	}
+
+	if w.CaseInsensitive {
+		query = "LOWER(?) = LOWER(?)"
+		args = []interface{}{bun.Safe(w.Key), w.Value}
+		return
+	}
+
+	query = "? = ?"
+	args = []interface{}{bun.Safe(w.Key), w.Value}
+	return
+}

internal/db/params.go

@@ -22,9 +22,13 @@ package db
 type Where struct {
 	// The table to search on.
 	Key string
-	// The value that must be set.
+	// The value to match.
 	Value interface{}
 	// Whether the value (if a string) should be case sensitive or not.
 	// Defaults to false.
 	CaseInsensitive bool
+	// If set, reverse the where.
+	// `WHERE k = v` becomes `WHERE k != v`.
+	// `WHERE k IS NULL` becomes `WHERE k IS NOT NULL`
+	Not bool
 }

internal/federation/dereferencing/announce.go

@@ -60,7 +60,10 @@ func (d *deref) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Stat
 	announce.BoostOfID = boostedStatus.ID
 	announce.BoostOfAccountID = boostedStatus.AccountID
 	announce.Visibility = boostedStatus.Visibility
-	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+	announce.Federated = boostedStatus.Federated
+	announce.Boostable = boostedStatus.Boostable
+	announce.Replyable = boostedStatus.Replyable
+	announce.Likeable = boostedStatus.Likeable
 	announce.BoostOf = boostedStatus
 	return nil
 }

internal/federation/dereferencing/status_test.go

@@ -58,6 +58,10 @@ func (suite *StatusTestSuite) TestDereferenceSimpleStatus() {
 	dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
 	suite.NoError(err)
 	suite.Equal(status.ID, dbStatus.ID)
+	suite.True(dbStatus.Federated)
+	suite.True(dbStatus.Boostable)
+	suite.True(dbStatus.Replyable)
+	suite.True(dbStatus.Likeable)
 
 	// account should be in the database now too
 	account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)
@@ -96,6 +100,10 @@ func (suite *StatusTestSuite) TestDereferenceStatusWithMention() {
 	dbStatus, err := suite.db.GetStatusByURI(context.Background(), status.URI)
 	suite.NoError(err)
 	suite.Equal(status.ID, dbStatus.ID)
+	suite.True(dbStatus.Federated)
+	suite.True(dbStatus.Boostable)
+	suite.True(dbStatus.Replyable)
+	suite.True(dbStatus.Likeable)
 
 	// account should be in the database now too
 	account, err := suite.db.GetAccountByURI(context.Background(), status.AccountURI)

internal/gtsmodel/status.go

@@ -57,10 +57,13 @@ type Status struct {
 	Language                 string             `validate:"-" bun:",nullzero"`                                                                         // what language is this status written in?
 	CreatedWithApplicationID string             `validate:"required_if=Local true,omitempty,ulid" bun:"type:CHAR(26),nullzero"`                        // Which application was used to create this status?
 	CreatedWithApplication   *Application       `validate:"-" bun:"rel:belongs-to"`                                                                    // application corresponding to createdWithApplicationID
-	VisibilityAdvanced       VisibilityAdvanced `validate:"required" bun:",nullzero,notnull" `                                                         // advanced visibility for this status
 	ActivityStreamsType      string             `validate:"required" bun:",nullzero,notnull"`                                                          // What is the activitystreams type of this status? See: https://www.w3.org/TR/activitystreams-vocabulary/#object-types. Will probably almost always be Note but who knows!.
 	Text                     string             `validate:"-" bun:",nullzero"`                                                                         // Original text of the status without formatting
-	Pinned                   bool               `validate:"-" bun:",notnull,default:false" `                                                           // Has this status been pinned by its owner?
+	Pinned                   bool               `validate:"-" bun:",notnull,default:false"`                                                            // Has this status been pinned by its owner?
+	Federated                bool               `validate:"-" bun:",notnull"`                                                                          // This status will be federated beyond the local timeline(s)
+	Boostable                bool               `validate:"-" bun:",notnull"`                                                                          // This status can be boosted/reblogged
+	Replyable                bool               `validate:"-" bun:",notnull"`                                                                          // This status can be replied to
+	Likeable                 bool               `validate:"-" bun:",notnull"`                                                                          // This status can be liked/faved
 }
 
 // StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags.
@@ -96,21 +99,3 @@ const (
 	// VisibilityDefault is used when no other setting can be found.
 	VisibilityDefault Visibility = VisibilityUnlocked
 )
-
-// VisibilityAdvanced models flags for fine-tuning visibility and interactivity of a status.
-//
-// All flags default to true.
-//
-// If PUBLIC is selected, flags will all be overwritten to TRUE regardless of what is selected.
-//
-// If UNLOCKED is selected, any flags can be turned on or off in any combination.
-//
-// If FOLLOWERS-ONLY or MUTUALS-ONLY are selected, boostable will always be FALSE. Other flags can be turned on or off as desired.
-//
-// If DIRECT is selected, boostable will be FALSE, and all other flags will be TRUE.
-type VisibilityAdvanced struct {
-	Federated bool `validate:"-" bun:",notnull,default:true"` // This status will be federated beyond the local timeline(s)
-	Boostable bool `validate:"-" bun:",notnull,default:true"` // This status can be boosted/reblogged
-	Replyable bool `validate:"-" bun:",notnull,default:true"` // This status can be replied to
-	Likeable  bool `validate:"-" bun:",notnull,default:true"` // This status can be liked/faved
-}

internal/processing/fromclientapi.go

@@ -51,7 +51,7 @@ func (p *processor) ProcessFromClientAPI(ctx context.Context, clientMsg messages
 				return err
 			}
 
-			if status.VisibilityAdvanced.Federated {
+			if status.Federated {
 				return p.federateStatus(ctx, status)
 			}
 		case ap.ActivityFollow:

internal/processing/status/boost.go

@@ -46,7 +46,7 @@ func (p *processor) Boost(ctx context.Context, requestingAccount *gtsmodel.Accou
 	if !visible {
 		return nil, gtserror.NewErrorNotFound(errors.New("status is not visible"))
 	}
-	if !targetStatus.VisibilityAdvanced.Boostable {
+	if !targetStatus.Boostable {
 		return nil, gtserror.NewErrorForbidden(errors.New("status is not boostable"))
 	}
 

internal/processing/status/fave.go

@@ -49,7 +49,7 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun
 	if !visible {
 		return nil, gtserror.NewErrorNotFound(errors.New("status is not visible"))
 	}
-	if !targetStatus.VisibilityAdvanced.Likeable {
+	if !targetStatus.Likeable {
 		return nil, gtserror.NewErrorForbidden(errors.New("status is not faveable"))
 	}
 

internal/processing/status/util.go

@@ -33,12 +33,10 @@ import (
 
 func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
 	// by default all flags are set to true
-	gtsAdvancedVis := gtsmodel.VisibilityAdvanced{
-		Federated: true,
-		Boostable: true,
-		Replyable: true,
-		Likeable:  true,
-	}
+	federated := true
+	boostable := true
+	replyable := true
+	likeable := true
 
 	var vis gtsmodel.Visibility
 	// If visibility isn't set on the form, then just take the account default.
@@ -58,47 +56,50 @@ func (p *processor) ProcessVisibility(ctx context.Context, form *apimodel.Advanc
 	case gtsmodel.VisibilityUnlocked:
 		// for unlocked the user can set any combination of flags they like so look at them all to see if they're set and then apply them
 		if form.Federated != nil {
-			gtsAdvancedVis.Federated = *form.Federated
+			federated = *form.Federated
 		}
 
 		if form.Boostable != nil {
-			gtsAdvancedVis.Boostable = *form.Boostable
+			boostable = *form.Boostable
 		}
 
 		if form.Replyable != nil {
-			gtsAdvancedVis.Replyable = *form.Replyable
+			replyable = *form.Replyable
 		}
 
 		if form.Likeable != nil {
-			gtsAdvancedVis.Likeable = *form.Likeable
+			likeable = *form.Likeable
 		}
 
 	case gtsmodel.VisibilityFollowersOnly, gtsmodel.VisibilityMutualsOnly:
 		// for followers or mutuals only, boostable will *always* be false, but the other fields can be set so check and apply them
-		gtsAdvancedVis.Boostable = false
+		boostable = false
 
 		if form.Federated != nil {
-			gtsAdvancedVis.Federated = *form.Federated
+			federated = *form.Federated
 		}
 
 		if form.Replyable != nil {
-			gtsAdvancedVis.Replyable = *form.Replyable
+			replyable = *form.Replyable
 		}
 
 		if form.Likeable != nil {
-			gtsAdvancedVis.Likeable = *form.Likeable
+			likeable = *form.Likeable
 		}
 
 	case gtsmodel.VisibilityDirect:
 		// direct is pretty easy: there's only one possible setting so return it
-		gtsAdvancedVis.Federated = true
-		gtsAdvancedVis.Boostable = false
-		gtsAdvancedVis.Federated = true
-		gtsAdvancedVis.Likeable = true
+		federated = true
+		boostable = false
+		replyable = true
+		likeable = true
 	}
 
 	status.Visibility = vis
-	status.VisibilityAdvanced = gtsAdvancedVis
+	status.Federated = federated
+	status.Boostable = boostable
+	status.Replyable = replyable
+	status.Likeable = likeable
 	return nil
 }
 
@@ -123,7 +124,7 @@ func (p *processor) ProcessReplyToID(ctx context.Context, form *apimodel.Advance
 		}
 		return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err)
 	}
-	if !repliedStatus.VisibilityAdvanced.Replyable {
+	if !repliedStatus.Replyable {
 		return fmt.Errorf("status with id %s is marked as not replyable", form.InReplyToID)
 	}
 

internal/timeline/get_test.go

@@ -73,8 +73,8 @@ func (suite *GetTestSuite) TestGetDefault() {
 		suite.FailNow(err.Error())
 	}
 
-	// we only have 12 statuses in the test suite
-	suite.Len(statuses, 12)
+	// we only have 13 statuses in the test suite
+	suite.Len(statuses, 13)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -166,8 +166,8 @@ func (suite *GetTestSuite) TestGetMinID() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 5 statuses back, since we asked for a min ID that excludes some of our entries
-	suite.Len(statuses, 5)
+	// we should only get 6 statuses back, since we asked for a min ID that excludes some of our entries
+	suite.Len(statuses, 6)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -188,8 +188,8 @@ func (suite *GetTestSuite) TestGetSinceID() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries
-	suite.Len(statuses, 5)
+	// we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries
+	suite.Len(statuses, 6)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string
@@ -210,8 +210,8 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
 		suite.FailNow(err.Error())
 	}
 
-	// we should only get 5 statuses back, since we asked for a since ID that excludes some of our entries
-	suite.Len(statuses, 5)
+	// we should only get 6 statuses back, since we asked for a since ID that excludes some of our entries
+	suite.Len(statuses, 6)
 
 	// statuses should be sorted highest to lowest ID
 	var highest string

internal/timeline/index_test.go

@@ -66,7 +66,7 @@ func (suite *IndexTestSuite) TestIndexBeforeLowID() {
 	// the oldest indexed post should be the lowest one we have in our testrig
 	postID, err := suite.timeline.OldestIndexedPostID(context.Background())
 	suite.NoError(err)
-	suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", postID)
+	suite.Equal("01F8MHAMCHF6Y650WCRSCP4WMY", postID)
 
 	indexLength := suite.timeline.PostIndexLength(context.Background())
 	suite.Equal(10, indexLength)
@@ -95,7 +95,7 @@ func (suite *IndexTestSuite) TestIndexBehindHighID() {
 	// the newest indexed post should be the highest one we have in our testrig
 	postID, err := suite.timeline.NewestIndexedPostID(context.Background())
 	suite.NoError(err)
-	suite.Equal("01FCTA44PW9H1TB328S9AQXKDS", postID)
+	suite.Equal("01FF25D5Q0DH7CHD57CTRS6WK0", postID)
 
 	// indexLength should be 10 because that's all this user has hometimelineable
 	indexLength := suite.timeline.PostIndexLength(context.Background())

internal/timeline/manager_test.go

@@ -67,9 +67,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 	err = suite.manager.PrepareXFromTop(context.Background(), testAccount.ID, 20)
 	suite.NoError(err)
 
-	// local_account_1 can see 12 statuses out of the testrig statuses in its home timeline
+	// local_account_1 can see 13 statuses out of the testrig statuses in its home timeline
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(12, indexedLen)
+	suite.Equal(13, indexedLen)
 
 	// oldest should now be set
 	oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@@ -79,7 +79,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 	// get hometimeline
 	statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false)
 	suite.NoError(err)
-	suite.Len(statuses, 12)
+	suite.Len(statuses, 13)
 
 	// now wipe the last status from all timelines, as though it had been deleted by the owner
 	err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R")
@@ -87,7 +87,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 
 	// timeline should be shorter
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(11, indexedLen)
+	suite.Equal(12, indexedLen)
 
 	// oldest should now be different
 	oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@@ -101,7 +101,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 
 	// timeline should be shorter
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(10, indexedLen)
+	suite.Equal(11, indexedLen)
 
 	// oldest should now be different
 	oldestIndexed, err = suite.manager.GetOldestIndexedID(context.Background(), testAccount.ID)
@@ -112,9 +112,9 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 	err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID)
 	suite.NoError(err)
 
-	// timeline should be empty now
+	// timeline should be shorter
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(5, indexedLen)
+	suite.Equal(6, indexedLen)
 
 	// ingest 1 into the timeline
 	status1 := suite.testStatuses["admin_account_status_1"]
@@ -130,7 +130,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
 
 	// timeline should be longer now
 	indexedLen = suite.manager.GetIndexedLength(context.Background(), testAccount.ID)
-	suite.Equal(7, indexedLen)
+	suite.Equal(8, indexedLen)
 
 	// try to ingest status 2 again
 	ingested, err = suite.manager.IngestAndPrepare(context.Background(), status2, testAccount.ID)

internal/trans/decoders.go

@@ -0,0 +1,138 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mitchellh/mapstructure"
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func newDecoder(target interface{}) (*mapstructure.Decoder, error) {
+	decoderConfig := &mapstructure.DecoderConfig{
+		DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), // this is needed to decode time.Time entries serialized as string
+		Result:     target,
+	}
+	return mapstructure.NewDecoder(decoderConfig)
+}
+
+func (i *importer) accountDecode(e transmodel.Entry) (*transmodel.Account, error) {
+	a := &transmodel.Account{}
+	if err := i.simpleDecode(e, a); err != nil {
+		return nil, err
+	}
+
+	// extract public key
+	publicKeyBlock, _ := pem.Decode([]byte(a.PublicKeyString))
+	if publicKeyBlock == nil {
+		return nil, errors.New("accountDecode: error decoding account public key")
+	}
+	publicKey, err := x509.ParsePKCS1PublicKey(publicKeyBlock.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("accountDecode: error parsing account public key: %s", err)
+	}
+	a.PublicKey = publicKey
+
+	if a.Domain == "" {
+		// extract private key (local account)
+		privateKeyBlock, _ := pem.Decode([]byte(a.PrivateKeyString))
+		if privateKeyBlock == nil {
+			return nil, errors.New("accountDecode: error decoding account private key")
+		}
+		privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyBlock.Bytes)
+		if err != nil {
+			return nil, fmt.Errorf("accountDecode: error parsing account private key: %s", err)
+		}
+		a.PrivateKey = privateKey
+	}
+
+	return a, nil
+}
+
+func (i *importer) blockDecode(e transmodel.Entry) (*transmodel.Block, error) {
+	b := &transmodel.Block{}
+	if err := i.simpleDecode(e, b); err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (i *importer) domainBlockDecode(e transmodel.Entry) (*transmodel.DomainBlock, error) {
+	b := &transmodel.DomainBlock{}
+	if err := i.simpleDecode(e, b); err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func (i *importer) followDecode(e transmodel.Entry) (*transmodel.Follow, error) {
+	f := &transmodel.Follow{}
+	if err := i.simpleDecode(e, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func (i *importer) followRequestDecode(e transmodel.Entry) (*transmodel.FollowRequest, error) {
+	f := &transmodel.FollowRequest{}
+	if err := i.simpleDecode(e, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func (i *importer) instanceDecode(e transmodel.Entry) (*transmodel.Instance, error) {
+	inst := &transmodel.Instance{}
+	if err := i.simpleDecode(e, inst); err != nil {
+		return nil, err
+	}
+
+	return inst, nil
+}
+
+func (i *importer) userDecode(e transmodel.Entry) (*transmodel.User, error) {
+	u := &transmodel.User{}
+	if err := i.simpleDecode(e, u); err != nil {
+		return nil, err
+	}
+
+	return u, nil
+}
+
+func (i *importer) simpleDecode(entry transmodel.Entry, target interface{}) error {
+	decoder, err := newDecoder(target)
+	if err != nil {
+		return fmt.Errorf("simpleDecode: error creating decoder: %s", err)
+	}
+
+	if err := decoder.Decode(&entry); err != nil {
+		return fmt.Errorf("simpleDecode: error decoding: %s", err)
+	}
+
+	return nil
+}

internal/trans/encoders.go

@@ -0,0 +1,83 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"os"
+
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+// accountEncode handles special fields like private + public keys on accounts
+func (e *exporter) accountEncode(ctx context.Context, f *os.File, a *transmodel.Account) error {
+	a.Type = transmodel.TransAccount
+
+	// marshal public key
+	encodedPublicKey := x509.MarshalPKCS1PublicKey(a.PublicKey)
+	if encodedPublicKey == nil {
+		return errors.New("could not MarshalPKCS1PublicKey")
+	}
+	publicKeyBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PUBLIC KEY",
+		Bytes: encodedPublicKey,
+	})
+	a.PublicKeyString = string(publicKeyBytes)
+
+	if a.Domain == "" {
+		// marshal private key for local account
+		encodedPrivateKey := x509.MarshalPKCS1PrivateKey(a.PrivateKey)
+		if encodedPrivateKey == nil {
+			return errors.New("could not MarshalPKCS1PrivateKey")
+		}
+		privateKeyBytes := pem.EncodeToMemory(&pem.Block{
+			Type:  "RSA PRIVATE KEY",
+			Bytes: encodedPrivateKey,
+		})
+		a.PrivateKeyString = string(privateKeyBytes)
+	}
+
+	return e.simpleEncode(ctx, f, a, a.ID)
+}
+
+// simpleEncode can be used for any type that doesn't have special keys which need handling differently,
+// or for types where special keys have already been handled.
+//
+// Beware, the 'type' key on the passed interface should already have been set, since simpleEncode won't know
+// what type it is! If you try to decode stuff you've encoded with a missing type key, you're going to have a bad time.
+func (e *exporter) simpleEncode(ctx context.Context, file *os.File, i interface{}, id string) error {
+	_, alreadyWritten := e.writtenIDs[id]
+	if alreadyWritten {
+		// this exporter has already exported an entry with this ID, no need to do it twice
+		return nil
+	}
+
+	err := json.NewEncoder(file).Encode(i)
+	if err != nil {
+		return fmt.Errorf("simpleEncode: error encoding entry with id %s: %s", id, err)
+	}
+
+	e.writtenIDs[id] = true
+	return nil
+}

internal/trans/export.go

@@ -0,0 +1,223 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/package trans
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func (e *exporter) exportAccounts(ctx context.Context, where []db.Where, file *os.File) ([]*transmodel.Account, error) {
+	// select using the 'where' we've been provided
+	accounts := []*transmodel.Account{}
+	if err := e.db.GetWhere(ctx, where, &accounts); err != nil {
+		return nil, fmt.Errorf("exportAccounts: error selecting accounts: %s", err)
+	}
+
+	// write any accounts found to file
+	for _, a := range accounts {
+		if err := e.accountEncode(ctx, file, a); err != nil {
+			return nil, fmt.Errorf("exportAccounts: error encoding account: %s", err)
+		}
+	}
+
+	return accounts, nil
+}
+
+func (e *exporter) exportBlocks(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Block, error) {
+	blocksUnique := make(map[string]*transmodel.Block)
+
+	// for each account we want to export both where it's blocking and where it's blocked
+	for _, a := range accounts {
+		// 1. export blocks owned by given account
+		whereBlocking := []db.Where{{Key: "account_id", Value: a.ID}}
+		blocking := []*transmodel.Block{}
+		if err := e.db.GetWhere(ctx, whereBlocking, &blocking); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error selecting blocks owned by account %s: %s", a.ID, err)
+		}
+		for _, b := range blocking {
+			b.Type = transmodel.TransBlock
+			if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
+				return nil, fmt.Errorf("exportBlocks: error encoding block owned by account %s: %s", a.ID, err)
+			}
+			blocksUnique[b.ID] = b
+		}
+
+		// 2. export blocks that target given account
+		whereBlocked := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		blocked := []*transmodel.Block{}
+		if err := e.db.GetWhere(ctx, whereBlocked, &blocked); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error selecting blocks targeting account %s: %s", a.ID, err)
+		}
+		for _, b := range blocked {
+			b.Type = transmodel.TransBlock
+			if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
+				return nil, fmt.Errorf("exportBlocks: error encoding block targeting account %s: %s", a.ID, err)
+			}
+			blocksUnique[b.ID] = b
+		}
+	}
+
+	// now return all the blocks we found
+	blocks := []*transmodel.Block{}
+	for _, b := range blocksUnique {
+		blocks = append(blocks, b)
+	}
+
+	return blocks, nil
+}
+
+func (e *exporter) exportDomainBlocks(ctx context.Context, file *os.File) ([]*transmodel.DomainBlock, error) {
+	domainBlocks := []*transmodel.DomainBlock{}
+
+	if err := e.db.GetAll(ctx, &domainBlocks); err != nil {
+		return nil, fmt.Errorf("exportBlocks: error selecting domain blocks: %s", err)
+	}
+
+	for _, b := range domainBlocks {
+		b.Type = transmodel.TransDomainBlock
+		if err := e.simpleEncode(ctx, file, b, b.ID); err != nil {
+			return nil, fmt.Errorf("exportBlocks: error encoding domain block: %s", err)
+		}
+	}
+
+	return domainBlocks, nil
+}
+
+func (e *exporter) exportFollows(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.Follow, error) {
+	followsUnique := make(map[string]*transmodel.Follow)
+
+	// for each account we want to export both where it's following and where it's followed
+	for _, a := range accounts {
+		// 1. export follows owned by given account
+		whereFollowing := []db.Where{{Key: "account_id", Value: a.ID}}
+		following := []*transmodel.Follow{}
+		if err := e.db.GetWhere(ctx, whereFollowing, &following); err != nil {
+			return nil, fmt.Errorf("exportFollows: error selecting follows owned by account %s: %s", a.ID, err)
+		}
+		for _, follow := range following {
+			follow.Type = transmodel.TransFollow
+			if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil {
+				return nil, fmt.Errorf("exportFollows: error encoding follow owned by account %s: %s", a.ID, err)
+			}
+			followsUnique[follow.ID] = follow
+		}
+
+		// 2. export follows that target given account
+		whereFollowed := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		followed := []*transmodel.Follow{}
+		if err := e.db.GetWhere(ctx, whereFollowed, &followed); err != nil {
+			return nil, fmt.Errorf("exportFollows: error selecting follows targeting account %s: %s", a.ID, err)
+		}
+		for _, follow := range followed {
+			follow.Type = transmodel.TransFollow
+			if err := e.simpleEncode(ctx, file, follow, follow.ID); err != nil {
+				return nil, fmt.Errorf("exportFollows: error encoding follow targeting account %s: %s", a.ID, err)
+			}
+			followsUnique[follow.ID] = follow
+		}
+	}
+
+	// now return all the follows we found
+	follows := []*transmodel.Follow{}
+	for _, follow := range followsUnique {
+		follows = append(follows, follow)
+	}
+
+	return follows, nil
+}
+
+func (e *exporter) exportFollowRequests(ctx context.Context, accounts []*transmodel.Account, file *os.File) ([]*transmodel.FollowRequest, error) {
+	frsUnique := make(map[string]*transmodel.FollowRequest)
+
+	// for each account we want to export both where it's following and where it's followed
+	for _, a := range accounts {
+		// 1. export follow requests owned by given account
+		whereRequesting := []db.Where{{Key: "account_id", Value: a.ID}}
+		requesting := []*transmodel.FollowRequest{}
+		if err := e.db.GetWhere(ctx, whereRequesting, &requesting); err != nil {
+			return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests owned by account %s: %s", a.ID, err)
+		}
+		for _, fr := range requesting {
+			fr.Type = transmodel.TransFollowRequest
+			if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil {
+				return nil, fmt.Errorf("exportFollowRequests: error encoding follow request owned by account %s: %s", a.ID, err)
+			}
+			frsUnique[fr.ID] = fr
+		}
+
+		// 2. export follow requests that target given account
+		whereRequested := []db.Where{{Key: "target_account_id", Value: a.ID}}
+		requested := []*transmodel.FollowRequest{}
+		if err := e.db.GetWhere(ctx, whereRequested, &requested); err != nil {
+			return nil, fmt.Errorf("exportFollowRequests: error selecting follow requests targeting account %s: %s", a.ID, err)
+		}
+		for _, fr := range requested {
+			fr.Type = transmodel.TransFollowRequest
+			if err := e.simpleEncode(ctx, file, fr, fr.ID); err != nil {
+				return nil, fmt.Errorf("exportFollowRequests: error encoding follow request targeting account %s: %s", a.ID, err)
+			}
+			frsUnique[fr.ID] = fr
+		}
+	}
+
+	// now return all the followRequests we found
+	followRequests := []*transmodel.FollowRequest{}
+	for _, fr := range frsUnique {
+		followRequests = append(followRequests, fr)
+	}
+
+	return followRequests, nil
+}
+
+func (e *exporter) exportInstances(ctx context.Context, file *os.File) ([]*transmodel.Instance, error) {
+	instances := []*transmodel.Instance{}
+
+	if err := e.db.GetAll(ctx, &instances); err != nil {
+		return nil, fmt.Errorf("exportInstances: error selecting instance: %s", err)
+	}
+
+	for _, u := range instances {
+		u.Type = transmodel.TransInstance
+		if err := e.simpleEncode(ctx, file, u, u.ID); err != nil {
+			return nil, fmt.Errorf("exportInstances: error encoding instance: %s", err)
+		}
+	}
+
+	return instances, nil
+}
+
+func (e *exporter) exportUsers(ctx context.Context, file *os.File) ([]*transmodel.User, error) {
+	users := []*transmodel.User{}
+
+	if err := e.db.GetAll(ctx, &users); err != nil {
+		return nil, fmt.Errorf("exportUsers: error selecting users: %s", err)
+	}
+
+	for _, u := range users {
+		u.Type = transmodel.TransUser
+		if err := e.simpleEncode(ctx, file, u, u.ID); err != nil {
+			return nil, fmt.Errorf("exportUsers: error encoding user: %s", err)
+		}
+	}
+
+	return users, nil
+}

internal/trans/exporter.go

@@ -0,0 +1,46 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+// Exporter wraps functionality for exporting entries from the database to a file.
+type Exporter interface {
+	ExportMinimal(ctx context.Context, path string) error
+}
+
+type exporter struct {
+	db         db.DB
+	log        *logrus.Logger
+	writtenIDs map[string]bool
+}
+
+// NewExporter returns a new Exporter that will use the given db and logger.
+func NewExporter(db db.DB, log *logrus.Logger) Exporter {
+	return &exporter{
+		db:         db,
+		log:        log,
+		writtenIDs: make(map[string]bool),
+	}
+}

internal/trans/exportminimal.go

@@ -0,0 +1,150 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+func (e *exporter) ExportMinimal(ctx context.Context, path string) error {
+	if path == "" {
+		return errors.New("ExportMinimal: path empty")
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: couldn't export to %s: %s", path, err)
+	}
+
+	// export all local accounts we have in the database
+	localAccounts, err := e.exportAccounts(ctx, []db.Where{{Key: "domain", Value: nil}}, file)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting accounts: %s", err)
+	}
+
+	// export all blocks that relate to local accounts
+	blocks, err := e.exportBlocks(ctx, localAccounts, file)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting blocks: %s", err)
+	}
+
+	// for each block, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
+	for _, b := range blocks {
+		_, alreadyWritten := e.writtenIDs[b.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.AccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting block owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[b.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: b.TargetAccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting block target account: %s", err)
+			}
+		}
+	}
+
+	// export all follows that relate to local accounts
+	follows, err := e.exportFollows(ctx, localAccounts, file)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting follows: %s", err)
+	}
+
+	// for each follow, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
+	for _, follow := range follows {
+		_, alreadyWritten := e.writtenIDs[follow.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.AccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[follow.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: follow.TargetAccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow target account: %s", err)
+			}
+		}
+	}
+
+	// export all follow requests that relate to local accounts
+	followRequests, err := e.exportFollowRequests(ctx, localAccounts, file)
+	if err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting follow requests: %s", err)
+	}
+
+	// for each follow request, make sure we've written out the account owning it, or targeted by it --
+	// this might include non-local accounts, but we need these so we don't lose anything
+	for _, fr := range followRequests {
+		_, alreadyWritten := e.writtenIDs[fr.AccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.AccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow request owner account: %s", err)
+			}
+		}
+
+		_, alreadyWritten = e.writtenIDs[fr.TargetAccountID]
+		if !alreadyWritten {
+			_, err := e.exportAccounts(ctx, []db.Where{{Key: "id", Value: fr.TargetAccountID}}, file)
+			if err != nil {
+				return fmt.Errorf("ExportMinimal: error exporting follow request target account: %s", err)
+			}
+		}
+	}
+
+	// export all domain blocks
+	if _, err := e.exportDomainBlocks(ctx, file); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting domain blocks: %s", err)
+	}
+
+	// export all users
+	if _, err := e.exportUsers(ctx, file); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting users: %s", err)
+	}
+
+	// export all instances
+	if _, err := e.exportInstances(ctx, file); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting instances: %s", err)
+	}
+
+	// export all SUSPENDED accounts to make sure the suspension sticks across db migration etc
+	whereSuspended := []db.Where{{
+		Key:   "suspended_at",
+		Not:   true,
+		Value: nil,
+	}}
+	if _, err := e.exportAccounts(ctx, whereSuspended, file); err != nil {
+		return fmt.Errorf("ExportMinimal: error exporting suspended accounts: %s", err)
+	}
+
+	return neatClose(file)
+}

internal/trans/exportminimal_test.go

@@ -0,0 +1,54 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+)
+
+type ExportMinimalTestSuite struct {
+	TransTestSuite
+}
+
+func (suite *ExportMinimalTestSuite) TestExportMinimalOK() {
+	// use a temporary file path that will be cleaned when the test is closed
+	tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString())
+
+	// export to the tempFilePath
+	exporter := trans.NewExporter(suite.db, suite.log)
+	err := exporter.ExportMinimal(context.Background(), tempFilePath)
+	suite.NoError(err)
+
+	// we should have some bytes in that file now
+	b, err := os.ReadFile(tempFilePath)
+	suite.NoError(err)
+	suite.NotEmpty(b)
+	fmt.Println(string(b))
+}
+
+func TestExportMinimalTestSuite(t *testing.T) {
+	suite.Run(t, &ExportMinimalTestSuite{})
+}

internal/trans/import.go

@@ -0,0 +1,146 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	transmodel "github.com/superseriousbusiness/gotosocial/internal/trans/model"
+)
+
+func (i *importer) Import(ctx context.Context, path string) error {
+	if path == "" {
+		return errors.New("Export: path empty")
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("Import: couldn't export to %s: %s", path, err)
+	}
+
+	decoder := json.NewDecoder(file)
+	decoder.UseNumber()
+
+	for {
+		entry := transmodel.Entry{}
+		err := decoder.Decode(&entry)
+		if err != nil {
+			if err == io.EOF {
+				i.log.Infof("Import: reached end of file")
+				return neatClose(file)
+			}
+			return fmt.Errorf("Import: error decoding in readLoop: %s", err)
+		}
+		if err := i.inputEntry(ctx, entry); err != nil {
+			return fmt.Errorf("Import: error inputting entry: %s", err)
+		}
+	}
+}
+
+func (i *importer) inputEntry(ctx context.Context, entry transmodel.Entry) error {
+	t, ok := entry[transmodel.TypeKey].(string)
+	if !ok {
+		return errors.New("inputEntry: could not derive entry type: missing or malformed 'type' key in json")
+	}
+
+	switch transmodel.Type(t) {
+	case transmodel.TransAccount:
+		account, err := i.accountDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into account: %s", err)
+		}
+		if err := i.putInDB(ctx, account); err != nil {
+			return fmt.Errorf("inputEntry: error adding account to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added account with id %s", account.ID)
+		return nil
+	case transmodel.TransBlock:
+		block, err := i.blockDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into block: %s", err)
+		}
+		if err := i.putInDB(ctx, block); err != nil {
+			return fmt.Errorf("inputEntry: error adding block to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added block with id %s", block.ID)
+		return nil
+	case transmodel.TransDomainBlock:
+		block, err := i.domainBlockDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into domain block: %s", err)
+		}
+		if err := i.putInDB(ctx, block); err != nil {
+			return fmt.Errorf("inputEntry: error adding domain block to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added domain block with id %s", block.ID)
+		return nil
+	case transmodel.TransFollow:
+		follow, err := i.followDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into follow: %s", err)
+		}
+		if err := i.putInDB(ctx, follow); err != nil {
+			return fmt.Errorf("inputEntry: error adding follow to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added follow with id %s", follow.ID)
+		return nil
+	case transmodel.TransFollowRequest:
+		fr, err := i.followRequestDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into follow request: %s", err)
+		}
+		if err := i.putInDB(ctx, fr); err != nil {
+			return fmt.Errorf("inputEntry: error adding follow request to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added follow request with id %s", fr.ID)
+		return nil
+	case transmodel.TransInstance:
+		inst, err := i.instanceDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into instance: %s", err)
+		}
+		if err := i.putInDB(ctx, inst); err != nil {
+			return fmt.Errorf("inputEntry: error adding instance to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added instance with id %s", inst.ID)
+		return nil
+	case transmodel.TransUser:
+		user, err := i.userDecode(entry)
+		if err != nil {
+			return fmt.Errorf("inputEntry: error decoding entry into user: %s", err)
+		}
+		if err := i.putInDB(ctx, user); err != nil {
+			return fmt.Errorf("inputEntry: error adding user to database: %s", err)
+		}
+		i.log.Infof("inputEntry: added user with id %s", user.ID)
+		return nil
+	}
+
+	i.log.Errorf("inputEntry: didn't recognize transtype '%s', skipping it", t)
+	return nil
+}
+
+func (i *importer) putInDB(ctx context.Context, entry interface{}) error {
+	return i.db.Put(ctx, entry)
+}

internal/trans/import_test.go

@@ -0,0 +1,91 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/trans"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type ImportMinimalTestSuite struct {
+	TransTestSuite
+}
+
+func (suite *ImportMinimalTestSuite) TestImportMinimalOK() {
+	ctx := context.Background()
+
+	// use a temporary file path
+	tempFilePath := fmt.Sprintf("%s/%s", suite.T().TempDir(), uuid.NewString())
+
+	// export to the tempFilePath
+	exporter := trans.NewExporter(suite.db, suite.log)
+	err := exporter.ExportMinimal(ctx, tempFilePath)
+	suite.NoError(err)
+
+	// we should have some bytes in that file now
+	b, err := os.ReadFile(tempFilePath)
+	suite.NoError(err)
+	suite.NotEmpty(b)
+	fmt.Println(string(b))
+
+	// create a new database with just the tables created, no entries
+	testrig.StandardDBTeardown(suite.db)
+	newDB := testrig.NewTestDB()
+	testrig.CreateTestTables(newDB)
+
+	importer := trans.NewImporter(newDB, suite.log)
+	err = importer.Import(ctx, tempFilePath)
+	suite.NoError(err)
+
+	// we should have some accounts in the database
+	accounts := []*gtsmodel.Account{}
+	err = newDB.GetAll(ctx, &accounts)
+	suite.NoError(err)
+	suite.NotEmpty(accounts)
+
+	// we should have some blocks in the database
+	blocks := []*gtsmodel.Block{}
+	err = newDB.GetAll(ctx, &blocks)
+	suite.NoError(err)
+	suite.NotEmpty(blocks)
+
+	// we should have some follows in the database
+	follows := []*gtsmodel.Follow{}
+	err = newDB.GetAll(ctx, &follows)
+	suite.NoError(err)
+	suite.NotEmpty(follows)
+
+	// we should have some domain blocks in the database
+	domainBlocks := []*gtsmodel.DomainBlock{}
+	err = newDB.GetAll(ctx, &domainBlocks)
+	suite.NoError(err)
+	suite.NotEmpty(domainBlocks)
+}
+
+func TestImportMinimalTestSuite(t *testing.T) {
+	suite.Run(t, &ImportMinimalTestSuite{})
+}

internal/trans/importer.go

@@ -0,0 +1,44 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"context"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+)
+
+// Importer wraps functionality for importing entries from a file into the database.
+type Importer interface {
+	Import(ctx context.Context, path string) error
+}
+
+type importer struct {
+	db  db.DB
+	log *logrus.Logger
+}
+
+// NewImporter returns a new Importer interface that uses the given db and logger.
+func NewImporter(db db.DB, log *logrus.Logger) Importer {
+	return &importer{
+		db:  db,
+		log: log,
+	}
+}

internal/trans/model/account.go

@@ -0,0 +1,54 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"crypto/rsa"
+	"time"
+)
+
+// Account represents the minimum viable representation of an account for export/import.
+type Account struct {
+	Type                  Type            `json:"type" bun:"-"`
+	ID                    string          `json:"id" bun:",nullzero"`
+	CreatedAt             *time.Time      `json:"createdAt" bun:",nullzero"`
+	Username              string          `json:"username" bun:",nullzero"`
+	DisplayName           string          `json:"displayName,omitempty" bun:",nullzero"`
+	Note                  string          `json:"note,omitempty" bun:",nullzero"`
+	Domain                string          `json:"domain,omitempty" bun:",nullzero"`
+	HeaderRemoteURL       string          `json:"headerRemoteURL,omitempty" bun:",nullzero"`
+	AvatarRemoteURL       string          `json:"avatarRemoteURL,omitempty" bun:",nullzero"`
+	Locked                bool            `json:"locked"`
+	Language              string          `json:"language,omitempty" bun:",nullzero"`
+	URI                   string          `json:"uri" bun:",nullzero"`
+	URL                   string          `json:"url" bun:",nullzero"`
+	InboxURI              string          `json:"inboxURI" bun:",nullzero"`
+	OutboxURI             string          `json:"outboxURI" bun:",nullzero"`
+	FollowingURI          string          `json:"followingUri" bun:",nullzero"`
+	FollowersURI          string          `json:"followersUri" bun:",nullzero"`
+	FeaturedCollectionURI string          `json:"featuredCollectionUri" bun:",nullzero"`
+	ActorType             string          `json:"actorType" bun:",nullzero"`
+	PrivateKey            *rsa.PrivateKey `json:"-" mapstructure:"-"`
+	PrivateKeyString      string          `json:"privateKey,omitempty" mapstructure:"privateKey" bun:"-"`
+	PublicKey             *rsa.PublicKey  `json:"-" mapstructure:"-"`
+	PublicKeyString       string          `json:"publicKey,omitempty" mapstructure:"publicKey" bun:"-"`
+	PublicKeyURI          string          `json:"publicKeyUri" bun:",nullzero"`
+	SuspendedAt           *time.Time      `json:"suspendedAt,omitempty" bun:",nullzero"`
+	SuspensionOrigin      string          `json:"suspensionOrigin,omitempty" bun:",nullzero"`
+}

internal/trans/model/block.go

@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+// Block represents an account block as serialized in an exported file.
+type Block struct {
+	Type            Type       `json:"type" bun:"-"`
+	ID              string     `json:"id" bun:",nullzero"`
+	CreatedAt       *time.Time `json:"createdAt" bun:",nullzero"`
+	URI             string     `json:"uri" bun:",nullzero"`
+	AccountID       string     `json:"accountId" bun:",nullzero"`
+	TargetAccountID string     `json:"targetAccountId" bun:",nullzero"`
+}

internal/trans/model/domainblock.go

@@ -0,0 +1,34 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+// DomainBlock represents a domain block as serialized in an exported file.
+type DomainBlock struct {
+	Type               Type       `json:"type" bun:"-"`
+	ID                 string     `json:"id" bun:",nullzero"`
+	CreatedAt          *time.Time `json:"createdAt" bun:",nullzero"`
+	Domain             string     `json:"domain" bun:",nullzero"`
+	CreatedByAccountID string     `json:"createdByAccountID" bun:",nullzero"`
+	PrivateComment     string     `json:"privateComment,omitempty" bun:",nullzero"`
+	PublicComment      string     `json:"publicComment,omitempty" bun:",nullzero"`
+	Obfuscate          bool       `json:"obfuscate" bun:",nullzero"`
+	SubscriptionID     string     `json:"subscriptionID,omitempty" bun:",nullzero"`
+}

internal/trans/model/follow.go

@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+// Follow represents an account follow as serialized in an export file.
+type Follow struct {
+	Type            Type       `json:"type" bun:"-"`
+	ID              string     `json:"id" bun:",nullzero"`
+	CreatedAt       *time.Time `json:"createdAt" bun:",nullzero"`
+	URI             string     `json:"uri" bun:",nullzero"`
+	AccountID       string     `json:"accountId" bun:",nullzero"`
+	TargetAccountID string     `json:"targetAccountId" bun:",nullzero"`
+}

internal/trans/model/followrequest.go

@@ -0,0 +1,31 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import "time"
+
+// FollowRequest represents an account follow request as serialized in an export file.
+type FollowRequest struct {
+	Type            Type       `json:"type" bun:"-"`
+	ID              string     `json:"id" bun:",nullzero"`
+	CreatedAt       *time.Time `json:"createdAt" bun:",nullzero"`
+	URI             string     `json:"uri" bun:",nullzero"`
+	AccountID       string     `json:"accountId" bun:",nullzero"`
+	TargetAccountID string     `json:"targetAccountId" bun:",nullzero"`
+}

internal/trans/model/instance.go

@@ -0,0 +1,43 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"time"
+)
+
+// Instance represents an instance entry as serialized in an export file.
+type Instance struct {
+	Type                   Type       `json:"type" bun:"-"`
+	ID                     string     `json:"id" bun:",nullzero"`
+	CreatedAt              *time.Time `json:"createdAt" bun:",nullzero"`
+	Domain                 string     `json:"domain" bun:",nullzero"`
+	Title                  string     `json:"title,omitempty" bun:",nullzero"`
+	URI                    string     `json:"uri" bun:",nullzero"`
+	SuspendedAt            *time.Time `json:"suspendedAt,omitempty" bun:",nullzero"`
+	DomainBlockID          string     `json:"domainBlockID,omitempty" bun:",nullzero"`
+	ShortDescription       string     `json:"shortDescription,omitempty" bun:",nullzero"`
+	Description            string     `json:"description,omitempty" bun:",nullzero"`
+	Terms                  string     `json:"terms,omitempty" bun:",nullzero"`
+	ContactEmail           string     `json:"contactEmail,omitempty" bun:",nullzero"`
+	ContactAccountUsername string     `json:"contactAccountUsername,omitempty" bun:",nullzero"`
+	ContactAccountID       string     `json:"contactAccountID,omitempty" bun:",nullzero"`
+	Reputation             int64      `json:"reputation"`
+	Version                string     `json:"version,omitempty" bun:",nullzero"`
+}

internal/trans/model/type.go

@@ -0,0 +1,41 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+// TypeKey should be set on a TransEntry to indicate the type of entry it is.
+const TypeKey = "type"
+
+// Type describes the type of a trans entry, and how it should be read/serialized.
+type Type string
+
+// Type of the trans entry. Describes how it should be read from file.
+const (
+	TransAccount          Type = "account"
+	TransBlock            Type = "block"
+	TransDomainBlock      Type = "domainBlock"
+	TransEmailDomainBlock Type = "emailDomainBlock"
+	TransFollow           Type = "follow"
+	TransFollowRequest    Type = "followRequest"
+	TransInstance         Type = "instance"
+	TransUser             Type = "user"
+)
+
+// Entry is used for deserializing trans entries into a rough interface so that
+// the TypeKey can be fetched, before continuing with full parsing.
+type Entry map[string]interface{}

internal/trans/model/user.go

@@ -0,0 +1,50 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"time"
+)
+
+// User represents a local instance user as serialized to an export file.
+type User struct {
+	Type                Type       `json:"type" bun:"-"`
+	ID                  string     `json:"id" bun:",nullzero"`
+	CreatedAt           *time.Time `json:"createdAt" bun:",nullzero"`
+	Email               string     `json:"email,omitempty" bun:",nullzero"`
+	AccountID           string     `json:"accountID" bun:",nullzero"`
+	EncryptedPassword   string     `json:"encryptedPassword" bun:",nullzero"`
+	CurrentSignInAt     *time.Time `json:"currentSignInAt,omitempty" bun:",nullzero"`
+	LastSignInAt        *time.Time `json:"lastSignInAt,omitempty" bun:",nullzero"`
+	InviteID            string     `json:"inviteID,omitempty" bun:",nullzero"`
+	ChosenLanguages     []string   `json:"chosenLanguages,omitempty" bun:",nullzero"`
+	FilteredLanguages   []string   `json:"filteredLanguage,omitempty" bun:",nullzero"`
+	Locale              string     `json:"locale" bun:",nullzero"`
+	LastEmailedAt       time.Time  `json:"lastEmailedAt,omitempty" bun:",nullzero"`
+	ConfirmationToken   string     `json:"confirmationToken,omitempty" bun:",nullzero"`
+	ConfirmationSentAt  *time.Time `json:"confirmationTokenSentAt,omitempty" bun:",nullzero"`
+	ConfirmedAt         *time.Time `json:"confirmedAt,omitempty" bun:",nullzero"`
+	UnconfirmedEmail    string     `json:"unconfirmedEmail,omitempty" bun:",nullzero"`
+	Moderator           bool       `json:"moderator"`
+	Admin               bool       `json:"admin"`
+	Disabled            bool       `json:"disabled"`
+	Approved            bool       `json:"approved"`
+	ResetPasswordToken  string     `json:"resetPasswordToken,omitempty" bun:",nullzero"`
+	ResetPasswordSentAt *time.Time `json:"resetPasswordSentAt,omitempty" bun:",nullzero"`
+}

internal/trans/trans_test.go

@@ -0,0 +1,42 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans_test
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type TransTestSuite struct {
+	suite.Suite
+	db  db.DB
+	log *logrus.Logger
+}
+
+func (suite *TransTestSuite) SetupTest() {
+	suite.db = testrig.NewTestDB()
+	suite.log = testrig.NewTestLog()
+	testrig.StandardDBSetup(suite.db, nil)
+}
+
+func (suite *TransTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+}

internal/trans/util.go

@@ -0,0 +1,32 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package trans
+
+import (
+	"fmt"
+	"os"
+)
+
+func neatClose(f *os.File) error {
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("error closing file: %s", err)
+	}
+
+	return nil
+}

internal/typeutils/astointernal.go

@@ -325,6 +325,11 @@ func (c *converter) ASStatusToStatus(ctx context.Context, statusable ap.Statusab
 
 	// advanced visibility for this status
 	// TODO: a lot of work to be done here -- a new type needs to be created for this in go-fed/activity using ASTOOL
+	// for now we just set everything to true
+	status.Federated = true
+	status.Boostable = true
+	status.Replyable = true
+	status.Likeable = true
 
 	// sensitive
 	// TODO: this is a bool

internal/typeutils/internal.go

@@ -70,7 +70,10 @@ func (c *converter) StatusToBoost(ctx context.Context, s *gtsmodel.Status, boost
 		BoostOfID:           s.ID,
 		BoostOfAccountID:    s.AccountID,
 		Visibility:          s.Visibility,
-		VisibilityAdvanced:  s.VisibilityAdvanced,
+		Federated:           s.Federated,
+		Boostable:           s.Boostable,
+		Replyable:           s.Replyable,
+		Likeable:            s.Likeable,
 
 		// attach these here for convenience -- the boosted status/account won't go in the DB
 		// but they're needed in the processor and for the frontend. Since we have them, we can

internal/validate/status_test.go

@@ -63,15 +63,13 @@ func happyStatus() *gtsmodel.Status {
 		Language:                 "en",
 		CreatedWithApplicationID: "01FEBBZHF4GFVRXSJVXD0JTZZ2",
 		CreatedWithApplication:   nil,
-		VisibilityAdvanced: gtsmodel.VisibilityAdvanced{
-			Federated: true,
-			Boostable: true,
-			Replyable: true,
-			Likeable:  true,
-		},
-		ActivityStreamsType: ap.ObjectNote,
-		Text:                "Test status! #hello",
-		Pinned:              false,
+		Federated:                true,
+		Boostable:                true,
+		Replyable:                true,
+		Likeable:                 true,
+		ActivityStreamsType:      ap.ObjectNote,
+		Text:                     "Test status! #hello",
+		Pinned:                   false,
 	}
 }