Database updates (#144)

* start moving some database stuff around

* continue moving db stuff around

* more fiddling

* more updates

* and some more

* and yet more

* i broke SOMETHING but what, it's a mystery

* tidy up

* vendor ttlcache

* use ttlcache

* fix up some tests

* rename some stuff

* little reminder

* some more updates

internal/federation/federatingdb/accept.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (

internal/federation/federatingdb/announce.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (

internal/federation/federatingdb/create.go

@@ -112,8 +112,8 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error {
 				}
 				status.ID = statusID
 
-				if err := f.db.Put(status); err != nil {
-					if _, ok := err.(db.ErrAlreadyExists); ok {
+				if err := f.db.PutStatus(status); err != nil {
+					if err == db.ErrAlreadyExists {
 						// the status already exists in the database, which means we've already handled everything else,
 						// so we can just return nil here and be done with it.
 						return nil

internal/federation/federatingdb/delete.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (
@@ -6,7 +24,6 @@ import (
 	"net/url"
 
 	"github.com/sirupsen/logrus"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
@@ -52,10 +69,8 @@ func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error {
 
 	// in a delete we only get the URI, we can't know if we have a status or a profile or something else,
 	// so we have to try a few different things...
-	where := []db.Where{{Key: "uri", Value: id.String()}}
-
-	s := &gtsmodel.Status{}
-	if err := f.db.GetWhere(where, s); err == nil {
+	s, err := f.db.GetStatusByURI(id.String())
+	if err == nil {
 		// it's a status
 		l.Debugf("uri is for status with id: %s", s.ID)
 		if err := f.db.DeleteByID(s.ID, &gtsmodel.Status{}); err != nil {
@@ -69,8 +84,8 @@ func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error {
 		}
 	}
 
-	a := &gtsmodel.Account{}
-	if err := f.db.GetWhere(where, a); err == nil {
+	a, err := f.db.GetAccountByURI(id.String())
+	if err == nil {
 		// it's an account
 		l.Debugf("uri is for an account with id: %s", s.ID)
 		if err := f.db.DeleteByID(a.ID, &gtsmodel.Account{}); err != nil {

internal/federation/federatingdb/exists.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (

internal/federation/federatingdb/followers.go

@@ -31,7 +31,8 @@ func (f *federatingDB) Followers(c context.Context, actorIRI *url.URL) (follower
 	acct := &gtsmodel.Account{}
 
 	if util.IsUserPath(actorIRI) {
-		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
+		acct, err = f.db.GetAccountByURI(actorIRI.String())
+		if err != nil {
 			return nil, fmt.Errorf("FOLLOWERS: db error getting account with uri %s: %s", actorIRI.String(), err)
 		}
 	} else if util.IsFollowersPath(actorIRI) {
@@ -42,8 +43,8 @@ func (f *federatingDB) Followers(c context.Context, actorIRI *url.URL) (follower
 		return nil, fmt.Errorf("FOLLOWERS: could not parse actor IRI %s as users or followers path", actorIRI.String())
 	}
 
-	acctFollowers := []gtsmodel.Follow{}
-	if err := f.db.GetFollowersByAccountID(acct.ID, &acctFollowers, false); err != nil {
+	acctFollowers, err := f.db.GetAccountFollowedBy(acct.ID, false)
+	if err != nil {
 		return nil, fmt.Errorf("FOLLOWERS: db error getting followers for account id %s: %s", acct.ID, err)
 	}
 

internal/federation/federatingdb/following.go

@@ -8,7 +8,6 @@ import (
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/sirupsen/logrus"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
@@ -28,21 +27,37 @@ func (f *federatingDB) Following(c context.Context, actorIRI *url.URL) (followin
 	)
 	l.Debugf("entering FOLLOWING function with actorIRI %s", actorIRI.String())
 
-	acct := &gtsmodel.Account{}
+	var acct *gtsmodel.Account
 	if util.IsUserPath(actorIRI) {
-		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil {
+		username, err := util.ParseUserPath(actorIRI)
+		if err != nil {
+			return nil, fmt.Errorf("FOLLOWING: error parsing user path: %s", err)
+		}
+
+		a, err := f.db.GetLocalAccountByUsername(username)
+		if err != nil {
 			return nil, fmt.Errorf("FOLLOWING: db error getting account with uri %s: %s", actorIRI.String(), err)
 		}
+
+		acct = a
 	} else if util.IsFollowingPath(actorIRI) {
-		if err := f.db.GetWhere([]db.Where{{Key: "following_uri", Value: actorIRI.String()}}, acct); err != nil {
+		username, err := util.ParseFollowingPath(actorIRI)
+		if err != nil {
+			return nil, fmt.Errorf("FOLLOWING: error parsing following path: %s", err)
+		}
+
+		a, err := f.db.GetLocalAccountByUsername(username)
+		if err != nil {
 			return nil, fmt.Errorf("FOLLOWING: db error getting account with following uri %s: %s", actorIRI.String(), err)
 		}
+
+		acct = a
 	} else {
 		return nil, fmt.Errorf("FOLLOWING: could not parse actor IRI %s as users or following path", actorIRI.String())
 	}
 
-	acctFollowing := []gtsmodel.Follow{}
-	if err := f.db.GetFollowingByAccountID(acct.ID, &acctFollowing); err != nil {
+	acctFollowing, err := f.db.GetAccountFollows(acct.ID)
+	if err != nil {
 		return nil, fmt.Errorf("FOLLOWING: db error getting following for account id %s: %s", acct.ID, err)
 	}
 

internal/federation/federatingdb/get.go

@@ -43,8 +43,8 @@ func (f *federatingDB) Get(c context.Context, id *url.URL) (value vocab.Type, er
 	l.Debug("entering GET function")
 
 	if util.IsUserPath(id) {
-		acct := &gtsmodel.Account{}
-		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: id.String()}}, acct); err != nil {
+		acct, err := f.db.GetAccountByURI(id.String())
+		if err != nil {
 			return nil, err
 		}
 		l.Debug("is user path! returning account")

internal/federation/federatingdb/inbox.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (

internal/federation/federatingdb/liked.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (

internal/federation/federatingdb/outbox.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (
@@ -62,7 +80,7 @@ func (f *federatingDB) OutboxForInbox(c context.Context, inboxIRI *url.URL) (out
 	}
 	acct := &gtsmodel.Account{}
 	if err := f.db.GetWhere([]db.Where{{Key: "inbox_uri", Value: inboxIRI.String()}}, acct); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
+		if err == db.ErrNoEntries {
 			return nil, fmt.Errorf("no actor found that corresponds to inbox %s", inboxIRI.String())
 		}
 		return nil, fmt.Errorf("db error searching for actor with inbox %s", inboxIRI.String())

internal/federation/federatingdb/owns.go

@@ -54,16 +54,16 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 		if err != nil {
 			return false, fmt.Errorf("error parsing statuses path for url %s: %s", id.String(), err)
 		}
-		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uid}}, &gtsmodel.Status{}); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
+		status, err := f.db.GetStatusByURI(uid)
+		if err != nil {
+			if err == db.ErrNoEntries {
 				// there are no entries for this status
 				return false, nil
 			}
 			// an actual error happened
 			return false, fmt.Errorf("database error fetching status with id %s: %s", uid, err)
 		}
-		l.Debugf("we own url %s", id.String())
-		return true, nil
+		return status.Local, nil
 	}
 
 	if util.IsUserPath(id) {
@@ -71,8 +71,8 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 		if err != nil {
 			return false, fmt.Errorf("error parsing statuses path for url %s: %s", id.String(), err)
 		}
-		if err := f.db.GetLocalAccountByUsername(username, &gtsmodel.Account{}); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
+		if _, err := f.db.GetLocalAccountByUsername(username); err != nil {
+			if err == db.ErrNoEntries {
 				// there are no entries for this username
 				return false, nil
 			}
@@ -88,8 +88,8 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 		if err != nil {
 			return false, fmt.Errorf("error parsing statuses path for url %s: %s", id.String(), err)
 		}
-		if err := f.db.GetLocalAccountByUsername(username, &gtsmodel.Account{}); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
+		if _, err := f.db.GetLocalAccountByUsername(username); err != nil {
+			if err == db.ErrNoEntries {
 				// there are no entries for this username
 				return false, nil
 			}
@@ -105,8 +105,8 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 		if err != nil {
 			return false, fmt.Errorf("error parsing statuses path for url %s: %s", id.String(), err)
 		}
-		if err := f.db.GetLocalAccountByUsername(username, &gtsmodel.Account{}); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
+		if _, err := f.db.GetLocalAccountByUsername(username); err != nil {
+			if err == db.ErrNoEntries {
 				// there are no entries for this username
 				return false, nil
 			}
@@ -122,8 +122,8 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 		if err != nil {
 			return false, fmt.Errorf("error parsing like path for url %s: %s", id.String(), err)
 		}
-		if err := f.db.GetLocalAccountByUsername(username, &gtsmodel.Account{}); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
+		if _, err := f.db.GetLocalAccountByUsername(username); err != nil {
+			if err == db.ErrNoEntries {
 				// there are no entries for this username
 				return false, nil
 			}
@@ -131,7 +131,7 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 			return false, fmt.Errorf("database error fetching account with username %s: %s", username, err)
 		}
 		if err := f.db.GetByID(likeID, &gtsmodel.StatusFave{}); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
+			if err == db.ErrNoEntries {
 				// there are no entries
 				return false, nil
 			}
@@ -147,8 +147,8 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 		if err != nil {
 			return false, fmt.Errorf("error parsing block path for url %s: %s", id.String(), err)
 		}
-		if err := f.db.GetLocalAccountByUsername(username, &gtsmodel.Account{}); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
+		if _, err := f.db.GetLocalAccountByUsername(username); err != nil {
+			if err == db.ErrNoEntries {
 				// there are no entries for this username
 				return false, nil
 			}
@@ -156,7 +156,7 @@ func (f *federatingDB) Owns(c context.Context, id *url.URL) (bool, error) {
 			return false, fmt.Errorf("database error fetching account with username %s: %s", username, err)
 		}
 		if err := f.db.GetByID(blockID, &gtsmodel.Block{}); err != nil {
-			if _, ok := err.(db.ErrNoEntries); ok {
+			if err == db.ErrNoEntries {
 				// there are no entries
 				return false, nil
 			}

internal/federation/federatingdb/undo.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (

internal/federation/federatingdb/update.go

@@ -1,3 +1,21 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 package federatingdb
 
 import (

internal/federation/federatingdb/util.go

@@ -97,8 +97,8 @@ func (f *federatingDB) NewID(c context.Context, t vocab.Type) (idURL *url.URL, e
 			for iter := actorProp.Begin(); iter != actorProp.End(); iter = iter.Next() {
 				// take the IRI of the first actor we can find (there should only be one)
 				if iter.IsIRI() {
-					actorAccount := &gtsmodel.Account{}
-					if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: iter.GetIRI().String()}}, actorAccount); err == nil { // if there's an error here, just use the fallback behavior -- we don't need to return an error here
+					// if there's an error here, just use the fallback behavior -- we don't need to return an error here
+					if actorAccount, err := f.db.GetAccountByURI(iter.GetIRI().String()); err == nil {
 						newID, err := id.NewRandomULID()
 						if err != nil {
 							return nil, err
@@ -213,7 +213,7 @@ func (f *federatingDB) ActorForOutbox(c context.Context, outboxIRI *url.URL) (ac
 	}
 	acct := &gtsmodel.Account{}
 	if err := f.db.GetWhere([]db.Where{{Key: "outbox_uri", Value: outboxIRI.String()}}, acct); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
+		if err == db.ErrNoEntries {
 			return nil, fmt.Errorf("no actor found that corresponds to outbox %s", outboxIRI.String())
 		}
 		return nil, fmt.Errorf("db error searching for actor with outbox %s", outboxIRI.String())
@@ -238,7 +238,7 @@ func (f *federatingDB) ActorForInbox(c context.Context, inboxIRI *url.URL) (acto
 	}
 	acct := &gtsmodel.Account{}
 	if err := f.db.GetWhere([]db.Where{{Key: "inbox_uri", Value: inboxIRI.String()}}, acct); err != nil {
-		if _, ok := err.(db.ErrNoEntries); ok {
+		if err == db.ErrNoEntries {
 			return nil, fmt.Errorf("no actor found that corresponds to inbox %s", inboxIRI.String())
 		}
 		return nil, fmt.Errorf("db error searching for actor with inbox %s", inboxIRI.String())