[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

vendor/modernc.org/sqlite/sqlite.go

@@ -3,8 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:generate go run generator.go
-//go:generate gofmt -l -s -w .
-//go:generate gofmt -l -s -w .
 
 package sqlite // import "modernc.org/sqlite"
 
@@ -687,7 +685,13 @@ type tx struct {
 
 func newTx(c *conn) (*tx, error) {
 	r := &tx{c: c}
-	if err := r.exec(context.Background(), "begin"); err != nil {
+	var sql string
+	if c.beginMode != "" {
+		sql = "begin " + c.beginMode
+	} else {
+		sql = "begin"
+	}
+	if err := r.exec(context.Background(), sql); err != nil {
 		return nil, err
 	}
 
@@ -745,6 +749,7 @@ type conn struct {
 	sync.Mutex
 
 	writeTimeFormat string
+	beginMode       string
 }
 
 func newConn(dsn string) (*conn, error) {
@@ -809,6 +814,14 @@ func applyQueryParams(c *conn, query string) error {
 		return nil
 	}
 
+	if v := q.Get("_txlock"); v != "" {
+		lower := strings.ToLower(v)
+		if lower != "deferred" && lower != "immediate" && lower != "exclusive" {
+			return fmt.Errorf("unknown _txlock %q", v)
+		}
+		c.beginMode = v
+	}
+
 	return nil
 }
 
@@ -1388,6 +1401,27 @@ func newDriver() *Driver { return &Driver{} }
 // efficient re-use.
 //
 // The returned connection is only used by one goroutine at a time.
+//
+// If name contains a '?', what follows is treated as a query string. This
+// driver supports the following query parameters:
+//
+// _pragma: Each value will be run as a "PRAGMA ..." statement (with the PRAGMA
+// keyword added for you). May be specified more than once. Example:
+// "_pragma=foreign_keys(1)" will enable foreign key enforcement. More
+// information on supported PRAGMAs is available from the SQLite documentation:
+// https://www.sqlite.org/pragma.html
+//
+// _time_format: The name of a format to use when writing time values to the
+// database. Currently the only supported value is "sqlite", which corresponds
+// to format 7 from https://www.sqlite.org/lang_datefunc.html#time_values,
+// including the timezone specifier. If this parameter is not specified, then
+// the default String() format will be used.
+//
+// _txlock: The locking behavior to use when beginning a transaction. May be
+// "deferred", "immediate", or "exclusive" (case insensitive). The default is to
+// not specify one, which SQLite maps to "deferred". More information is
+// available at
+// https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions
 func (d *Driver) Open(name string) (driver.Conn, error) {
 	return newConn(name)
 }