[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

internal/config/defaults.go

@@ -53,6 +53,7 @@ var Defaults = Values{
 	MediaVideoMaxSize:        10485760, // 10mb
 	MediaDescriptionMinChars: 0,
 	MediaDescriptionMaxChars: 500,
+	MediaRemoteCacheDays:     30,
 
 	StorageBackend:       "local",
 	StorageLocalBasePath: "/gotosocial/storage",

internal/config/keys.go

@@ -58,6 +58,7 @@ type KeyNames struct {
 	MediaVideoMaxSize        string
 	MediaDescriptionMinChars string
 	MediaDescriptionMaxChars string
+	MediaRemoteCacheDays     string
 
 	// storage
 	StorageBackend       string
@@ -138,6 +139,7 @@ var Keys = KeyNames{
 	MediaVideoMaxSize:        "media-video-max-size",
 	MediaDescriptionMinChars: "media-description-min-chars",
 	MediaDescriptionMaxChars: "media-description-max-chars",
+	MediaRemoteCacheDays:     "media-remote-cache-days",
 
 	StorageBackend:       "storage-backend",
 	StorageLocalBasePath: "storage-local-base-path",

internal/config/values.go

@@ -51,6 +51,7 @@ type Values struct {
 	MediaVideoMaxSize        int
 	MediaDescriptionMinChars int
 	MediaDescriptionMaxChars int
+	MediaRemoteCacheDays     int
 
 	StorageBackend       string
 	StorageLocalBasePath string