[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 committed by GitHub
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions

View file

@ -3,6 +3,8 @@
package libc // import "modernc.org/libc"
var CAPI = map[string]struct{}{
"_CurrentRuneLocale": {},
"_DefaultRuneLocale": {},
"_IO_putc": {},
"_NSGetEnviron": {},
"___errno_location": {},
@ -36,6 +38,8 @@ var CAPI = map[string]struct{}{
"__builtin_exit": {},
"__builtin_expect": {},
"__builtin_fabs": {},
"__builtin_fabsf": {},
"__builtin_fabsl": {},
"__builtin_free": {},
"__builtin_getentropy": {},
"__builtin_huge_val": {},
@ -44,6 +48,7 @@ var CAPI = map[string]struct{}{
"__builtin_inff": {},
"__builtin_infl": {},
"__builtin_isnan": {},
"__builtin_isunordered": {},
"__builtin_malloc": {},
"__builtin_memcmp": {},
"__builtin_memcpy": {},
@ -77,6 +82,7 @@ var CAPI = map[string]struct{}{
"__ccgo_sqlite3_log": {},
"__cmsg_nxthdr": {},
"__ctype_get_mb_cur_max": {},
"__darwin_check_fd_set_overflow": {},
"__darwin_fd_clr": {},
"__darwin_fd_isset": {},
"__darwin_fd_set": {},
@ -93,33 +99,37 @@ var CAPI = map[string]struct{}{
"__inline_isnanf": {},
"__inline_isnanl": {},
"__intscan": {},
"__isalnum_l": {},
"__isalpha_l": {},
"__isdigit_l": {},
"__islower_l": {},
"__isctype": {},
"__isnan": {},
"__isnanf": {},
"__isnanl": {},
"__isoc99_sscanf": {},
"__isprint_l": {},
"__isupper_l": {},
"__isxdigit_l": {},
"__istype": {},
"__lookup_ipliteral": {},
"__lookup_name": {},
"__lookup_serv": {},
"__maskrune": {},
"__mb_cur_max": {},
"__putenv": {},
"__shgetc": {},
"__shlim": {},
"__sincos_stret": {},
"__sincosf_stret": {},
"__sincospi_stret": {},
"__sincospif_stret": {},
"__stderrp": {},
"__stdinp": {},
"__stdoutp": {},
"__strchrnul": {},
"__strncasecmp_l": {},
"__sync_add_and_fetch_uint32": {},
"__sync_sub_and_fetch_uint32": {},
"__tolower": {},
"__toread": {},
"__toread_needs_stdio_exit": {},
"__toupper": {},
"__uflow": {},
"__wcwidth": {},
"_exit": {},
"_longjmp": {},
"_obstack_begin": {},
@ -143,6 +153,7 @@ var CAPI = map[string]struct{}{
"atoi": {},
"atol": {},
"bind": {},
"bsearch": {},
"bzero": {},
"calloc": {},
"ceil": {},
@ -167,11 +178,15 @@ var CAPI = map[string]struct{}{
"cos": {},
"cosf": {},
"cosh": {},
"ctime": {},
"ctime_r": {},
"digittoint": {},
"dlclose": {},
"dlerror": {},
"dlopen": {},
"dlsym": {},
"dup2": {},
"endpwent": {},
"environ": {},
"exit": {},
"exp": {},
@ -240,6 +255,7 @@ var CAPI = map[string]struct{}{
"getnameinfo": {},
"getpeername": {},
"getpid": {},
"getprogname": {},
"getpwnam": {},
"getpwnam_r": {},
"getpwuid": {},
@ -260,19 +276,31 @@ var CAPI = map[string]struct{}{
"inet_ntoa": {},
"inet_ntop": {},
"inet_pton": {},
"initstate": {},
"initstate_r": {},
"ioctl": {},
"isalnum": {},
"isalpha": {},
"isascii": {},
"isatty": {},
"isblank": {},
"iscntrl": {},
"isdigit": {},
"isgraph": {},
"ishexnumber": {},
"isideogram": {},
"islower": {},
"isnan": {},
"isnanf": {},
"isnanl": {},
"isnumber": {},
"isphonogram": {},
"isprint": {},
"ispunct": {},
"isrune": {},
"issetugid": {},
"isspace": {},
"isspecial": {},
"isupper": {},
"iswalnum": {},
"iswspace": {},
@ -281,6 +309,7 @@ var CAPI = map[string]struct{}{
"ldexp": {},
"link": {},
"listen": {},
"localeconv": {},
"localtime": {},
"localtime_r": {},
"log": {},
@ -305,7 +334,9 @@ var CAPI = map[string]struct{}{
"mkdir": {},
"mkfifo": {},
"mknod": {},
"mkostemp": {},
"mkstemp": {},
"mkstemp64": {},
"mkstemps": {},
"mkstemps64": {},
"mktime": {},
@ -372,6 +403,7 @@ var CAPI = map[string]struct{}{
"rand": {},
"rand_r": {},
"random": {},
"random_r": {},
"read": {},
"readdir": {},
"readlink": {},
@ -403,6 +435,7 @@ var CAPI = map[string]struct{}{
"setlocale": {},
"setsid": {},
"setsockopt": {},
"setstate": {},
"setvbuf": {},
"shmat": {},
"shmctl": {},
@ -435,9 +468,11 @@ var CAPI = map[string]struct{}{
"strcspn": {},
"strdup": {},
"strerror": {},
"strerror_r": {},
"strlcat": {},
"strlcpy": {},
"strlen": {},
"strncasecmp": {},
"strncat": {},
"strncmp": {},
"strncpy": {},
@ -466,6 +501,7 @@ var CAPI = map[string]struct{}{
"tcsetattr": {},
"time": {},
"tmpfile": {},
"toascii": {},
"tolower": {},
"toupper": {},
"trunc": {},
@ -479,6 +515,10 @@ var CAPI = map[string]struct{}{
"usleep": {},
"utime": {},
"utimes": {},
"uuid_copy": {},
"uuid_generate_random": {},
"uuid_parse": {},
"uuid_unparse": {},
"vasprintf": {},
"vfprintf": {},
"vprintf": {},