mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-11-15 01:57:29 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Clean up/uncache remote media (#407)

Browse source 
* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 • committed by GitHub
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions
Download patch file Download diff file Expand all files Collapse all files

4
vendor/modernc.org/libc/errno/capi_darwin_amd64.go generated vendored
View file

@ -2,4 +2,6 @@
package errno
var CAPI = map[string]struct{}{}
var CAPI = map[string]struct{}{
"__darwin_check_fd_set_overflow": {},
}

4
vendor/modernc.org/libc/errno/capi_darwin_arm64.go generated vendored
View file

@ -2,4 +2,6 @@
package errno
var CAPI = map[string]struct{}{}
var CAPI = map[string]struct{}{
"__darwin_check_fd_set_overflow": {},
}

5
vendor/modernc.org/libc/errno/capi_freebsd_386.go generated vendored Normal file
View file

@ -0,0 +1,5 @@
// Code generated by 'ccgo errno/gen.c -crt-import-path -export-defines -export-enums -export-externs X -export-fields F -export-structs -export-typedefs -header -hide _OSSwapInt16,_OSSwapInt32,_OSSwapInt64 -o errno/errno_freebsd_386.go -pkgname errno', DO NOT EDIT.
package errno
var CAPI = map[string]struct{}{}

5
vendor/modernc.org/libc/errno/capi_openbsd_amd64.go generated vendored Normal file
View file

@ -0,0 +1,5 @@
// Code generated by 'ccgo errno/gen.c -crt-import-path -export-defines -export-enums -export-externs X -export-fields F -export-structs -export-typedefs -header -hide _OSSwapInt16,_OSSwapInt32,_OSSwapInt64 -o errno/errno_openbsd_amd64.go -pkgname errno', DO NOT EDIT.
package errno
var CAPI = map[string]struct{}{}

2
vendor/modernc.org/libc/errno/capi_windows_386.go generated vendored
View file

@ -1,4 +1,4 @@
// Code generated by 'ccgo errno/gen.c -crt-import-path -export-defines -export-enums -export-externs X -export-fields F -export-structs -export-typedefs -header -hide _OSSwapInt16,_OSSwapInt32,_OSSwapInt64 -o errno/errno_windows_386.go -pkgname errno', DO NOT EDIT.
// Code generated by 'ccgo errno\gen.c -crt-import-path -export-defines -export-enums -export-externs X -export-fields F -export-structs -export-typedefs -header -hide _OSSwapInt16,_OSSwapInt32,_OSSwapInt64 -o errno\errno_windows_386.go -pkgname errno', DO NOT EDIT.
package errno

9
vendor/modernc.org/libc/errno/errno_darwin_amd64.go generated vendored
View file

@ -154,6 +154,8 @@ type X__uint128_t = struct {
type X__builtin_va_list = uintptr /* <builtin>:46:14 */
type X__float128 = float64 /* <builtin>:47:21 */
var X__darwin_check_fd_set_overflow uintptr /* <builtin>:146:5: */
// Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
//
// @APPLE_LICENSE_HEADER_START@
@ -312,6 +314,13 @@ type X__float128 = float64 /* <builtin>:47:21 */
// in between its arguments. __CONCAT can also concatenate double-quoted
// strings produced by the __STRING macro, but this only works with ANSI C.
// In non-ANSI C environments, new programs will want ANSI-only C keywords
// deleted from the program and old programs will want them left alone.
// When using a compiler other than gcc, programs using the ANSI C keywords
// const, inline etc. as normal identifiers should define -DNO_ANSI_KEYWORDS.
// When using "gcc -traditional", we assume that this is the intent; if
// __GNUC__ is defined but __STDC__ is not, we leave the new keywords alone.
// __unused denotes variables and functions that may not be used, preventing
// the compiler from warning about it if not used.

9
vendor/modernc.org/libc/errno/errno_darwin_arm64.go generated vendored
View file

@ -156,6 +156,8 @@ type X__uint128_t = struct {
type X__builtin_va_list = uintptr /* <builtin>:46:14 */
type X__float128 = float64 /* <builtin>:47:21 */
var X__darwin_check_fd_set_overflow uintptr /* <builtin>:146:5: */
// Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
//
// @APPLE_LICENSE_HEADER_START@
@ -314,6 +316,13 @@ type X__float128 = float64 /* <builtin>:47:21 */
// in between its arguments. __CONCAT can also concatenate double-quoted
// strings produced by the __STRING macro, but this only works with ANSI C.
// In non-ANSI C environments, new programs will want ANSI-only C keywords
// deleted from the program and old programs will want them left alone.
// When using a compiler other than gcc, programs using the ANSI C keywords
// const, inline etc. as normal identifiers should define -DNO_ANSI_KEYWORDS.
// When using "gcc -traditional", we assume that this is the intent; if
// __GNUC__ is defined but __STDC__ is not, we leave the new keywords alone.
// __pure2 can be used for functions that are only a function of their scalar
// arguments (meaning they can't dereference pointers).
//

158
vendor/modernc.org/libc/errno/errno_freebsd_386.go generated vendored Normal file
View file

@ -0,0 +1,158 @@
// Code generated by 'ccgo errno/gen.c -crt-import-path "" -export-defines "" -export-enums "" -export-externs X -export-fields F -export-structs "" -export-typedefs "" -header -hide _OSSwapInt16,_OSSwapInt32,_OSSwapInt64 -o errno/errno_freebsd_386.go -pkgname errno', DO NOT EDIT.
package errno
import (
"math"
"reflect"
"sync/atomic"
"unsafe"
)
var _ = math.Pi
var _ reflect.Kind
var _ atomic.Value
var _ unsafe.Pointer
const (
E2BIG = 7
EACCES = 13
EADDRINUSE = 48
EADDRNOTAVAIL = 49
EAFNOSUPPORT = 47
EAGAIN = 35
EALREADY = 37
EAUTH = 80
EBADF = 9
EBADMSG = 89
EBADRPC = 72
EBUSY = 16
ECANCELED = 85
ECAPMODE = 94
ECHILD = 10
ECONNABORTED = 53
ECONNREFUSED = 61
ECONNRESET = 54
EDEADLK = 11
EDESTADDRREQ = 39
EDOM = 33
EDOOFUS = 88
EDQUOT = 69
EEXIST = 17
EFAULT = 14
EFBIG = 27
EFTYPE = 79
EHOSTDOWN = 64
EHOSTUNREACH = 65
EIDRM = 82
EILSEQ = 86
EINPROGRESS = 36
EINTEGRITY = 97
EINTR = 4
EINVAL = 22
EIO = 5
EISCONN = 56
EISDIR = 21
ELAST = 97
ELOOP = 62
EMFILE = 24
EMLINK = 31
EMSGSIZE = 40
EMULTIHOP = 90
ENAMETOOLONG = 63
ENEEDAUTH = 81
ENETDOWN = 50
ENETRESET = 52
ENETUNREACH = 51
ENFILE = 23
ENOATTR = 87
ENOBUFS = 55
ENODEV = 19
ENOENT = 2
ENOEXEC = 8
ENOLCK = 77
ENOLINK = 91
ENOMEM = 12
ENOMSG = 83
ENOPROTOOPT = 42
ENOSPC = 28
ENOSYS = 78
ENOTBLK = 15
ENOTCAPABLE = 93
ENOTCONN = 57
ENOTDIR = 20
ENOTEMPTY = 66
ENOTRECOVERABLE = 95
ENOTSOCK = 38
ENOTSUP = 45
ENOTTY = 25
ENXIO = 6
EOPNOTSUPP = 45
EOVERFLOW = 84
EOWNERDEAD = 96
EPERM = 1
EPFNOSUPPORT = 46
EPIPE = 32
EPROCLIM = 67
EPROCUNAVAIL = 76
EPROGMISMATCH = 75
EPROGUNAVAIL = 74
EPROTO = 92
EPROTONOSUPPORT = 43
EPROTOTYPE = 41
ERANGE = 34
EREMOTE = 71
EROFS = 30
ERPCMISMATCH = 73
ESHUTDOWN = 58
ESOCKTNOSUPPORT = 44
ESPIPE = 29
ESRCH = 3
ESTALE = 70
ETIMEDOUT = 60
ETOOMANYREFS = 59
ETXTBSY = 26
EUSERS = 68
EWOULDBLOCK = 35
EXDEV = 18
X_ERRNO_T_DEFINED = 0
X_FILE_OFFSET_BITS = 64
X_ILP32 = 1
X_Nonnull = 0
X_Null_unspecified = 0
X_Nullable = 0
X_SYS_CDEFS_H_ = 0
X_SYS_ERRNO_H_ = 0
I386 = 1
Unix = 1
)
type Ptrdiff_t = int32 /* <builtin>:3:26 */
type Size_t = uint32 /* <builtin>:9:23 */
type Wchar_t = int32 /* <builtin>:15:24 */
type X__builtin_va_list = uintptr /* <builtin>:46:14 */
type X__float128 = float64 /* <builtin>:47:21 */
// 11 was EAGAIN
// math software
// non-blocking and interrupt i/o
// ipc/network software -- argument errors
// ipc/network software -- operational errors
// should be rearranged
// quotas & mush
// Network File System
// ISO/IEC 9899:2011 K.3.2.2
type Errno_t = int32 /* errno.h:204:13 */
var _ int8 /* gen.c:2:13: */

142
vendor/modernc.org/libc/errno/errno_openbsd_amd64.go generated vendored Normal file
View file

@ -0,0 +1,142 @@
// Code generated by 'ccgo errno/gen.c -crt-import-path "" -export-defines "" -export-enums "" -export-externs X -export-fields F -export-structs "" -export-typedefs "" -header -hide _OSSwapInt16,_OSSwapInt32,_OSSwapInt64 -o errno/errno_openbsd_amd64.go -pkgname errno', DO NOT EDIT.
package errno
import (
"math"
"reflect"
"sync/atomic"
"unsafe"
)
var _ = math.Pi
var _ reflect.Kind
var _ atomic.Value
var _ unsafe.Pointer
const (
E2BIG = 7
EACCES = 13
EADDRINUSE = 48
EADDRNOTAVAIL = 49
EAFNOSUPPORT = 47
EAGAIN = 35
EALREADY = 37
EAUTH = 80
EBADF = 9
EBADMSG = 92
EBADRPC = 72
EBUSY = 16
ECANCELED = 88
ECHILD = 10
ECONNABORTED = 53
ECONNREFUSED = 61
ECONNRESET = 54
EDEADLK = 11
EDESTADDRREQ = 39
EDOM = 33
EDQUOT = 69
EEXIST = 17
EFAULT = 14
EFBIG = 27
EFTYPE = 79
EHOSTDOWN = 64
EHOSTUNREACH = 65
EIDRM = 89
EILSEQ = 84
EINPROGRESS = 36
EINTR = 4
EINVAL = 22
EIO = 5
EIPSEC = 82
EISCONN = 56
EISDIR = 21
ELAST = 95
ELOOP = 62
EMEDIUMTYPE = 86
EMFILE = 24
EMLINK = 31
EMSGSIZE = 40
ENAMETOOLONG = 63
ENEEDAUTH = 81
ENETDOWN = 50
ENETRESET = 52
ENETUNREACH = 51
ENFILE = 23
ENOATTR = 83
ENOBUFS = 55
ENODEV = 19
ENOENT = 2
ENOEXEC = 8
ENOLCK = 77
ENOMEDIUM = 85
ENOMEM = 12
ENOMSG = 90
ENOPROTOOPT = 42
ENOSPC = 28
ENOSYS = 78
ENOTBLK = 15
ENOTCONN = 57
ENOTDIR = 20
ENOTEMPTY = 66
ENOTRECOVERABLE = 93
ENOTSOCK = 38
ENOTSUP = 91
ENOTTY = 25
ENXIO = 6
EOPNOTSUPP = 45
EOVERFLOW = 87
EOWNERDEAD = 94
EPERM = 1
EPFNOSUPPORT = 46
EPIPE = 32
EPROCLIM = 67
EPROCUNAVAIL = 76
EPROGMISMATCH = 75
EPROGUNAVAIL = 74
EPROTO = 95
EPROTONOSUPPORT = 43
EPROTOTYPE = 41
ERANGE = 34
EREMOTE = 71
EROFS = 30
ERPCMISMATCH = 73
ESHUTDOWN = 58
ESOCKTNOSUPPORT = 44
ESPIPE = 29
ESRCH = 3
ESTALE = 70
ETIMEDOUT = 60
ETOOMANYREFS = 59
ETXTBSY = 26
EUSERS = 68
EWOULDBLOCK = 35
EXDEV = 18
X_ERRNO_H_ = 0
X_FILE_OFFSET_BITS = 64
X_LP64 = 1
X_MACHINE_CDEFS_H_ = 0
X_RET_PROTECTOR = 1
X_SYS_CDEFS_H_ = 0
Unix = 1
)
type Ptrdiff_t = int64 /* <builtin>:3:26 */
type Size_t = uint64 /* <builtin>:9:23 */
type Wchar_t = int32 /* <builtin>:15:24 */
type X__int128_t = struct {
Flo int64
Fhi int64
} /* <builtin>:21:43 */ // must match modernc.org/mathutil.Int128
type X__uint128_t = struct {
Flo uint64
Fhi uint64
} /* <builtin>:22:44 */ // must match modernc.org/mathutil.Int128
type X__builtin_va_list = uintptr /* <builtin>:46:14 */
type X__float128 = float64 /* <builtin>:47:21 */
var _ int8 /* gen.c:2:13: */

61
vendor/modernc.org/libc/errno/errno_windows_386.go generated vendored
View file

@ -1,4 +1,4 @@
// Code generated by 'ccgo errno/gen.c -crt-import-path "" -export-defines "" -export-enums "" -export-externs X -export-fields F -export-structs "" -export-typedefs "" -header -hide _OSSwapInt16,_OSSwapInt32,_OSSwapInt64 -o errno/errno_windows_386.go -pkgname errno', DO NOT EDIT.
// Code generated by 'ccgo errno\gen.c -crt-import-path "" -export-defines "" -export-enums "" -export-externs X -export-fields F -export-structs "" -export-typedefs "" -header -hide _OSSwapInt16,_OSSwapInt32,_OSSwapInt64 -o errno\errno_windows_386.go -pkgname errno', DO NOT EDIT.
package errno
@ -112,7 +112,9 @@ const (
EWOULDBLOCK = 140
EXDEV = 18
MINGW_DDK_H = 0
MINGW_DDRAW_VERSION = 7
MINGW_HAS_DDK_H = 1
MINGW_HAS_DDRAW_H = 1
MINGW_HAS_SECURE_API = 1
MINGW_SDK_INIT = 0
STRUNCATE = 80
@ -136,12 +138,9 @@ const (
X_CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES = 0
X_CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES_COUNT = 0
X_CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES_MEMORY = 0
X_CRT_USE_WINAPI_FAMILY_DESKTOP_APP = 0
X_DLL = 0
X_ERRCODE_DEFINED = 0
X_FILE_OFFSET_BITS = 64
X_ILP32 = 1
X_INC_CORECRT = 0
X_INC_CRTDEFS = 0
X_INC_CRTDEFS_MACRO = 0
X_INC_ERRNO = 0
@ -156,6 +155,7 @@ const (
X_PGLOBAL = 0
X_PTRDIFF_T_ = 0
X_PTRDIFF_T_DEFINED = 0
X_REENTRANT = 1
X_RSIZE_T_DEFINED = 0
X_SECURECRT_ERRCODE_VALUES_DEFINED = 0
X_SECURECRT_FILL_BUFFER_PATTERN = 0xFD
@ -210,11 +210,6 @@ type Va_list = X__builtin_va_list /* <builtin>:50:27 */
// This file is part of the mingw-w64 runtime package.
// No warranty is given; refer to the file DISCLAIMER.PD within this package.
// *
// This file has no copyright assigned and is placed in the Public Domain.
// This file is part of the mingw-w64 runtime package.
// No warranty is given; refer to the file DISCLAIMER.PD within this package.
// This macro holds an monotonic increasing value, which indicates
// a specific fix/patch is present on trunk. This value isn't related to
// minor/major version-macros. It is increased on demand, if a big
@ -235,12 +230,6 @@ type Va_list = X__builtin_va_list /* <builtin>:50:27 */
// MinGW-w64 has some additional C99 printf/scanf feature support.
// So we add some helper macros to ease recognition of them.
// If _FORTIFY_SOURCE is enabled, some inline functions may use
// __builtin_va_arg_pack(). GCC may report an error if the address
// of such a function is used. Set _FORTIFY_VA_ARG=0 in this case.
// Enable workaround for ABI incompatibility on affected platforms
// *
// This file has no copyright assigned and is placed in the Public Domain.
// This file is part of the mingw-w64 runtime package.
@ -281,28 +270,26 @@ type Va_list = X__builtin_va_list /* <builtin>:50:27 */
// This file is part of the mingw-w64 runtime package.
// No warranty is given; refer to the file DISCLAIMER.PD within this package.
// for backward compatibility
type X__gnuc_va_list = X__builtin_va_list /* vadefs.h:24:29 */
type Ssize_t = int32 /* corecrt.h:52:13 */
type Ssize_t = int32 /* crtdefs.h:47:13 */
type Rsize_t = Size_t /* corecrt.h:57:16 */
type Rsize_t = Size_t /* crtdefs.h:52:16 */
type Intptr_t = int32 /* corecrt.h:69:13 */
type Intptr_t = int32 /* crtdefs.h:64:13 */
type Uintptr_t = uint32 /* corecrt.h:82:22 */
type Uintptr_t = uint32 /* crtdefs.h:77:22 */
type Wint_t = uint16 /* corecrt.h:111:24 */
type Wctype_t = uint16 /* corecrt.h:112:24 */
type Wint_t = uint16 /* crtdefs.h:106:24 */
type Wctype_t = uint16 /* crtdefs.h:107:24 */
type Errno_t = int32 /* corecrt.h:118:13 */
type Errno_t = int32 /* crtdefs.h:113:13 */
type X__time32_t = int32 /* corecrt.h:123:14 */
type X__time32_t = int32 /* crtdefs.h:118:14 */
type X__time64_t = int64 /* corecrt.h:128:35 */
type X__time64_t = int64 /* crtdefs.h:123:35 */
type Time_t = X__time32_t /* corecrt.h:141:20 */
type Time_t = X__time32_t /* crtdefs.h:136:20 */
type Threadlocaleinfostruct = struct {
Frefcount int32
@ -328,29 +315,29 @@ type Threadlocaleinfostruct = struct {
Fpclmap uintptr
Fpcumap uintptr
Flc_time_curr uintptr
} /* corecrt.h:435:1 */
} /* crtdefs.h:422:1 */
type Pthreadlocinfo = uintptr /* corecrt.h:437:39 */
type Pthreadmbcinfo = uintptr /* corecrt.h:438:36 */
type Pthreadlocinfo = uintptr /* crtdefs.h:424:39 */
type Pthreadmbcinfo = uintptr /* crtdefs.h:425:36 */
type Localeinfo_struct = struct {
Flocinfo Pthreadlocinfo
Fmbcinfo Pthreadmbcinfo
} /* corecrt.h:441:9 */
} /* crtdefs.h:428:9 */
type X_locale_tstruct = Localeinfo_struct /* corecrt.h:444:3 */
type X_locale_t = uintptr /* corecrt.h:444:19 */
type X_locale_tstruct = Localeinfo_struct /* crtdefs.h:431:3 */
type X_locale_t = uintptr /* crtdefs.h:431:19 */
type TagLC_ID = struct {
FwLanguage uint16
FwCountry uint16
FwCodePage uint16
} /* corecrt.h:435:1 */
} /* crtdefs.h:422:1 */
type LC_ID = TagLC_ID /* corecrt.h:452:3 */
type LPLC_ID = uintptr /* corecrt.h:452:9 */
type LC_ID = TagLC_ID /* crtdefs.h:439:3 */
type LPLC_ID = uintptr /* crtdefs.h:439:9 */
type Threadlocinfo = Threadlocaleinfostruct /* corecrt.h:487:3 */
type Threadlocinfo = Threadlocaleinfostruct /* crtdefs.h:468:3 */
// Posix thread extensions.