mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-12-15 04:43:01 -06:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Clean up/uncache remote media (#407)

Browse source 
* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 • committed by GitHub
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions
Download patch file Download diff file Expand all files Collapse all files

177
vendor/modernc.org/libc/libc_linux.go generated vendored
View file

@ -7,13 +7,10 @@ package libc // import "modernc.org/libc"
import (
"encoding/hex"
"fmt"
"math"
"math/rand"
"os"
"os/exec"
"path/filepath"
"runtime/debug"
"sync"
"syscall"
"time"
"unsafe"
@ -30,7 +27,6 @@ import (
"modernc.org/libc/netinet/in"
"modernc.org/libc/signal"
"modernc.org/libc/stdio"
"modernc.org/libc/stdlib"
"modernc.org/libc/sys/socket"
"modernc.org/libc/sys/stat"
"modernc.org/libc/sys/types"
@ -685,61 +681,6 @@ func Xfileno(t *TLS, stream uintptr) int32 {
return -1
}
// int mkstemps(char *template, int suffixlen);
func Xmkstemps(t *TLS, template uintptr, suffixlen int32) int32 {
return Xmkstemps64(t, template, suffixlen)
}
// int mkstemps(char *template, int suffixlen);
func Xmkstemps64(t *TLS, template uintptr, suffixlen int32) int32 {
len := uintptr(Xstrlen(t, template))
x := template + uintptr(len-6) - uintptr(suffixlen)
for i := uintptr(0); i < 6; i++ {
if *(*byte)(unsafe.Pointer(x + i)) != 'X' {
t.setErrno(errno.EINVAL)
return -1
}
}
fd, err := tempFile(template, x, 0)
if err != 0 {
t.setErrno(err)
return -1
}
return int32(fd)
}
// int mkostemp(char *template, int flags);
func Xmkostemp(t *TLS, template uintptr, flags int32) int32 {
len := uintptr(Xstrlen(t, template))
x := template + uintptr(len-6)
for i := uintptr(0); i < 6; i++ {
if *(*byte)(unsafe.Pointer(x + i)) != 'X' {
t.setErrno(errno.EINVAL)
return -1
}
}
fd, err := tempFile(template, x, flags)
if err != 0 {
t.setErrno(err)
return -1
}
return int32(fd)
}
// int mkstemp(char *template);
func Xmkstemp(t *TLS, template uintptr) int32 {
return Xmkstemp64(t, template)
}
// int mkstemp(char *template);
func Xmkstemp64(t *TLS, template uintptr) int32 {
return Xmkstemps64(t, template, 0)
}
func newFtsent(t *TLS, info int, path string, stat *unix.Stat_t, err syscall.Errno) (r *fts.FTSENT) {
var statp uintptr
if stat != nil {
@ -950,11 +891,6 @@ func Xstrerror(t *TLS, errnum int32) uintptr {
return uintptr(unsafe.Pointer(&strerrorBuf[0]))
}
// int strerror_r(int errnum, char *buf, size_t buflen);
func Xstrerror_r(t *TLS, errnum int32, buf uintptr, buflen size_t) int32 {
panic(todo(""))
}
// void *dlopen(const char *filename, int flags);
func Xdlopen(t *TLS, filename uintptr, flags int32) uintptr {
panic(todo("%q", GoString(filename)))
@ -1377,11 +1313,6 @@ func Xwritev(t *TLS, fd int32, iov uintptr, iovcnt int32) types.Ssize_t {
return types.Ssize_t(n)
}
// void endpwent(void);
func Xendpwent(t *TLS) {
// nop
}
// int __isoc99_sscanf(const char *str, const char *format, ...);
func X__isoc99_sscanf(t *TLS, str, format, va uintptr) int32 {
r := Xsscanf(t, str, format, va)
@ -1391,21 +1322,21 @@ func X__isoc99_sscanf(t *TLS, str, format, va uintptr) int32 {
return r
}
var ctimeStaticBuf [32]byte
// char *ctime(const time_t *timep);
func Xctime(t *TLS, timep uintptr) uintptr {
return Xctime_r(t, timep, uintptr(unsafe.Pointer(&ctimeStaticBuf[0])))
}
// char *ctime_r(const time_t *timep, char *buf);
func Xctime_r(t *TLS, timep, buf uintptr) uintptr {
ut := *(*unix.Time_t)(unsafe.Pointer(timep))
tm := time.Unix(int64(ut), 0).Local()
s := tm.Format(time.ANSIC) + "\n\x00"
copy((*RawMem)(unsafe.Pointer(buf))[:26:26], s)
return buf
}
// var ctimeStaticBuf [32]byte
//
// // char *ctime(const time_t *timep);
// func Xctime(t *TLS, timep uintptr) uintptr {
// return Xctime_r(t, timep, uintptr(unsafe.Pointer(&ctimeStaticBuf[0])))
// }
//
// // char *ctime_r(const time_t *timep, char *buf);
// func Xctime_r(t *TLS, timep, buf uintptr) uintptr {
// ut := *(*unix.Time_t)(unsafe.Pointer(timep))
// tm := time.Unix(int64(ut), 0).Local()
// s := tm.Format(time.ANSIC) + "\n\x00"
// copy((*RawMem)(unsafe.Pointer(buf))[:26:26], s)
// return buf
// }
// ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset);
func Xpwrite(t *TLS, fd int32, buf uintptr, count types.Size_t, offset types.Off_t) types.Ssize_t {
@ -1464,16 +1395,20 @@ func Xposix_fadvise(t *TLS, fd int32, offset, len types.Off_t, advice int32) int
return 0
}
// void uuid_generate_random(uuid_t out);
func Xuuid_generate_random(t *TLS, out uintptr) {
panic(todo(""))
// int fgetc(FILE *stream);
func Xfgetc(t *TLS, stream uintptr) int32 {
fd := int((*stdio.FILE)(unsafe.Pointer(stream)).F_fileno)
var buf [1]byte
if n, _ := unix.Read(fd, buf[:]); n != 0 {
return int32(buf[0])
}
return stdio.EOF
}
// void uuid_unparse(uuid_t uu, char *out);
func Xuuid_unparse(t *TLS, uu, out uintptr) {
s := (*guuid.UUID)(unsafe.Pointer(uu)).String()
copy((*RawMem)(unsafe.Pointer(out))[:], s)
*(*byte)(unsafe.Pointer(out + uintptr(len(s)))) = 0
// void uuid_copy(uuid_t dst, uuid_t src);
func Xuuid_copy(t *TLS, dst, src uintptr) {
*(*uuid.Uuid_t)(unsafe.Pointer(dst)) = *(*uuid.Uuid_t)(unsafe.Pointer(src))
}
// int uuid_parse( char *in, uuid_t uu);
@ -1486,61 +1421,3 @@ func Xuuid_parse(t *TLS, in uintptr, uu uintptr) int32 {
copy((*RawMem)(unsafe.Pointer(uu))[:unsafe.Sizeof(uuid.Uuid_t{})], r[:])
return 0
}
// The initstate_r() function is like initstate(3) except that it initializes
// the state in the object pointed to by buf, rather than initializing the
// global state variable. Before calling this function, the buf.state field
// must be initialized to NULL. The initstate_r() function records a pointer
// to the statebuf argument inside the structure pointed to by buf. Thus,
// state‐ buf should not be deallocated so long as buf is still in use. (So,
// statebuf should typically be allocated as a static variable, or allocated on
// the heap using malloc(3) or similar.)
//
// char *initstate_r(unsigned int seed, char *statebuf, size_t statelen, struct random_data *buf);
func Xinitstate_r(t *TLS, seed uint32, statebuf uintptr, statelen types.Size_t, buf uintptr) int32 {
if buf == 0 {
panic(todo(""))
}
randomDataMu.Lock()
defer randomDataMu.Unlock()
randomData[buf] = rand.New(rand.NewSource(int64(seed)))
return 0
}
var (
randomData = map[uintptr]*rand.Rand{}
randomDataMu sync.Mutex
)
// int random_r(struct random_data *buf, int32_t *result);
func Xrandom_r(t *TLS, buf, result uintptr) int32 {
randomDataMu.Lock()
defer randomDataMu.Unlock()
mr := randomData[buf]
if stdlib.RAND_MAX != math.MaxInt32 {
panic(todo(""))
}
*(*int32)(unsafe.Pointer(result)) = mr.Int31()
return 0
}
// void uuid_copy(uuid_t dst, uuid_t src);
func Xuuid_copy(t *TLS, dst, src uintptr) {
*(*uuid.Uuid_t)(unsafe.Pointer(dst)) = *(*uuid.Uuid_t)(unsafe.Pointer(src))
}
// int fgetc(FILE *stream);
func Xfgetc(t *TLS, stream uintptr) int32 {
fd := int((*stdio.FILE)(unsafe.Pointer(stream)).F_fileno)
var buf [1]byte
if n, _ := unix.Read(fd, buf[:]); n != 0 {
return int32(buf[0])
}
return stdio.EOF
}