mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-12-15 15:43:02 -06:00
[feature] Clean up/uncache remote media (#407)
* Add whereNotEmptyAndNotNull * Add GetRemoteOlderThanDays * Add GetRemoteOlderThanDays * Add PruneRemote to Manager interface * Start implementing PruneRemote * add new attachment + status to tests * fix up and test GetRemoteOlderThan * fix bad import * PruneRemote: return number pruned * add Cached column to mediaattachment * update + test pruneRemote * update mediaTest * use Cached column * upstep bun to latest version * embed structs in mediaAttachment * migrate mediaAttachment to new format * don't default cached to true * select only remote media * update db dependencies * step bun back to last working version * update pruneRemote to use Cached field * fix storage path of test attachments * add recache logic to manager * fix trimmed aspect ratio * test prune and recache * return errwithcode * tidy up different paths for emoji vs attachment * fix incorrect thumbnail type being stored * expose TransportController to media processor * implement tee-ing recached content * add thoughts of dog to test fedi attachments * test get remote files * add comment on PruneRemote * add postData cleanup to recache * test thumbnail fetching * add incredible diagram * go mod tidy * buffer pipes for recache streaming * test for client stops reading after 1kb * add media-remote-cache-days to config * add cron package * wrap logrus so it's available to cron * start and stop cron jobs gracefully
This commit is contained in:
tobi
GitHub
parent
100f1280a6
commit
07727753b9
424 changed files with 637100 additions and 176498 deletions
161
vendor/modernc.org/libc/libc_unix.go
generated
vendored
161
vendor/modernc.org/libc/libc_unix.go
generated
vendored
|
|
@ -2,22 +2,27 @@
|
|||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build linux || darwin || freebsd || netbsd
|
||||
// +build linux darwin freebsd netbsd
|
||||
//go:build linux || darwin || freebsd || netbsd || openbsd
|
||||
// +build linux darwin freebsd netbsd openbsd
|
||||
|
||||
package libc // import "modernc.org/libc"
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"math/rand"
|
||||
"os"
|
||||
gosignal "os/signal"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
guuid "github.com/google/uuid"
|
||||
"golang.org/x/sys/unix"
|
||||
"modernc.org/libc/errno"
|
||||
"modernc.org/libc/grp"
|
||||
|
|
@ -25,7 +30,9 @@ import (
|
|||
"modernc.org/libc/pwd"
|
||||
"modernc.org/libc/signal"
|
||||
"modernc.org/libc/stdio"
|
||||
"modernc.org/libc/stdlib"
|
||||
"modernc.org/libc/sys/types"
|
||||
ctime "modernc.org/libc/time"
|
||||
)
|
||||
|
||||
var staticGetpwnam pwd.Passwd
|
||||
|
|
@ -835,3 +842,153 @@ func Xgetpwuid_r(t *TLS, uid types.Uid_t, cpwd, buf uintptr, buflen types.Size_t
|
|||
*(*uintptr)(unsafe.Pointer(result)) = 0
|
||||
return 0
|
||||
}
|
||||
|
||||
// int mkostemp(char *template, int flags);
|
||||
func Xmkostemp(t *TLS, template uintptr, flags int32) int32 {
|
||||
len := uintptr(Xstrlen(t, template))
|
||||
x := template + uintptr(len-6)
|
||||
for i := uintptr(0); i < 6; i++ {
|
||||
if *(*byte)(unsafe.Pointer(x + i)) != 'X' {
|
||||
t.setErrno(errno.EINVAL)
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
fd, err := tempFile(template, x, flags)
|
||||
if err != nil {
|
||||
t.setErrno(err)
|
||||
return -1
|
||||
}
|
||||
|
||||
return int32(fd)
|
||||
}
|
||||
|
||||
// void uuid_generate_random(uuid_t out);
|
||||
func Xuuid_generate_random(t *TLS, out uintptr) {
|
||||
x := guuid.New()
|
||||
copy((*RawMem)(unsafe.Pointer(out))[:], x[:])
|
||||
}
|
||||
|
||||
// void uuid_unparse(uuid_t uu, char *out);
|
||||
func Xuuid_unparse(t *TLS, uu, out uintptr) {
|
||||
s := (*guuid.UUID)(unsafe.Pointer(uu)).String()
|
||||
copy((*RawMem)(unsafe.Pointer(out))[:], s)
|
||||
*(*byte)(unsafe.Pointer(out + uintptr(len(s)))) = 0
|
||||
}
|
||||
|
||||
var staticRandomData = &rand.Rand{}
|
||||
|
||||
// char *initstate(unsigned seed, char *state, size_t size);
|
||||
func Xinitstate(t *TLS, seed uint32, statebuf uintptr, statelen types.Size_t) uintptr {
|
||||
staticRandomData = rand.New(rand.NewSource(int64(seed)))
|
||||
return 0
|
||||
}
|
||||
|
||||
// char *setstate(const char *state);
|
||||
func Xsetstate(t *TLS, state uintptr) uintptr {
|
||||
t.setErrno(errno.EINVAL) //TODO
|
||||
return 0
|
||||
}
|
||||
|
||||
// The initstate_r() function is like initstate(3) except that it initializes
|
||||
// the state in the object pointed to by buf, rather than initializing the
|
||||
// global state variable. Before calling this function, the buf.state field
|
||||
// must be initialized to NULL. The initstate_r() function records a pointer
|
||||
// to the statebuf argument inside the structure pointed to by buf. Thus,
|
||||
// state‐ buf should not be deallocated so long as buf is still in use. (So,
|
||||
// statebuf should typically be allocated as a static variable, or allocated on
|
||||
// the heap using malloc(3) or similar.)
|
||||
//
|
||||
// char *initstate_r(unsigned int seed, char *statebuf, size_t statelen, struct random_data *buf);
|
||||
func Xinitstate_r(t *TLS, seed uint32, statebuf uintptr, statelen types.Size_t, buf uintptr) int32 {
|
||||
if buf == 0 {
|
||||
panic(todo(""))
|
||||
}
|
||||
|
||||
randomDataMu.Lock()
|
||||
|
||||
defer randomDataMu.Unlock()
|
||||
|
||||
randomData[buf] = rand.New(rand.NewSource(int64(seed)))
|
||||
return 0
|
||||
}
|
||||
|
||||
var (
|
||||
randomData = map[uintptr]*rand.Rand{}
|
||||
randomDataMu sync.Mutex
|
||||
)
|
||||
|
||||
// int mkstemps(char *template, int suffixlen);
|
||||
func Xmkstemps(t *TLS, template uintptr, suffixlen int32) int32 {
|
||||
return Xmkstemps64(t, template, suffixlen)
|
||||
}
|
||||
|
||||
// int mkstemps(char *template, int suffixlen);
|
||||
func Xmkstemps64(t *TLS, template uintptr, suffixlen int32) int32 {
|
||||
len := uintptr(Xstrlen(t, template))
|
||||
x := template + uintptr(len-6) - uintptr(suffixlen)
|
||||
for i := uintptr(0); i < 6; i++ {
|
||||
if *(*byte)(unsafe.Pointer(x + i)) != 'X' {
|
||||
t.setErrno(errno.EINVAL)
|
||||
return -1
|
||||
}
|
||||
}
|
||||
|
||||
fd, err := tempFile(template, x, 0)
|
||||
if err != nil {
|
||||
t.setErrno(err)
|
||||
return -1
|
||||
}
|
||||
|
||||
return int32(fd)
|
||||
}
|
||||
|
||||
// int mkstemp(char *template);
|
||||
func Xmkstemp(t *TLS, template uintptr) int32 {
|
||||
return Xmkstemp64(t, template)
|
||||
}
|
||||
|
||||
// int mkstemp(char *template);
|
||||
func Xmkstemp64(t *TLS, template uintptr) int32 {
|
||||
return Xmkstemps64(t, template, 0)
|
||||
}
|
||||
|
||||
// int random_r(struct random_data *buf, int32_t *result);
|
||||
func Xrandom_r(t *TLS, buf, result uintptr) int32 {
|
||||
randomDataMu.Lock()
|
||||
|
||||
defer randomDataMu.Unlock()
|
||||
|
||||
mr := randomData[buf]
|
||||
if stdlib.RAND_MAX != math.MaxInt32 {
|
||||
panic(todo(""))
|
||||
}
|
||||
*(*int32)(unsafe.Pointer(result)) = mr.Int31()
|
||||
return 0
|
||||
}
|
||||
|
||||
// int strerror_r(int errnum, char *buf, size_t buflen);
|
||||
func Xstrerror_r(t *TLS, errnum int32, buf uintptr, buflen size_t) int32 {
|
||||
panic(todo(""))
|
||||
}
|
||||
|
||||
// void endpwent(void);
|
||||
func Xendpwent(t *TLS) {
|
||||
// nop
|
||||
}
|
||||
|
||||
var ctimeStaticBuf [32]byte
|
||||
|
||||
// char *ctime(const time_t *timep);
|
||||
func Xctime(t *TLS, timep uintptr) uintptr {
|
||||
return Xctime_r(t, timep, uintptr(unsafe.Pointer(&ctimeStaticBuf[0])))
|
||||
}
|
||||
|
||||
// char *ctime_r(const time_t *timep, char *buf);
|
||||
func Xctime_r(t *TLS, timep, buf uintptr) uintptr {
|
||||
ut := *(*ctime.Time_t)(unsafe.Pointer(timep))
|
||||
tm := time.Unix(int64(ut), 0).Local()
|
||||
s := tm.Format(time.ANSIC) + "\n\x00"
|
||||
copy((*RawMem)(unsafe.Pointer(buf))[:26:26], s)
|
||||
return buf
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue