mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-31 04:42:31 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Clean up/uncache remote media (#407)

Browse source 
* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 • committed by GitHub
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions
Download patch file Download diff file Expand all files Collapse all files

6
vendor/github.com/uptrace/bun/schema/formatter.go generated vendored
View file

@ -177,8 +177,6 @@ type NamedArgAppender interface {
AppendNamedArg(fmter Formatter, b []byte, name string) ([]byte, bool)
}
//------------------------------------------------------------------------------
type namedArgList struct {
arg NamedArgAppender
next *namedArgList
@ -219,13 +217,13 @@ func (a *namedArg) AppendNamedArg(fmter Formatter, b []byte, name string) ([]byt
//------------------------------------------------------------------------------
var _ NamedArgAppender = (*structArgs)(nil)
type structArgs struct {
table *Table
strct reflect.Value
}
var _ NamedArgAppender = (*structArgs)(nil)
func newStructArgs(fmter Formatter, strct interface{}) (*structArgs, bool) {
v := reflect.ValueOf(strct)
if !v.IsValid() {

48
vendor/github.com/uptrace/bun/schema/table.go generated vendored
View file

@ -203,7 +203,7 @@ func (t *Table) fieldByGoName(name string) *Field {
func (t *Table) initFields() {
t.Fields = make([]*Field, 0, t.Type.NumField())
t.FieldMap = make(map[string]*Field, t.Type.NumField())
t.addFields(t.Type, nil)
t.addFields(t.Type, "", nil)
if len(t.PKs) == 0 {
for _, name := range []string{"id", "uuid", "pk_" + t.ModelName} {
@ -230,7 +230,7 @@ func (t *Table) initFields() {
}
}
func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
func (t *Table) addFields(typ reflect.Type, prefix string, index []int) {
for i := 0; i < typ.NumField(); i++ {
f := typ.Field(i)
unexported := f.PkgPath != ""
@ -242,10 +242,6 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
continue
}
// Make a copy so the slice is not shared between fields.
index := make([]int, len(baseIndex))
copy(index, baseIndex)
if f.Anonymous {
if f.Name == "BaseModel" && f.Type == baseModelType {
if len(index) == 0 {
@ -258,7 +254,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
if fieldType.Kind() != reflect.Struct {
continue
}
t.addFields(fieldType, append(index, f.Index...))
t.addFields(fieldType, "", withIndex(index, f.Index))
tag := tagparser.Parse(f.Tag.Get("bun"))
if _, inherit := tag.Options["inherit"]; inherit {
@ -274,7 +270,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
continue
}
if field := t.newField(f, index); field != nil {
if field := t.newField(f, prefix, index); field != nil {
t.addField(field)
}
}
@ -315,10 +311,20 @@ func (t *Table) processBaseModelField(f reflect.StructField) {
}
//nolint
func (t *Table) newField(f reflect.StructField, index []int) *Field {
sqlName := internal.Underscore(f.Name)
func (t *Table) newField(f reflect.StructField, prefix string, index []int) *Field {
tag := tagparser.Parse(f.Tag.Get("bun"))
if prefix, ok := tag.Option("embed"); ok {
fieldType := indirectType(f.Type)
if fieldType.Kind() != reflect.Struct {
panic(fmt.Errorf("bun: embed %s.%s: got %s, wanted reflect.Struct",
t.TypeName, f.Name, fieldType.Kind()))
}
t.addFields(fieldType, prefix, withIndex(index, f.Index))
return nil
}
sqlName := internal.Underscore(f.Name)
if tag.Name != "" && tag.Name != sqlName {
if isKnownFieldOption(tag.Name) {
internal.Warn.Printf(
@ -328,10 +334,10 @@ func (t *Table) newField(f reflect.StructField, index []int) *Field {
}
sqlName = tag.Name
}
if s, ok := tag.Option("column"); ok {
sqlName = s
}
sqlName = prefix + sqlName
for name := range tag.Options {
if !isKnownFieldOption(name) {
@ -339,7 +345,7 @@ func (t *Table) newField(f reflect.StructField, index []int) *Field {
}
}
index = append(index, f.Index...)
index = withIndex(index, f.Index)
if field := t.fieldWithLock(sqlName); field != nil {
if indexEqual(field.Index, index) {
return field
@ -795,7 +801,7 @@ func (t *Table) inlineFields(field *Field, seen map[reflect.Type]struct{}) {
f.GoName = field.GoName + "_" + f.GoName
f.Name = field.Name + "__" + f.Name
f.SQLName = t.quoteIdent(f.Name)
f.Index = appendNew(field.Index, f.Index...)
f.Index = withIndex(field.Index, f.Index)
t.fieldsMapMu.Lock()
if _, ok := t.FieldMap[f.Name]; !ok {
@ -834,7 +840,7 @@ func (t *Table) AppendNamedArg(
fmter Formatter, b []byte, name string, strct reflect.Value,
) ([]byte, bool) {
if field, ok := t.FieldMap[name]; ok {
return fmter.appendArg(b, field.Value(strct).Interface()), true
return field.AppendValue(fmter, b, strct), true
}
return b, false
}
@ -853,13 +859,6 @@ func (t *Table) quoteIdent(s string) Safe {
return Safe(NewFormatter(t.dialect).AppendIdent(nil, s))
}
func appendNew(dst []int, src ...int) []int {
cp := make([]int, len(dst)+len(src))
copy(cp, dst)
copy(cp[len(dst):], src)
return cp
}
func isKnownTableOption(name string) bool {
switch name {
case "table", "alias", "select":
@ -991,3 +990,10 @@ func softDeleteFieldUpdaterFallback(field *Field) func(fv reflect.Value, tm time
return field.ScanWithCheck(fv, tm)
}
}
func withIndex(a, b []int) []int {
dest := make([]int, 0, len(a)+len(b))
dest = append(dest, a...)
dest = append(dest, b...)
return dest
}