[feature] Clean up/uncache remote media (#407)

* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully

vendor/modernc.org/ccgo/v3/lib/Makefile

@@ -74,6 +74,7 @@ test_windows_amd64:
 build_all_targets:
 	GOOS=darwin GOARCH=amd64 go test -c -o /dev/null
 	GOOS=darwin GOARCH=arm64 go test -c -o /dev/null
+	GOOS=freebsd GOARCH=386 go test -c -o /dev/null
 	GOOS=freebsd GOARCH=amd64 go test -c -o /dev/null
 	GOOS=linux GOARCH=386 go test -c -o /dev/null
 	GOOS=linux GOARCH=amd64 go test -c -o /dev/null
@@ -81,6 +82,7 @@ build_all_targets:
 	GOOS=linux GOARCH=arm64 go test -c -o /dev/null
 	GOOS=linux GOARCH=s390x go test -c -o /dev/null
 	GOOS=netbsd GOARCH=amd64 go test -c -o /dev/null
+	GOOS=openbsd GOARCH=amd64 go test -c -o /dev/null
 	GOOS=windows GOARCH=386 go test -c -o /dev/null
 	GOOS=windows GOARCH=amd64 go test -c -o /dev/null
 

vendor/modernc.org/ccgo/v3/lib/ccgo.go

@@ -211,6 +211,7 @@ double __builtin_huge_val (void);
 double __builtin_inf (void);
 double __builtin_nan (const char *str);
 float __builtin_copysignf ( float x, float y );
+float __builtin_fabsf(float x);
 float __builtin_huge_valf (void);
 float __builtin_inff (void);
 float __builtin_nanf (const char *str);
@@ -221,6 +222,7 @@ int __builtin__snprintf_chk(char * str, size_t maxlen, int flag, size_t strlen,
 int __builtin_abs(int j);
 int __builtin_add_overflow();
 int __builtin_clz (unsigned);
+int __builtin_isunordered(double x, double y);
 int __builtin_clzl (unsigned long);
 int __builtin_clzll (unsigned long long);
 int __builtin_constant_p_impl(int, ...);
@@ -236,6 +238,7 @@ int __builtin_sprintf(char *str, const char *format, ...);
 int __builtin_strcmp(const char *s1, const char *s2);
 int __builtin_sub_overflow();
 long __builtin_expect (long exp, long c);
+long double __builtin_fabsl(long double x);
 long double __builtin_nanl (const char *str);
 long long __builtin_llabs(long long j);
 size_t __builtin_object_size (void * ptr, int type);
@@ -279,6 +282,10 @@ void __ccgo_va_start(__builtin_va_list ap);
 unsigned __sync_add_and_fetch_uint32(unsigned*, unsigned);
 unsigned __sync_sub_and_fetch_uint32(unsigned*, unsigned);
 
+#ifdef __APPLE__
+int (*__darwin_check_fd_set_overflow)(int, void *, int);
+#endif
+
 `
 	defaultCrt = "modernc.org/libc"
 )
@@ -1483,6 +1490,22 @@ out:
 		command = append([]string{sh, "-c"}, join(" ", command[0], "SHELL='sh -x'", command[1:]))
 		cmd = exec.Command(command[0], command[1:]...)
 		parser = makeXParser
+	case "openbsd":
+		switch command[0] {
+		case "make", "gmake":
+			// ok
+		default:
+			return fmt.Errorf("usupported build command: %s", command[0])
+		}
+
+		sh, err := exec.LookPath("sh")
+		if err != nil {
+			return err
+		}
+
+		command = append([]string{sh, "-c"}, join(" ", command[0], "SHELL='sh -x'", command[1:]))
+		cmd = exec.Command(command[0], command[1:]...)
+		parser = makeXParser2
 	case "windows":
 		if command[0] != "make" {
 			return fmt.Errorf("usupported build command: %s", command[0])
@@ -1573,7 +1596,7 @@ func isCreateArchive(s string) bool {
 	b := []byte(s)
 	sort.Slice(b, func(i, j int) bool { return b[i] < b[j] })
 	switch string(b) {
-	case "cq", "cr", "crs", "cru":
+	case "cq", "cr", "crs", "cru", "r":
 		return true
 	}
 	return false
@@ -1608,7 +1631,48 @@ func makeXParser(s string) (r []string, err error) {
 	}
 
 	s = s[1:]
+	if dmesgs {
+		dmesg("%v: source line `%s`, caller %v:", origin(1), s, origin(2))
+	}
 	r, err = shellquote.Split(s)
+	if dmesgs {
+		dmesg("%v: shellquote.Split -> %v %[2]q, %v", origin(1), r, err)
+	}
+	if err != nil {
+		if strings.Contains(err.Error(), "Unterminated single-quoted string") {
+			return nil, nil // ignore
+		}
+	}
+	if len(r) != 0 && filepath.Base(r[0]) == "libtool" {
+		r[0] = "libtool"
+	}
+	return r, err
+}
+
+func makeXParser2(s string) (r []string, err error) {
+	s = strings.TrimSpace(s)
+	switch {
+	case strings.HasPrefix(s, "libtool: link: ar "):
+		s = s[len("libtool: link:"):]
+	case strings.HasPrefix(s, "libtool: compile: "):
+		s = s[len("libtool: compile:"):]
+		for strings.HasPrefix(s, "  ") {
+			s = s[1:]
+		}
+	default:
+		var n int
+		if n, s = hasPlusPrefix(s); n != 0 {
+			return nil, nil
+		}
+	}
+
+	if dmesgs {
+		dmesg("%v: source line `%s`, caller %v:", origin(1), s, origin(2))
+	}
+	r, err = shellquote.Split(s)
+	if dmesgs {
+		dmesg("%v: shellquote.Split -> %v %[2]q, %v", origin(1), r, err)
+	}
 	if err != nil {
 		if strings.Contains(err.Error(), "Unterminated single-quoted string") {
 			return nil, nil // ignore
@@ -1802,26 +1866,29 @@ func (it *cdbItem) sources(cc, ar string) (r []string) {
 }
 
 type cdbMakeWriter struct {
-	b      bytes.Buffer
-	cc     string
 	ar     string
 	arBase string
+	b      bytes.Buffer
+	cc     string
 	dir    string
 	err    error
 	it     cdbItem
 	parser func(s string) ([]string, error)
+	prefix string
 	sc     *bufio.Scanner
+	t      *Task
 	w      *cdbWriter
 }
 
 func (t *Task) newCdbMakeWriter(w *cdbWriter, dir string, parser func(s string) ([]string, error)) *cdbMakeWriter {
 	const sz = 1 << 16
 	r := &cdbMakeWriter{
-		cc:     t.ccLookPath,
 		ar:     t.arLookPath,
 		arBase: filepath.Base(t.arLookPath),
+		cc:     t.ccLookPath,
 		dir:    dir,
 		parser: parser,
+		t:      t,
 		w:      w,
 	}
 	r.sc = bufio.NewScanner(&r.b)
@@ -1842,7 +1909,15 @@ func (w *cdbMakeWriter) Write(b []byte) (int, error) {
 			panic(todo("internal error"))
 		}
 
-		s := strings.TrimSpace(w.sc.Text())
+		s := w.sc.Text()
+		if strings.HasSuffix(s, "\\") {
+			w.prefix += s[:len(s)-1]
+			continue
+		}
+
+		s = w.prefix + s
+		w.prefix = ""
+		s = strings.TrimSpace(s)
 		if edx := strings.Index(s, "Entering directory"); edx >= 0 {
 			s = s[edx+len("Entering directory"):]
 			s = strings.TrimSpace(s)
@@ -1875,7 +1950,13 @@ func (w *cdbMakeWriter) Write(b []byte) (int, error) {
 			continue
 		}
 
+		if dmesgs {
+			dmesg("%v: source line `%s`", origin(1), s)
+		}
 		args, err := w.parser(s)
+		if dmesgs {
+			dmesg("%v: parser -> %v %[2]q, %v", origin(1), args, err)
+		}
 		if err != nil {
 			w.fail(err)
 			continue
@@ -1892,16 +1973,25 @@ func (w *cdbMakeWriter) Write(b []byte) (int, error) {
 		err = nil
 		switch args[0] {
 		case w.cc:
+			if w.t.verboseCompiledb {
+				fmt.Printf("source line: %q\n", s)
+			}
 			fmt.Printf("CCGO CC: %q\n", args)
 			err = w.handleGCC(args)
 		case w.ar:
 			fallthrough
 		case w.arBase:
 			if isCreateArchive(args[1]) {
+				if w.t.verboseCompiledb {
+					fmt.Printf("source line: %q\n", s)
+				}
 				fmt.Printf("CCGO AR: %q\n", args)
 				err = w.handleAR(args)
 			}
 		case "libtool":
+			if w.t.verboseCompiledb {
+				fmt.Printf("source line: %q\n", s)
+			}
 			fmt.Printf("CCGO LIBTOOL: %q\n", args)
 			err = w.handleLibtool(args)
 		}

vendor/modernc.org/ccgo/v3/lib/dmesg.go

@@ -32,7 +32,8 @@ func dmesg(s string, args ...interface{}) {
 	if s == "" {
 		s = strings.Repeat("%v ", len(args))
 	}
-	s = fmt.Sprintf(pid+s, args...)
+	s = fmt.Sprintf(s, args...)
+	s = pid + s
 	switch {
 	case len(s) != 0 && s[len(s)-1] == '\n':
 		fmt.Fprint(logf, s)

vendor/modernc.org/ccgo/v3/lib/go.go

@@ -3117,6 +3117,9 @@ func (p *project) declaratorDecay(n cc.Node, f *function, d *cc.Declarator, t cc
 				return
 			}
 
+			if !local.isPinned {
+				p.err(n, "%v: %v: missed pinning", n.Position(), d.Position(), d.Name())
+			}
 			p.w("(%s%s)/* &%s[0] */", f.bpName, nonZeroUintptr(local.off), local.name)
 			return
 		}
@@ -3663,7 +3666,7 @@ func (p *project) declaratorAddrOfNormal(n cc.Node, f *function, d *cc.Declarato
 		x.used = true
 		p.w("uintptr(unsafe.Pointer(&%sX%s))", x.qualifier, d.Name())
 	default:
-		panic(todo("%v: %v: %q %T", n.Position(), p.pos(d), d.Name(), x))
+		p.err(n, "undefined: %s", d.Name())
 	}
 }
 
@@ -8135,6 +8138,11 @@ func (p *project) castExpressionValue(f *function, n *cc.CastExpression, t cc.Ty
 	case cc.CastExpressionUnary: // UnaryExpression
 		p.unaryExpression(f, n.UnaryExpression, t, mode, flags)
 	case cc.CastExpressionCast: // '(' TypeName ')' CastExpression
+		if f != nil && p.pass1 && n.TypeName.Type().IsIntegerType() && n.CastExpression.Operand.Type().Kind() == cc.Array {
+			if d := n.CastExpression.Declarator(); d != nil {
+				f.pin(n, d)
+			}
+		}
 		switch k := p.opKind(f, n.CastExpression, n.CastExpression.Operand.Type()); k {
 		case opNormal, opBitfield:
 			p.castExpressionValueNormal(f, n, t, mode, flags)
@@ -8168,8 +8176,12 @@ func (p *project) castExpressionValueFunction(f *function, n *cc.CastExpression,
 		switch {
 		case tn.Kind() == cc.Ptr && t.Kind() == cc.Ptr:
 			p.castExpression(f, n.CastExpression, op.Type(), exprValue, flags)
+		case tn.IsIntegerType():
+			p.w("%s(", p.typ(n, tn))
+			p.castExpression(f, n.CastExpression, op.Type(), exprValue, flags)
+			p.w(")")
 		default:
-			panic(todo("", n.Position()))
+			panic(todo("%v: tn %v expr %v", n.Position(), tn, op.Type()))
 		}
 	default:
 		panic(todo("%v: %v -> %v -> %v", p.pos(n), op.Type(), tn, t))
@@ -12557,7 +12569,7 @@ func (p *project) iterationStatement(f *function, n *cc.IterationStatement) {
 			break
 		}
 
-		v := "ok"
+		v := "__ccgo"
 		if !p.pass1 {
 			v = f.scope.take(cc.String(v))
 		}