mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 18:12:25 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Clean up/uncache remote media (#407)

Browse source 
* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 • committed by GitHub
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions
Download patch file Download diff file Expand all files Collapse all files

101
vendor/github.com/uptrace/bun/query_base.go generated vendored
View file

@ -43,6 +43,7 @@ var (
// IDB is a common interface for *bun.DB, bun.Conn, and bun.Tx.
type IDB interface {
IConn
Dialect() schema.Dialect
NewValues(model interface{}) *ValuesQuery
NewSelect() *SelectQuery
@ -59,9 +60,9 @@ type IDB interface {
}
var (
_ IConn = (*DB)(nil)
_ IConn = (*Conn)(nil)
_ IConn = (*Tx)(nil)
_ IDB = (*DB)(nil)
_ IDB = (*Conn)(nil)
_ IDB = (*Tx)(nil)
)
type baseQuery struct {
@ -74,10 +75,10 @@ type baseQuery struct {
tableModel TableModel
table *schema.Table
with []withQuery
modelTable schema.QueryWithArgs
tables []schema.QueryWithArgs
columns []schema.QueryWithArgs
with []withQuery
modelTableName schema.QueryWithArgs
tables []schema.QueryWithArgs
columns []schema.QueryWithArgs
flags internal.Flag
}
@ -86,13 +87,6 @@ func (q *baseQuery) DB() *DB {
return q.db
}
type query interface {
GetModel() Model
GetTableName() string
}
var _ query = (*baseQuery)(nil)
func (q *baseQuery) GetModel() Model {
return q.model
}
@ -103,15 +97,16 @@ func (q *baseQuery) GetTableName() string {
}
for _, wq := range q.with {
if v, ok := wq.query.(query); ok {
if v, ok := wq.query.(Query); ok {
if model := v.GetModel(); model != nil {
return v.GetTableName()
}
}
}
if q.modelTable.Query != "" {
return q.modelTable.Query
if q.modelTableName.Query != "" {
b, _ := q.modelTableName.AppendQuery(q.db.fmter, nil)
return string(b)
}
if len(q.tables) > 0 {
return q.tables[0].Query
@ -304,8 +299,8 @@ func (q *baseQuery) _excludeColumn(column string) bool {
//------------------------------------------------------------------------------
func (q *baseQuery) modelHasTableName() bool {
if !q.modelTable.IsZero() {
return q.modelTable.Query != ""
if !q.modelTableName.IsZero() {
return q.modelTableName.Query != ""
}
return q.table != nil
}
@ -332,8 +327,8 @@ func (q *baseQuery) _appendTables(
startLen := len(b)
if q.modelHasTableName() {
if !q.modelTable.IsZero() {
b, err = q.modelTable.AppendQuery(fmter, b)
if !q.modelTableName.IsZero() {
b, err = q.modelTableName.AppendQuery(fmter, b)
if err != nil {
return nil, err
}
@ -372,8 +367,8 @@ func (q *baseQuery) appendFirstTableWithAlias(
func (q *baseQuery) _appendFirstTable(
fmter schema.Formatter, b []byte, withAlias bool,
) ([]byte, error) {
if !q.modelTable.IsZero() {
return q.modelTable.AppendQuery(fmter, b)
if !q.modelTableName.IsZero() {
return q.modelTableName.AppendQuery(fmter, b)
}
if q.table != nil {
@ -473,7 +468,7 @@ func (q *baseQuery) scan(
model Model,
hasDest bool,
) (sql.Result, error) {
ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, q.model)
ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, query, q.model)
rows, err := q.conn.QueryContext(ctx, query)
if err != nil {
@ -503,16 +498,10 @@ func (q *baseQuery) exec(
iquery Query,
query string,
) (sql.Result, error) {
ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, q.model)
ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, query, q.model)
res, err := q.conn.ExecContext(ctx, query)
if err != nil {
q.db.afterQuery(ctx, event, nil, err)
return res, err
}
q.db.afterQuery(ctx, event, res, err)
return res, nil
q.db.afterQuery(ctx, event, nil, err)
return res, err
}
//------------------------------------------------------------------------------
@ -607,34 +596,30 @@ func (q *whereBaseQuery) addWhereCols(cols []string) {
q.setErr(err)
return
}
var fields []*schema.Field
if cols == nil {
if err := q.table.CheckPKs(); err != nil {
q.setErr(err)
return
}
fields = q.table.PKs
} else {
fields = make([]*schema.Field, len(cols))
for i, col := range cols {
field, err := q.table.Field(col)
if err != nil {
q.setErr(err)
return
}
fields[i] = field
}
}
if q.whereFields != nil {
err := errors.New("bun: WherePK can only be called once")
q.setErr(err)
return
}
q.whereFields = fields
if cols == nil {
if err := q.table.CheckPKs(); err != nil {
q.setErr(err)
return
}
q.whereFields = q.table.PKs
return
}
q.whereFields = make([]*schema.Field, len(cols))
for i, col := range cols {
field, err := q.table.Field(col)
if err != nil {
q.setErr(err)
return
}
q.whereFields[i] = field
}
}
func (q *whereBaseQuery) mustAppendWhere(
@ -951,6 +936,7 @@ func (q setQuery) appendSet(fmter schema.Formatter, b []byte) (_ []byte, err err
//------------------------------------------------------------------------------
type cascadeQuery struct {
cascade bool
restrict bool
}
@ -958,10 +944,11 @@ func (q cascadeQuery) appendCascade(fmter schema.Formatter, b []byte) []byte {
if !fmter.HasFeature(feature.TableCascade) {
return b
}
if q.cascade {
b = append(b, " CASCADE"...)
}
if q.restrict {
b = append(b, " RESTRICT"...)
} else {
b = append(b, " CASCADE"...)
}
return b
}