mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 16:02:25 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Clean up/uncache remote media (#407)

Browse source 
* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 • committed by GitHub
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions
Download patch file Download diff file Expand all files Collapse all files

38
vendor/github.com/uptrace/bun/query_select.go generated vendored
View file

@ -34,6 +34,8 @@ type SelectQuery struct {
union []union
}
var _ Query = (*SelectQuery)(nil)
func NewSelectQuery(db *DB) *SelectQuery {
return &SelectQuery{
whereBaseQuery: whereBaseQuery{
@ -90,7 +92,7 @@ func (q *SelectQuery) TableExpr(query string, args ...interface{}) *SelectQuery
}
func (q *SelectQuery) ModelTableExpr(query string, args ...interface{}) *SelectQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}
@ -342,9 +344,9 @@ func (q *SelectQuery) selectJoins(ctx context.Context, joins []relationJoin) err
case schema.HasOneRelation, schema.BelongsToRelation:
err = q.selectJoins(ctx, j.JoinModel.getJoins())
case schema.HasManyRelation:
err = j.selectMany(ctx, q.db.NewSelect())
err = j.selectMany(ctx, q.db.NewSelect().Conn(q.conn))
case schema.ManyToManyRelation:
err = j.selectM2M(ctx, q.db.NewSelect())
err = j.selectM2M(ctx, q.db.NewSelect().Conn(q.conn))
default:
panic("not reached")
}
@ -369,6 +371,10 @@ func (q *SelectQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, e
func (q *SelectQuery) appendQuery(
fmter schema.Formatter, b []byte, count bool,
) (_ []byte, err error) {
if q.err != nil {
return nil, q.err
}
fmter = formatterWithModel(fmter, q)
cteCount := count && (len(q.group) > 0 || q.distinctOn != nil)
@ -767,7 +773,7 @@ func (q *SelectQuery) Count(ctx context.Context) (int, error) {
}
query := internal.String(queryBytes)
ctx, event := q.db.beforeQuery(ctx, qq, query, nil, q.model)
ctx, event := q.db.beforeQuery(ctx, qq, query, nil, query, q.model)
var num int
err = q.conn.QueryRowContext(ctx, query).Scan(&num)
@ -778,6 +784,13 @@ func (q *SelectQuery) Count(ctx context.Context) (int, error) {
}
func (q *SelectQuery) ScanAndCount(ctx context.Context, dest ...interface{}) (int, error) {
if _, ok := q.conn.(*DB); ok {
return q.scanAndCountConc(ctx, dest...)
}
return q.scanAndCountSeq(ctx, dest...)
}
func (q *SelectQuery) scanAndCountConc(ctx context.Context, dest ...interface{}) (int, error) {
var count int
var wg sync.WaitGroup
var mu sync.Mutex
@ -817,6 +830,21 @@ func (q *SelectQuery) ScanAndCount(ctx context.Context, dest ...interface{}) (in
return count, firstErr
}
func (q *SelectQuery) scanAndCountSeq(ctx context.Context, dest ...interface{}) (int, error) {
var firstErr error
if q.limit >= 0 {
firstErr = q.Scan(ctx, dest...)
}
count, err := q.Count(ctx)
if err != nil && firstErr == nil {
firstErr = err
}
return count, firstErr
}
func (q *SelectQuery) Exists(ctx context.Context) (bool, error) {
if q.err != nil {
return false, q.err
@ -830,7 +858,7 @@ func (q *SelectQuery) Exists(ctx context.Context) (bool, error) {
}
query := internal.String(queryBytes)
ctx, event := q.db.beforeQuery(ctx, qq, query, nil, q.model)
ctx, event := q.db.beforeQuery(ctx, qq, query, nil, query, q.model)
var exists bool
err = q.conn.QueryRowContext(ctx, query).Scan(&exists)