mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-30 16:52:25 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature] Clean up/uncache remote media (#407)

Browse source 
* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 • committed by GitHub
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions
Download patch file Download diff file Expand all files Collapse all files

55
vendor/github.com/uptrace/bun/migrate/migrator.go generated vendored
View file

@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"io/ioutil"
"log"
"path/filepath"
"regexp"
"time"
@ -59,11 +58,16 @@ func (m *Migrator) DB() *bun.DB {
// MigrationsWithStatus returns migrations with status in ascending order.
func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, error) {
sorted, _, err := m.migrationsWithStatus(ctx)
return sorted, err
}
func (m *Migrator) migrationsWithStatus(ctx context.Context) (MigrationSlice, int64, error) {
sorted := m.migrations.Sorted()
applied, err := m.selectAppliedMigrations(ctx)
if err != nil {
return nil, err
return nil, 0, err
}
appliedMap := migrationMap(applied)
@ -76,7 +80,7 @@ func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, er
}
}
return sorted, nil
return sorted, applied.LastGroupID(), nil
}
func (m *Migrator) Init(ctx context.Context) error {
@ -128,7 +132,7 @@ func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*Migra
}
defer m.Unlock(ctx) //nolint:errcheck
migrations, err := m.MigrationsWithStatus(ctx)
migrations, lastGroupID, err := m.migrationsWithStatus(ctx)
if err != nil {
return nil, err
}
@ -139,21 +143,22 @@ func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*Migra
if len(group.Migrations) == 0 {
return group, nil
}
group.ID = migrations.LastGroupID() + 1
group.ID = lastGroupID + 1
for i := range group.Migrations {
migration := &group.Migrations[i]
migration.GroupID = group.ID
// Always mark migration as applied so the rollback has a chance to fix the database.
if err := m.MarkApplied(ctx, migration); err != nil {
return nil, err
}
if !cfg.nop && migration.Up != nil {
if err := migration.Up(ctx, m.db); err != nil {
return group, err
}
}
if err := m.MarkApplied(ctx, migration); err != nil {
return nil, err
}
}
return group, nil
@ -195,36 +200,6 @@ func (m *Migrator) Rollback(ctx context.Context, opts ...MigrationOption) (*Migr
return lastGroup, nil
}
type MigrationStatus struct {
Migrations MigrationSlice
NewMigrations MigrationSlice
LastGroup *MigrationGroup
}
func (m *Migrator) Status(ctx context.Context) (*MigrationStatus, error) {
log.Printf(
"DEPRECATED: bun: replace Status(ctx) with " +
"MigrationsWithStatus(ctx)")
migrations, err := m.MigrationsWithStatus(ctx)
if err != nil {
return nil, err
}
return &MigrationStatus{
Migrations: migrations,
NewMigrations: migrations.Unapplied(),
LastGroup: migrations.LastGroup(),
}, nil
}
func (m *Migrator) MarkCompleted(ctx context.Context) (*MigrationGroup, error) {
log.Printf(
"DEPRECATED: bun: replace MarkCompleted(ctx) with " +
"Migrate(ctx, migrate.WithNopMigration())")
return m.Migrate(ctx, WithNopMigration())
}
type goMigrationConfig struct {
packageName string
}
@ -320,7 +295,7 @@ func (m *Migrator) genMigrationName(name string) (string, error) {
return fmt.Sprintf("%s_%s", version, name), nil
}
// MarkApplied marks the migration as applied (applied).
// MarkApplied marks the migration as applied (completed).
func (m *Migrator) MarkApplied(ctx context.Context, migration *Migration) error {
_, err := m.db.NewInsert().Model(migration).
ModelTableExpr(m.table).