mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-10-29 09:52:26 -05:00
[bugfix/chore] oauth entropy fix + media cleanup tasks rewrite (#1853)
This commit is contained in:
kim
GitHub
parent
c4cf6326d8
commit
9a22102fa8
38 changed files with 2076 additions and 1090 deletions
61
cmd/gotosocial/action/admin/media/prune/all.go
Normal file
61
cmd/gotosocial/action/admin/media/prune/all.go
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
// GoToSocial
|
||||
// Copyright (C) GoToSocial Authors admin@gotosocial.org
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package prune
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/config"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
)
|
||||
|
||||
// All performs all media clean actions
|
||||
var All action.GTSAction = func(ctx context.Context) error {
|
||||
// Setup pruning utilities.
|
||||
prune, err := setupPrune(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// Ensure pruner gets shutdown on exit.
|
||||
if err := prune.shutdown(ctx); err != nil {
|
||||
log.Error(ctx, err)
|
||||
}
|
||||
}()
|
||||
|
||||
if config.GetAdminMediaPruneDryRun() {
|
||||
log.Info(ctx, "prune DRY RUN")
|
||||
ctx = gtscontext.SetDryRun(ctx)
|
||||
}
|
||||
|
||||
days := config.GetMediaRemoteCacheDays()
|
||||
|
||||
// Perform the actual pruning with logging.
|
||||
prune.cleaner.Media().All(ctx, days)
|
||||
prune.cleaner.Emoji().All(ctx)
|
||||
|
||||
// Perform a cleanup of storage (for removed local dirs).
|
||||
if err := prune.storage.Storage.Clean(ctx); err != nil {
|
||||
log.Error(ctx, "error cleaning storage: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -21,8 +21,10 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/internal/cleaner"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/media"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/state"
|
||||
gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage"
|
||||
|
|
@ -32,6 +34,7 @@ type prune struct {
|
|||
dbService db.DB
|
||||
storage *gtsstorage.Driver
|
||||
manager *media.Manager
|
||||
cleaner *cleaner.Cleaner
|
||||
state *state.State
|
||||
}
|
||||
|
||||
|
|
@ -59,25 +62,31 @@ func setupPrune(ctx context.Context) (*prune, error) {
|
|||
//nolint:contextcheck
|
||||
manager := media.NewManager(&state)
|
||||
|
||||
//nolint:contextcheck
|
||||
cleaner := cleaner.New(&state)
|
||||
|
||||
return &prune{
|
||||
dbService: dbService,
|
||||
storage: storage,
|
||||
manager: manager,
|
||||
cleaner: cleaner,
|
||||
state: &state,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *prune) shutdown(ctx context.Context) error {
|
||||
var errs gtserror.MultiError
|
||||
|
||||
if err := p.storage.Close(); err != nil {
|
||||
return fmt.Errorf("error closing storage backend: %w", err)
|
||||
errs.Appendf("error closing storage backend: %v", err)
|
||||
}
|
||||
|
||||
if err := p.dbService.Stop(ctx); err != nil {
|
||||
return fmt.Errorf("error closing dbservice: %w", err)
|
||||
errs.Appendf("error stopping database: %v", err)
|
||||
}
|
||||
|
||||
p.state.Workers.Stop()
|
||||
p.state.Caches.Stop()
|
||||
|
||||
return nil
|
||||
return errs.Combine()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,32 +19,40 @@ package prune
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/config"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
)
|
||||
|
||||
// Orphaned prunes orphaned media from storage.
|
||||
var Orphaned action.GTSAction = func(ctx context.Context) error {
|
||||
// Setup pruning utilities.
|
||||
prune, err := setupPrune(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dry := config.GetAdminMediaPruneDryRun()
|
||||
defer func() {
|
||||
// Ensure pruner gets shutdown on exit.
|
||||
if err := prune.shutdown(ctx); err != nil {
|
||||
log.Error(ctx, err)
|
||||
}
|
||||
}()
|
||||
|
||||
pruned, err := prune.manager.PruneOrphaned(ctx, dry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error pruning: %s", err)
|
||||
if config.GetAdminMediaPruneDryRun() {
|
||||
log.Info(ctx, "prune DRY RUN")
|
||||
ctx = gtscontext.SetDryRun(ctx)
|
||||
}
|
||||
|
||||
if dry /* dick heyyoooooo */ {
|
||||
log.Infof(ctx, "DRY RUN: %d items are orphaned and eligible to be pruned", pruned)
|
||||
} else {
|
||||
log.Infof(ctx, "%d orphaned items were pruned", pruned)
|
||||
// Perform the actual pruning with logging.
|
||||
prune.cleaner.Media().LogPruneOrphaned(ctx)
|
||||
|
||||
// Perform a cleanup of storage (for removed local dirs).
|
||||
if err := prune.storage.Storage.Clean(ctx); err != nil {
|
||||
log.Error(ctx, "error cleaning storage: %v", err)
|
||||
}
|
||||
|
||||
return prune.shutdown(ctx)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,39 +19,44 @@ package prune
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/superseriousbusiness/gotosocial/cmd/gotosocial/action"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/config"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/gtscontext"
|
||||
"github.com/superseriousbusiness/gotosocial/internal/log"
|
||||
)
|
||||
|
||||
// Remote prunes old and/or unused remote media.
|
||||
var Remote action.GTSAction = func(ctx context.Context) error {
|
||||
// Setup pruning utilities.
|
||||
prune, err := setupPrune(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dry := config.GetAdminMediaPruneDryRun()
|
||||
defer func() {
|
||||
// Ensure pruner gets shutdown on exit.
|
||||
if err := prune.shutdown(ctx); err != nil {
|
||||
log.Error(ctx, err)
|
||||
}
|
||||
}()
|
||||
|
||||
pruned, err := prune.manager.PruneUnusedRemote(ctx, dry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error pruning: %w", err)
|
||||
if config.GetAdminMediaPruneDryRun() {
|
||||
log.Info(ctx, "prune DRY RUN")
|
||||
ctx = gtscontext.SetDryRun(ctx)
|
||||
}
|
||||
|
||||
uncached, err := prune.manager.UncacheRemote(ctx, config.GetMediaRemoteCacheDays(), dry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error pruning: %w", err)
|
||||
t := time.Now().Add(-24 * time.Hour * time.Duration(config.GetMediaRemoteCacheDays()))
|
||||
|
||||
// Perform the actual pruning with logging.
|
||||
prune.cleaner.Media().LogPruneUnused(ctx)
|
||||
prune.cleaner.Media().LogUncacheRemote(ctx, t)
|
||||
|
||||
// Perform a cleanup of storage (for removed local dirs).
|
||||
if err := prune.storage.Storage.Clean(ctx); err != nil {
|
||||
log.Error(ctx, "error cleaning storage: %v", err)
|
||||
}
|
||||
|
||||
total := pruned + uncached
|
||||
|
||||
if dry /* dick heyyoooooo */ {
|
||||
log.Infof(ctx, "DRY RUN: %d remote items are unused/stale and eligible to be pruned", total)
|
||||
} else {
|
||||
log.Infof(ctx, "%d unused/stale remote items were pruned", pruned)
|
||||
}
|
||||
|
||||
return prune.shutdown(ctx)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -170,7 +170,7 @@ func adminCommands() *cobra.Command {
|
|||
|
||||
adminMediaCmd := &cobra.Command{
|
||||
Use: "media",
|
||||
Short: "admin commands related stored media attachments/emojis",
|
||||
Short: "admin commands related to stored media / emojis",
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -178,7 +178,7 @@ func adminCommands() *cobra.Command {
|
|||
*/
|
||||
adminMediaPruneCmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "admin commands for pruning unused/orphaned media from storage",
|
||||
Short: "admin commands for pruning media from storage",
|
||||
}
|
||||
|
||||
adminMediaPruneOrphanedCmd := &cobra.Command{
|
||||
|
|
@ -196,7 +196,7 @@ func adminCommands() *cobra.Command {
|
|||
|
||||
adminMediaPruneRemoteCmd := &cobra.Command{
|
||||
Use: "remote",
|
||||
Short: "prune unused/stale remote media from storage, older than given number of days",
|
||||
Short: "prune unused / stale media from storage, older than given number of days",
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return preRun(preRunArgs{cmd: cmd})
|
||||
},
|
||||
|
|
@ -207,6 +207,19 @@ func adminCommands() *cobra.Command {
|
|||
config.AddAdminMediaPrune(adminMediaPruneRemoteCmd)
|
||||
adminMediaPruneCmd.AddCommand(adminMediaPruneRemoteCmd)
|
||||
|
||||
adminMediaPruneAllCmd := &cobra.Command{
|
||||
Use: "all",
|
||||
Short: "perform all media and emoji prune / cleaning commands",
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
return preRun(preRunArgs{cmd: cmd})
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return run(cmd.Context(), prune.All)
|
||||
},
|
||||
}
|
||||
config.AddAdminMediaPrune(adminMediaPruneAllCmd)
|
||||
adminMediaPruneCmd.AddCommand(adminMediaPruneAllCmd)
|
||||
|
||||
adminMediaCmd.AddCommand(adminMediaPruneCmd)
|
||||
|
||||
adminCmd.AddCommand(adminMediaCmd)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue