mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2025-10-28 13:52:25 -05:00
[bugfix] media CLI list missing thumbs and statics (#4379)
refactors some of the media CLI listing code, and updates it to include both attachment thumbnails and emoji static images, which previously were missing. addresses https://codeberg.org/superseriousbusiness/gotosocial/issues/4370 though in a follow-up i'll add support for regenerating attachment thumbnails and emoji static images. Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4379 Co-authored-by: kim <grufwub@gmail.com> Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim
kim
parent
e9b7e977a5
commit
a6bb45e5e4
1 changed files with 165 additions and 149 deletions
|
|
@ -18,12 +18,10 @@
|
|||
package media
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"code.superseriousbusiness.org/gotosocial/cmd/gotosocial/action"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/config"
|
||||
|
|
@ -33,84 +31,168 @@ import (
|
|||
"code.superseriousbusiness.org/gotosocial/internal/log"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/paging"
|
||||
"code.superseriousbusiness.org/gotosocial/internal/state"
|
||||
"codeberg.org/gruf/go-byteutil"
|
||||
"codeberg.org/gruf/go-fastpath/v2"
|
||||
)
|
||||
|
||||
// check function conformance.
|
||||
var _ action.GTSAction = ListAttachments
|
||||
var _ action.GTSAction = ListEmojis
|
||||
|
||||
// ListAttachments lists local, remote, or all attachment paths.
|
||||
func ListAttachments(ctx context.Context) error {
|
||||
list, err := setupList(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// Ensure lister gets shutdown on exit.
|
||||
if err := list.shutdown(); err != nil {
|
||||
log.Errorf(ctx, "error shutting down: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// List attachment media paths from db.
|
||||
return list.ListAttachmentPaths(ctx)
|
||||
}
|
||||
|
||||
// ListEmojis lists local, remote, or all emoji filepaths.
|
||||
func ListEmojis(ctx context.Context) error {
|
||||
list, err := setupList(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// Ensure lister gets shutdown on exit.
|
||||
if err := list.shutdown(); err != nil {
|
||||
log.Errorf(ctx, "error shutting down: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// List emoji media paths from db.
|
||||
return list.ListEmojiPaths(ctx)
|
||||
}
|
||||
|
||||
type list struct {
|
||||
dbService db.DB
|
||||
state *state.State
|
||||
page paging.Page
|
||||
localOnly bool
|
||||
remoteOnly bool
|
||||
out *bufio.Writer
|
||||
}
|
||||
|
||||
// Get a list of attachment using a custom filter
|
||||
func (l *list) GetAllAttachmentPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) {
|
||||
res := make([]string, 0, 100)
|
||||
func (l *list) ListAttachmentPaths(ctx context.Context) error {
|
||||
// Page reused for iterative
|
||||
// attachment queries, with
|
||||
// predefined limit.
|
||||
var page paging.Page
|
||||
page.Limit = 500
|
||||
|
||||
// Storage base path, used for path building.
|
||||
basePath := config.GetStorageLocalBasePath()
|
||||
|
||||
for {
|
||||
// Get the next page of media attachments up to max ID.
|
||||
attachments, err := l.dbService.GetAttachments(ctx, &l.page)
|
||||
// Get next page of media attachments up to max ID.
|
||||
medias, err := l.state.DB.GetAttachments(ctx, &page)
|
||||
if err != nil && !errors.Is(err, db.ErrNoEntries) {
|
||||
return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err)
|
||||
return fmt.Errorf("failed to fetch media from database: %w", err)
|
||||
}
|
||||
|
||||
// Get current max ID.
|
||||
maxID := l.page.Max.Value
|
||||
maxID := page.Max.Value
|
||||
|
||||
// If no attachments or the same group is returned, we reached the end.
|
||||
if len(attachments) == 0 || maxID == attachments[len(attachments)-1].ID {
|
||||
// If no media or the same group is returned, we reached end.
|
||||
if len(medias) == 0 || maxID == medias[len(medias)-1].ID {
|
||||
break
|
||||
}
|
||||
|
||||
// Use last ID as the next 'maxID' value.
|
||||
maxID = attachments[len(attachments)-1].ID
|
||||
l.page.Max = paging.MaxID(maxID)
|
||||
// Use last ID as the next 'maxID'.
|
||||
maxID = medias[len(medias)-1].ID
|
||||
page.Max.Value = maxID
|
||||
|
||||
for _, a := range attachments {
|
||||
v := filter(a)
|
||||
if v != "" {
|
||||
res = append(res, v)
|
||||
switch {
|
||||
case l.localOnly:
|
||||
// Only print local media paths.
|
||||
for _, media := range medias {
|
||||
if media.RemoteURL == "" {
|
||||
printMediaPaths(basePath, media)
|
||||
}
|
||||
}
|
||||
|
||||
case l.remoteOnly:
|
||||
// Only print remote media paths.
|
||||
for _, media := range medias {
|
||||
if media.RemoteURL != "" {
|
||||
printMediaPaths(basePath, media)
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
// Print all known media paths.
|
||||
for _, media := range medias {
|
||||
printMediaPaths(basePath, media)
|
||||
}
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get a list of emojis using a custom filter
|
||||
func (l *list) GetAllEmojisPaths(ctx context.Context, filter func(*gtsmodel.Emoji) string) ([]string, error) {
|
||||
res := make([]string, 0, 100)
|
||||
func (l *list) ListEmojiPaths(ctx context.Context) error {
|
||||
// Page reused for iterative
|
||||
// attachment queries, with
|
||||
// predefined limit.
|
||||
var page paging.Page
|
||||
page.Limit = 500
|
||||
|
||||
// Storage base path, used for path building.
|
||||
basePath := config.GetStorageLocalBasePath()
|
||||
|
||||
for {
|
||||
// Get the next page of emoji media up to max ID.
|
||||
attachments, err := l.dbService.GetEmojis(ctx, &l.page)
|
||||
emojis, err := l.state.DB.GetEmojis(ctx, &page)
|
||||
if err != nil && !errors.Is(err, db.ErrNoEntries) {
|
||||
return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err)
|
||||
return fmt.Errorf("failed to fetch emojis from database: %w", err)
|
||||
}
|
||||
|
||||
// Get current max ID.
|
||||
maxID := l.page.Max.Value
|
||||
maxID := page.Max.Value
|
||||
|
||||
// If no attachments or the same group is returned, we reached the end.
|
||||
if len(attachments) == 0 || maxID == attachments[len(attachments)-1].ID {
|
||||
// If no emojis or the same group is returned, we reached end.
|
||||
if len(emojis) == 0 || maxID == emojis[len(emojis)-1].ID {
|
||||
break
|
||||
}
|
||||
|
||||
// Use last ID as the next 'maxID' value.
|
||||
maxID = attachments[len(attachments)-1].ID
|
||||
l.page.Max = paging.MaxID(maxID)
|
||||
// Use last ID as the next 'maxID'.
|
||||
maxID = emojis[len(emojis)-1].ID
|
||||
page.Max.Value = maxID
|
||||
|
||||
for _, a := range attachments {
|
||||
v := filter(a)
|
||||
if v != "" {
|
||||
res = append(res, v)
|
||||
switch {
|
||||
case l.localOnly:
|
||||
// Only print local emoji paths.
|
||||
for _, emoji := range emojis {
|
||||
if emoji.ImageRemoteURL == "" {
|
||||
printEmojiPaths(basePath, emoji)
|
||||
}
|
||||
}
|
||||
|
||||
case l.remoteOnly:
|
||||
// Only print remote emoji paths.
|
||||
for _, emoji := range emojis {
|
||||
if emoji.ImageRemoteURL != "" {
|
||||
printEmojiPaths(basePath, emoji)
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
// Print all known emoji paths.
|
||||
for _, emoji := range emojis {
|
||||
printEmojiPaths(basePath, emoji)
|
||||
}
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func setupList(ctx context.Context) (*list, error) {
|
||||
|
|
@ -128,150 +210,84 @@ func setupList(ctx context.Context) (*list, error) {
|
|||
)
|
||||
}
|
||||
|
||||
// Initialize caches.
|
||||
state.Caches.Init()
|
||||
|
||||
// Ensure background cache tasks are running.
|
||||
if err := state.Caches.Start(); err != nil {
|
||||
return nil, fmt.Errorf("error starting caches: %w", err)
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
// Only set state DB connection.
|
||||
// Don't need Actions or Workers for this.
|
||||
dbService, err := bundb.NewBunDBService(ctx, &state)
|
||||
state.DB, err = bundb.NewBunDBService(ctx, &state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating dbservice: %w", err)
|
||||
}
|
||||
state.DB = dbService
|
||||
|
||||
return &list{
|
||||
dbService: dbService,
|
||||
state: &state,
|
||||
page: paging.Page{Limit: 200},
|
||||
localOnly: localOnly,
|
||||
remoteOnly: remoteOnly,
|
||||
out: bufio.NewWriter(os.Stdout),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (l *list) shutdown() error {
|
||||
l.out.Flush()
|
||||
err := l.dbService.Close()
|
||||
err := l.state.DB.Close()
|
||||
l.state.Caches.Stop()
|
||||
return err
|
||||
}
|
||||
|
||||
// ListAttachments lists local, remote, or all attachment paths.
|
||||
func ListAttachments(ctx context.Context) error {
|
||||
list, err := setupList(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
// reusable path building buffer,
|
||||
// only usable here as we're not
|
||||
// performing concurrent writes.
|
||||
var pb fastpath.Builder
|
||||
|
||||
// reusable string output buffer,
|
||||
// only usable here as we're not
|
||||
// performing concurrent writes.
|
||||
var outbuf byteutil.Buffer
|
||||
|
||||
func printMediaPaths(basePath string, media *gtsmodel.MediaAttachment) {
|
||||
// Append file path if present.
|
||||
if media.File.Path != "" {
|
||||
path := pb.Join(basePath, media.File.Path)
|
||||
_, _ = outbuf.WriteString(path + "\n")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// Ensure lister gets shutdown on exit.
|
||||
if err := list.shutdown(); err != nil {
|
||||
log.Error(ctx, err)
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
mediaPath = config.GetStorageLocalBasePath()
|
||||
filter func(*gtsmodel.MediaAttachment) string
|
||||
)
|
||||
|
||||
switch {
|
||||
case list.localOnly:
|
||||
filter = func(m *gtsmodel.MediaAttachment) string {
|
||||
if m.RemoteURL != "" {
|
||||
// Remote, not
|
||||
// interested.
|
||||
return ""
|
||||
}
|
||||
|
||||
return path.Join(mediaPath, m.File.Path)
|
||||
}
|
||||
|
||||
case list.remoteOnly:
|
||||
filter = func(m *gtsmodel.MediaAttachment) string {
|
||||
if m.RemoteURL == "" {
|
||||
// Local, not
|
||||
// interested.
|
||||
return ""
|
||||
}
|
||||
|
||||
return path.Join(mediaPath, m.File.Path)
|
||||
}
|
||||
|
||||
default:
|
||||
filter = func(m *gtsmodel.MediaAttachment) string {
|
||||
return path.Join(mediaPath, m.File.Path)
|
||||
}
|
||||
// Append thumb path if present.
|
||||
if media.Thumbnail.Path != "" {
|
||||
path := pb.Join(basePath, media.Thumbnail.Path)
|
||||
_, _ = outbuf.WriteString(path + "\n")
|
||||
}
|
||||
|
||||
attachments, err := list.GetAllAttachmentPaths(ctx, filter)
|
||||
if err != nil {
|
||||
return err
|
||||
// Only write if any
|
||||
// string was prepared.
|
||||
if outbuf.Len() > 0 {
|
||||
_, _ = os.Stdout.Write(outbuf.B)
|
||||
outbuf.Reset()
|
||||
}
|
||||
|
||||
for _, a := range attachments {
|
||||
_, _ = list.out.WriteString(a + "\n")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListEmojis lists local, remote, or all emoji filepaths.
|
||||
func ListEmojis(ctx context.Context) error {
|
||||
list, err := setupList(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
func printEmojiPaths(basePath string, emoji *gtsmodel.Emoji) {
|
||||
// Append image path if present.
|
||||
if emoji.ImagePath != "" {
|
||||
path := pb.Join(basePath, emoji.ImagePath)
|
||||
_, _ = outbuf.WriteString(path + "\n")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// Ensure lister gets shutdown on exit.
|
||||
if err := list.shutdown(); err != nil {
|
||||
log.Error(ctx, err)
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
mediaPath = config.GetStorageLocalBasePath()
|
||||
filter func(*gtsmodel.Emoji) string
|
||||
)
|
||||
|
||||
switch {
|
||||
case list.localOnly:
|
||||
filter = func(e *gtsmodel.Emoji) string {
|
||||
if e.ImageRemoteURL != "" {
|
||||
// Remote, not
|
||||
// interested.
|
||||
return ""
|
||||
}
|
||||
|
||||
return path.Join(mediaPath, e.ImagePath)
|
||||
}
|
||||
|
||||
case list.remoteOnly:
|
||||
filter = func(e *gtsmodel.Emoji) string {
|
||||
if e.ImageRemoteURL == "" {
|
||||
// Local, not
|
||||
// interested.
|
||||
return ""
|
||||
}
|
||||
|
||||
return path.Join(mediaPath, e.ImagePath)
|
||||
}
|
||||
|
||||
default:
|
||||
filter = func(e *gtsmodel.Emoji) string {
|
||||
return path.Join(mediaPath, e.ImagePath)
|
||||
}
|
||||
// Append static path if present.
|
||||
if emoji.ImageStaticPath != "" {
|
||||
path := pb.Join(basePath, emoji.ImageStaticPath)
|
||||
_, _ = outbuf.WriteString(path + "\n")
|
||||
}
|
||||
|
||||
emojis, err := list.GetAllEmojisPaths(ctx, filter)
|
||||
if err != nil {
|
||||
return err
|
||||
// Only write if any
|
||||
// string was prepared.
|
||||
if outbuf.Len() > 0 {
|
||||
_, _ = os.Stdout.Write(outbuf.B)
|
||||
outbuf.Reset()
|
||||
}
|
||||
|
||||
for _, e := range emojis {
|
||||
_, _ = list.out.WriteString(e + "\n")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue