[bugfix] media CLI list missing thumbs and statics (#4379)

refactors some of the media CLI listing code, and updates it to include both attachment thumbnails and emoji static images, which previously were missing.

addresses https://codeberg.org/superseriousbusiness/gotosocial/issues/4370

though in a follow-up i'll add support for regenerating attachment thumbnails and emoji static images.

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4379
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>

cmd/gotosocial/action/admin/media/list.go

@@ -18,12 +18,10 @@
 package media
 
 import (
-	"bufio"
 	"context"
 	"errors"
 	"fmt"
 	"os"
-	"path"
 
 	"code.superseriousbusiness.org/gotosocial/cmd/gotosocial/action"
 	"code.superseriousbusiness.org/gotosocial/internal/config"
@@ -33,84 +31,168 @@ import (
 	"code.superseriousbusiness.org/gotosocial/internal/log"
 	"code.superseriousbusiness.org/gotosocial/internal/paging"
 	"code.superseriousbusiness.org/gotosocial/internal/state"
+	"codeberg.org/gruf/go-byteutil"
+	"codeberg.org/gruf/go-fastpath/v2"
 )
 
 // check function conformance.
 var _ action.GTSAction = ListAttachments
 var _ action.GTSAction = ListEmojis
 
+// ListAttachments lists local, remote, or all attachment paths.
+func ListAttachments(ctx context.Context) error {
+	list, err := setupList(ctx)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		// Ensure lister gets shutdown on exit.
+		if err := list.shutdown(); err != nil {
+			log.Errorf(ctx, "error shutting down: %v", err)
+		}
+	}()
+
+	// List attachment media paths from db.
+	return list.ListAttachmentPaths(ctx)
+}
+
+// ListEmojis lists local, remote, or all emoji filepaths.
+func ListEmojis(ctx context.Context) error {
+	list, err := setupList(ctx)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		// Ensure lister gets shutdown on exit.
+		if err := list.shutdown(); err != nil {
+			log.Errorf(ctx, "error shutting down: %v", err)
+		}
+	}()
+
+	// List emoji media paths from db.
+	return list.ListEmojiPaths(ctx)
+}
+
 type list struct {
-	dbService  db.DB
 	state      *state.State
-	page       paging.Page
 	localOnly  bool
 	remoteOnly bool
-	out        *bufio.Writer
 }
 
-// Get a list of attachment using a custom filter
+func (l *list) ListAttachmentPaths(ctx context.Context) error {
-func (l *list) GetAllAttachmentPaths(ctx context.Context, filter func(*gtsmodel.MediaAttachment) string) ([]string, error) {
+	// Page reused for iterative
-	res := make([]string, 0, 100)
+	// attachment queries, with
+	// predefined limit.
+	var page paging.Page
+	page.Limit = 500
+
+	// Storage base path, used for path building.
+	basePath := config.GetStorageLocalBasePath()
 
 	for {
-		// Get the next page of media attachments up to max ID.
+		// Get next page of media attachments up to max ID.
-		attachments, err := l.dbService.GetAttachments(ctx, &l.page)
+		medias, err := l.state.DB.GetAttachments(ctx, &page)
 		if err != nil && !errors.Is(err, db.ErrNoEntries) {
-			return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err)
+			return fmt.Errorf("failed to fetch media from database: %w", err)
 		}
 
 		// Get current max ID.
-		maxID := l.page.Max.Value
+		maxID := page.Max.Value
 
-		// If no attachments or the same group is returned, we reached the end.
+		// If no media or the same group is returned, we reached end.
-		if len(attachments) == 0 || maxID == attachments[len(attachments)-1].ID {
+		if len(medias) == 0 || maxID == medias[len(medias)-1].ID {
 			break
 		}
 
-		// Use last ID as the next 'maxID' value.
+		// Use last ID as the next 'maxID'.
-		maxID = attachments[len(attachments)-1].ID
+		maxID = medias[len(medias)-1].ID
-		l.page.Max = paging.MaxID(maxID)
+		page.Max.Value = maxID
 
-		for _, a := range attachments {
+		switch {
-			v := filter(a)
+		case l.localOnly:
-			if v != "" {
+			// Only print local media paths.
-				res = append(res, v)
+			for _, media := range medias {
+				if media.RemoteURL == "" {
+					printMediaPaths(basePath, media)
+				}
+			}
+
+		case l.remoteOnly:
+			// Only print remote media paths.
+			for _, media := range medias {
+				if media.RemoteURL != "" {
+					printMediaPaths(basePath, media)
+				}
+			}
+
+		default:
+			// Print all known media paths.
+			for _, media := range medias {
+				printMediaPaths(basePath, media)
 			}
 		}
 	}
-	return res, nil
+
+	return nil
 }
 
-// Get a list of emojis using a custom filter
+func (l *list) ListEmojiPaths(ctx context.Context) error {
-func (l *list) GetAllEmojisPaths(ctx context.Context, filter func(*gtsmodel.Emoji) string) ([]string, error) {
+	// Page reused for iterative
-	res := make([]string, 0, 100)
+	// attachment queries, with
+	// predefined limit.
+	var page paging.Page
+	page.Limit = 500
+
+	// Storage base path, used for path building.
+	basePath := config.GetStorageLocalBasePath()
+
 	for {
 		// Get the next page of emoji media up to max ID.
-		attachments, err := l.dbService.GetEmojis(ctx, &l.page)
+		emojis, err := l.state.DB.GetEmojis(ctx, &page)
 		if err != nil && !errors.Is(err, db.ErrNoEntries) {
-			return nil, fmt.Errorf("failed to retrieve media metadata from database: %w", err)
+			return fmt.Errorf("failed to fetch emojis from database: %w", err)
 		}
 
 		// Get current max ID.
-		maxID := l.page.Max.Value
+		maxID := page.Max.Value
 
-		// If no attachments or the same group is returned, we reached the end.
+		// If no emojis or the same group is returned, we reached end.
-		if len(attachments) == 0 || maxID == attachments[len(attachments)-1].ID {
+		if len(emojis) == 0 || maxID == emojis[len(emojis)-1].ID {
 			break
 		}
 
-		// Use last ID as the next 'maxID' value.
+		// Use last ID as the next 'maxID'.
-		maxID = attachments[len(attachments)-1].ID
+		maxID = emojis[len(emojis)-1].ID
-		l.page.Max = paging.MaxID(maxID)
+		page.Max.Value = maxID
 
-		for _, a := range attachments {
+		switch {
-			v := filter(a)
+		case l.localOnly:
-			if v != "" {
+			// Only print local emoji paths.
-				res = append(res, v)
+			for _, emoji := range emojis {
+				if emoji.ImageRemoteURL == "" {
+					printEmojiPaths(basePath, emoji)
+				}
+			}
+
+		case l.remoteOnly:
+			// Only print remote emoji paths.
+			for _, emoji := range emojis {
+				if emoji.ImageRemoteURL != "" {
+					printEmojiPaths(basePath, emoji)
+				}
+			}
+
+		default:
+			// Print all known emoji paths.
+			for _, emoji := range emojis {
+				printEmojiPaths(basePath, emoji)
 			}
 		}
 	}
-	return res, nil
+
+	return nil
 }
 
 func setupList(ctx context.Context) (*list, error) {
@@ -128,150 +210,84 @@ func setupList(ctx context.Context) (*list, error) {
 		)
 	}
 
+	// Initialize caches.
 	state.Caches.Init()
+
+	// Ensure background cache tasks are running.
 	if err := state.Caches.Start(); err != nil {
 		return nil, fmt.Errorf("error starting caches: %w", err)
 	}
 
+	var err error
+
 	// Only set state DB connection.
 	// Don't need Actions or Workers for this.
-	dbService, err := bundb.NewBunDBService(ctx, &state)
+	state.DB, err = bundb.NewBunDBService(ctx, &state)
 	if err != nil {
 		return nil, fmt.Errorf("error creating dbservice: %w", err)
 	}
-	state.DB = dbService
 
 	return &list{
-		dbService:  dbService,
 		state:      &state,
-		page:       paging.Page{Limit: 200},
 		localOnly:  localOnly,
 		remoteOnly: remoteOnly,
-		out:        bufio.NewWriter(os.Stdout),
 	}, nil
 }
 
 func (l *list) shutdown() error {
-	l.out.Flush()
+	err := l.state.DB.Close()
-	err := l.dbService.Close()
 	l.state.Caches.Stop()
 	return err
 }
 
-// ListAttachments lists local, remote, or all attachment paths.
+// reusable path building buffer,
-func ListAttachments(ctx context.Context) error {
+// only usable here as we're not
-	list, err := setupList(ctx)
+// performing concurrent writes.
-	if err != nil {
+var pb fastpath.Builder
-		return err
+
+// reusable string output buffer,
+// only usable here as we're not
+// performing concurrent writes.
+var outbuf byteutil.Buffer
+
+func printMediaPaths(basePath string, media *gtsmodel.MediaAttachment) {
+	// Append file path if present.
+	if media.File.Path != "" {
+		path := pb.Join(basePath, media.File.Path)
+		_, _ = outbuf.WriteString(path + "\n")
 	}
 
-	defer func() {
+	// Append thumb path if present.
-		// Ensure lister gets shutdown on exit.
+	if media.Thumbnail.Path != "" {
-		if err := list.shutdown(); err != nil {
+		path := pb.Join(basePath, media.Thumbnail.Path)
-			log.Error(ctx, err)
+		_, _ = outbuf.WriteString(path + "\n")
-		}
-	}()
-
-	var (
-		mediaPath = config.GetStorageLocalBasePath()
-		filter    func(*gtsmodel.MediaAttachment) string
-	)
-
-	switch {
-	case list.localOnly:
-		filter = func(m *gtsmodel.MediaAttachment) string {
-			if m.RemoteURL != "" {
-				// Remote, not
-				// interested.
-				return ""
-			}
-
-			return path.Join(mediaPath, m.File.Path)
-		}
-
-	case list.remoteOnly:
-		filter = func(m *gtsmodel.MediaAttachment) string {
-			if m.RemoteURL == "" {
-				// Local, not
-				// interested.
-				return ""
-			}
-
-			return path.Join(mediaPath, m.File.Path)
-		}
-
-	default:
-		filter = func(m *gtsmodel.MediaAttachment) string {
-			return path.Join(mediaPath, m.File.Path)
-		}
 	}
 
-	attachments, err := list.GetAllAttachmentPaths(ctx, filter)
+	// Only write if any
-	if err != nil {
+	// string was prepared.
-		return err
+	if outbuf.Len() > 0 {
+		_, _ = os.Stdout.Write(outbuf.B)
+		outbuf.Reset()
 	}
-
-	for _, a := range attachments {
-		_, _ = list.out.WriteString(a + "\n")
-	}
-	return nil
 }
 
-// ListEmojis lists local, remote, or all emoji filepaths.
+func printEmojiPaths(basePath string, emoji *gtsmodel.Emoji) {
-func ListEmojis(ctx context.Context) error {
+	// Append image path if present.
-	list, err := setupList(ctx)
+	if emoji.ImagePath != "" {
-	if err != nil {
+		path := pb.Join(basePath, emoji.ImagePath)
-		return err
+		_, _ = outbuf.WriteString(path + "\n")
 	}
 
-	defer func() {
+	// Append static path if present.
-		// Ensure lister gets shutdown on exit.
+	if emoji.ImageStaticPath != "" {
-		if err := list.shutdown(); err != nil {
+		path := pb.Join(basePath, emoji.ImageStaticPath)
-			log.Error(ctx, err)
+		_, _ = outbuf.WriteString(path + "\n")
-		}
-	}()
-
-	var (
-		mediaPath = config.GetStorageLocalBasePath()
-		filter    func(*gtsmodel.Emoji) string
-	)
-
-	switch {
-	case list.localOnly:
-		filter = func(e *gtsmodel.Emoji) string {
-			if e.ImageRemoteURL != "" {
-				// Remote, not
-				// interested.
-				return ""
-			}
-
-			return path.Join(mediaPath, e.ImagePath)
-		}
-
-	case list.remoteOnly:
-		filter = func(e *gtsmodel.Emoji) string {
-			if e.ImageRemoteURL == "" {
-				// Local, not
-				// interested.
-				return ""
-			}
-
-			return path.Join(mediaPath, e.ImagePath)
-		}
-
-	default:
-		filter = func(e *gtsmodel.Emoji) string {
-			return path.Join(mediaPath, e.ImagePath)
-		}
 	}
 
-	emojis, err := list.GetAllEmojisPaths(ctx, filter)
+	// Only write if any
-	if err != nil {
+	// string was prepared.
-		return err
+	if outbuf.Len() > 0 {
+		_, _ = os.Stdout.Write(outbuf.B)
+		outbuf.Reset()
 	}
-
-	for _, e := range emojis {
-		_, _ = list.out.WriteString(e + "\n")
-	}
-	return nil
 }