mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 14:52:26 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[performance] update storage backend and make use of seek syscall when available (#2924)

Browse source 
* update to use go-storage/ instead of go-store/v2/storage/

* pull in latest version from codeberg

* remove test output 😇

* add code comments

* set the exclusive bit when creating new files in disk config

* bump to actual release version

* bump to v0.1.1 (tis a simple no-logic change)

* update readme

* only use a temporary read seeker when decoding video if required (should only be S3 now)

* use fastcopy library to use memory pooled buffers when calling TempFileSeeker()

* update to use seek call in serveFileRange()
This commit is contained in:
kim 2024-05-22 09:46:24 +00:00 • committed by GitHub
commit 3d3e99ae52
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
91 changed files with 1610 additions and 12737 deletions
Download patch file Download diff file Expand all files Collapse all files

14
internal/db/bundb/migrations/20220612091800_duplicated_media_cleanup.go
View file

@ -20,10 +20,11 @@ package migrations
import (
"context"
"database/sql"
"errors"
"fmt"
"path"
"codeberg.org/gruf/go-store/v2/storage"
"codeberg.org/gruf/go-storage"
"codeberg.org/gruf/go-storage/disk"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
@ -32,13 +33,13 @@ import (
func init() {
deleteAttachment := func(ctx context.Context, l log.Entry, a *gtsmodel.MediaAttachment, s storage.Storage, tx bun.Tx) {
if err := s.Remove(ctx, a.File.Path); err != nil && err != storage.ErrNotFound {
if err := s.Remove(ctx, a.File.Path); err != nil && !errors.Is(err, storage.ErrNotFound) {
l.Errorf("error removing file %s: %s", a.File.Path, err)
} else {
l.Debugf("deleted %s", a.File.Path)
}
if err := s.Remove(ctx, a.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
if err := s.Remove(ctx, a.Thumbnail.Path); err != nil && !errors.Is(err, storage.ErrNotFound) {
l.Errorf("error removing file %s: %s", a.Thumbnail.Path, err)
} else {
l.Debugf("deleted %s", a.Thumbnail.Path)
@ -68,13 +69,10 @@ func init() {
}
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
s, err := storage.OpenDisk(storageBasePath, &storage.DiskConfig{
LockFile: path.Join(storageBasePath, "store.lock"),
})
s, err := disk.Open(storageBasePath, nil)
if err != nil {
return fmt.Errorf("error creating storage backend: %s", err)
}
defer s.Close()
// step 1. select all media attachment remote URLs that have duplicates
var dupes int