[chore] bump dependencies (#4339)

- github.com/KimMachineGun/automemlimit v0.7.4
- github.com/miekg/dns v1.1.67
- github.com/minio/minio-go/v7 v7.0.95
- github.com/spf13/pflag v1.0.7
- github.com/tdewolff/minify/v2 v2.23.9
- github.com/uptrace/bun v1.2.15
- github.com/uptrace/bun/dialect/pgdialect v1.2.15
- github.com/uptrace/bun/dialect/sqlitedialect v1.2.15
- github.com/uptrace/bun/extra/bunotel v1.2.15
- golang.org/x/image v0.29.0
- golang.org/x/net v0.42.0

Reviewed-on: https://codeberg.org/superseriousbusiness/gotosocial/pulls/4339
Co-authored-by: kim <grufwub@gmail.com>
Co-committed-by: kim <grufwub@gmail.com>
This commit is contained in:
kim 2025-07-22 18:00:27 +02:00 committed by kim
commit c00cad2ceb
76 changed files with 5544 additions and 886 deletions

View file

@ -10,6 +10,6 @@ import (
"github.com/klauspost/cpuid/v2"
)
var hasAsm = cpuid.CPU.Supports(cpuid.ASIMD) && cpuid.CPU.Supports(cpuid.PMULL)
var hasAsm = cpuid.CPU.Supports(cpuid.ASIMD, cpuid.PMULL, cpuid.SHA3)
func updateAsm(crc uint64, p []byte) (checksum uint64)

View file

@ -127,6 +127,10 @@ func (c *Client) appendObjectDo(ctx context.Context, bucketName, objectName stri
if opts.checksumType.IsSet() {
reqMetadata.addCrc = &opts.checksumType
reqMetadata.customHeader.Set(amzChecksumAlgo, opts.checksumType.String())
if opts.checksumType.FullObjectRequested() {
reqMetadata.customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String())
}
}
// Execute PUT an objectName.
@ -183,8 +187,8 @@ func (c *Client) AppendObject(ctx context.Context, bucketName, objectName string
if err != nil {
return UploadInfo{}, err
}
if oinfo.ChecksumMode != ChecksumFullObjectMode.String() {
return UploadInfo{}, fmt.Errorf("append API is not allowed on objects that are not full_object checksum type: %s", oinfo.ChecksumMode)
if oinfo.ChecksumMode != "" && oinfo.ChecksumMode != ChecksumFullObjectMode.String() {
return UploadInfo{}, fmt.Errorf("Append() is not allowed on objects that are not of FULL_OBJECT checksum type: %s", oinfo.ChecksumMode)
}
opts.setChecksumParams(oinfo) // set the appropriate checksum params based on the existing object checksum metadata.
opts.setWriteOffset(oinfo.Size) // First append must set the current object size as the offset.

View file

@ -82,16 +82,12 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
// avoid sha256 with non-v4 signature request or
// HTTPS connection.
hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5, !opts.DisableContentSha256)
if len(hashSums) == 0 {
addAutoChecksumHeaders(&opts)
}
// Initiate a new multipart upload.
uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
if err != nil {
return UploadInfo{}, err
}
delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
defer func() {
if err != nil {
@ -145,11 +141,15 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
if hashSums["sha256"] != nil {
sha256Hex = hex.EncodeToString(hashSums["sha256"])
}
if len(hashSums) == 0 {
if opts.AutoChecksum.IsSet() {
crc.Reset()
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum))
customHeader.Set(amzChecksumAlgo, opts.AutoChecksum.String())
if opts.AutoChecksum.FullObjectRequested() {
customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String())
}
}
p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: rd, partNumber: partNumber, md5Base64: md5Base64, sha256Hex: sha256Hex, size: int64(length), sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader}

View file

@ -108,19 +108,14 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
if err != nil {
return UploadInfo{}, err
}
if opts.Checksum.IsSet() {
opts.AutoChecksum = opts.Checksum
}
withChecksum := c.trailingHeaderSupport
if withChecksum {
addAutoChecksumHeaders(&opts)
}
// Initiate a new multipart upload.
uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
if err != nil {
return UploadInfo{}, err
}
delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
withChecksum := c.trailingHeaderSupport
// Aborts the multipart upload in progress, if the
// function returns any error, since we do not resume
@ -297,15 +292,6 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
return UploadInfo{}, err
}
if opts.Checksum.IsSet() {
opts.AutoChecksum = opts.Checksum
opts.SendContentMd5 = false
}
if !opts.SendContentMd5 {
addAutoChecksumHeaders(&opts)
}
// Calculate the optimal parts info for a given size.
totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(size, opts.PartSize)
if err != nil {
@ -316,7 +302,6 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
if err != nil {
return UploadInfo{}, err
}
delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
// Aborts the multipart upload if the function returns
// any error, since we do not resume we should purge
@ -369,12 +354,18 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
md5Hash.Reset()
md5Hash.Write(buf[:length])
md5Base64 = base64.StdEncoding.EncodeToString(md5Hash.Sum(nil))
} else {
}
if opts.AutoChecksum.IsSet() {
// Add CRC32C instead.
crc.Reset()
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.KeyCapitalized(), base64.StdEncoding.EncodeToString(cSum))
customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum))
customHeader.Set(amzChecksumAlgo, opts.AutoChecksum.String())
if opts.AutoChecksum.FullObjectRequested() {
customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String())
}
}
// Update progress reader appropriately to the latest offset
@ -453,13 +444,6 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
if err = s3utils.CheckValidObjectName(objectName); err != nil {
return UploadInfo{}, err
}
if opts.Checksum.IsSet() {
opts.SendContentMd5 = false
opts.AutoChecksum = opts.Checksum
}
if !opts.SendContentMd5 {
addAutoChecksumHeaders(&opts)
}
// Cancel all when an error occurs.
ctx, cancel := context.WithCancel(ctx)
@ -476,7 +460,6 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
if err != nil {
return UploadInfo{}, err
}
delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
// Aborts the multipart upload if the function returns
// any error, since we do not resume we should purge
@ -541,18 +524,22 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
return UploadInfo{}, rerr
}
// Calculate md5sum.
customHeader := make(http.Header)
if !opts.SendContentMd5 {
// Add Checksum instead.
crc.Reset()
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum))
}
wg.Add(1)
go func(partNumber int) {
// Calculate md5sum.
customHeader := make(http.Header)
if opts.AutoChecksum.IsSet() {
// Add Checksum instead.
crc.Reset()
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum))
customHeader.Set(amzChecksumAlgo, opts.AutoChecksum.String())
if opts.AutoChecksum.FullObjectRequested() {
customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String())
}
}
// Avoid declaring variables in the for loop
var md5Base64 string
@ -664,9 +651,6 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, r
if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 {
return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'")
}
if opts.Checksum.IsSet() {
opts.SendContentMd5 = false
}
var readSeeker io.Seeker
if size > 0 {
@ -759,7 +743,7 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
}
}
if addCrc {
opts.AutoChecksum.SetDefault(ChecksumCRC32C)
opts.AutoChecksum.SetDefault(ChecksumFullObjectCRC32C)
reqMetadata.addCrc = &opts.AutoChecksum
}
}

View file

@ -26,6 +26,7 @@ import (
"io"
"net/http"
"sort"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/encrypt"
@ -267,7 +268,16 @@ func (opts PutObjectOptions) validate(c *Client) (err error) {
if opts.LegalHold != "" && !opts.LegalHold.IsValid() {
return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status")
}
if opts.Checksum.IsSet() {
checkCrc := false
for k := range opts.UserMetadata {
if strings.HasPrefix(k, "x-amz-checksum-") {
checkCrc = true
break
}
}
if opts.Checksum.IsSet() || checkCrc {
switch {
case !c.trailingHeaderSupport:
return errInvalidArgument("Checksum requires Client with TrailingHeaders enabled")
@ -307,10 +317,10 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part
// be reused for best outcomes for PutObject(), pass the size always.
//
// NOTE: Upon errors during upload multipart operation is entirely aborted.
func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64,
func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64,
opts PutObjectOptions,
) (info UploadInfo, err error) {
if objectSize < 0 && opts.DisableMultipart {
if size < 0 && opts.DisableMultipart {
return UploadInfo{}, errors.New("object size must be provided with disable multipart upload")
}
@ -319,15 +329,20 @@ func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, r
return UploadInfo{}, err
}
return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts)
}
func (c *Client) putObjectCommon(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) {
// Check for largest object size allowed.
if size > int64(maxMultipartPutObjectSize) {
return UploadInfo{}, errEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName)
}
opts.AutoChecksum.SetDefault(ChecksumCRC32C)
if opts.Checksum.IsSet() {
opts.AutoChecksum = opts.Checksum
opts.SendContentMd5 = false
}
if c.trailingHeaderSupport {
opts.AutoChecksum.SetDefault(ChecksumCRC32C)
addAutoChecksumHeaders(&opts)
}
// NOTE: Streaming signature is not supported by GCS.
if s3utils.IsGoogleEndpoint(*c.endpointURL) {
@ -385,20 +400,11 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
return UploadInfo{}, err
}
if opts.Checksum.IsSet() {
opts.SendContentMd5 = false
opts.AutoChecksum = opts.Checksum
}
if !opts.SendContentMd5 {
addAutoChecksumHeaders(&opts)
}
// Initiate a new multipart upload.
uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts)
if err != nil {
return UploadInfo{}, err
}
delete(opts.UserMetadata, "X-Amz-Checksum-Algorithm")
defer func() {
if err != nil {
@ -437,11 +443,17 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
hash.Write(buf[:length])
md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
hash.Close()
} else {
}
if opts.AutoChecksum.IsSet() {
crc.Reset()
crc.Write(buf[:length])
cSum := crc.Sum(nil)
customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum))
customHeader.Set(amzChecksumAlgo, opts.AutoChecksum.String())
if opts.AutoChecksum.FullObjectRequested() {
customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String())
}
}
// Update progress reader appropriately to the latest offset

View file

@ -43,12 +43,11 @@ import (
md5simd "github.com/minio/md5-simd"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/kvcache"
"github.com/minio/minio-go/v7/pkg/peeker"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/minio/minio-go/v7/pkg/signer"
"github.com/minio/minio-go/v7/pkg/singleflight"
"golang.org/x/net/publicsuffix"
internalutils "github.com/minio/minio-go/v7/pkg/utils"
)
// Client implements Amazon S3 compatible methods.
@ -163,7 +162,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v7.0.94"
libraryVersion = "v7.0.95"
)
// User Agent should always following the below style.
@ -625,7 +624,7 @@ func (c *Client) do(req *http.Request) (resp *http.Response, err error) {
// - Return the error XML bytes if an error is found
// - Make sure to always restablish the whole http response stream before returning
func tryParseErrRespFromBody(resp *http.Response) ([]byte, error) {
peeker := internalutils.NewPeekReadCloser(resp.Body, 5*humanize.MiByte)
peeker := peeker.NewPeekReadCloser(resp.Body, 5*humanize.MiByte)
defer func() {
peeker.ReplayFromStart()
resp.Body = peeker

View file

@ -29,6 +29,7 @@ import (
"math/bits"
"net/http"
"sort"
"strings"
"github.com/minio/crc64nvme"
)
@ -432,9 +433,19 @@ func addAutoChecksumHeaders(opts *PutObjectOptions) {
if opts.UserMetadata == nil {
opts.UserMetadata = make(map[string]string, 1)
}
opts.UserMetadata["X-Amz-Checksum-Algorithm"] = opts.AutoChecksum.String()
if opts.AutoChecksum.FullObjectRequested() {
opts.UserMetadata[amzChecksumMode] = ChecksumFullObjectMode.String()
addChecksum := true
for k := range opts.UserMetadata {
if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") {
addChecksum = false
}
}
if addChecksum && opts.AutoChecksum.IsSet() {
opts.UserMetadata[amzChecksumAlgo] = opts.AutoChecksum.String()
if opts.AutoChecksum.FullObjectRequested() {
opts.UserMetadata[amzChecksumMode] = ChecksumFullObjectMode.String()
}
}
}
@ -446,14 +457,17 @@ func applyAutoChecksum(opts *PutObjectOptions, allParts []ObjectPart) {
// Add composite hash of hashes.
crc, err := opts.AutoChecksum.CompositeChecksum(allParts)
if err == nil {
opts.UserMetadata = map[string]string{opts.AutoChecksum.Key(): crc.Encoded()}
opts.UserMetadata = map[string]string{
opts.AutoChecksum.Key(): crc.Encoded(),
amzChecksumMode: ChecksumCompositeMode.String(),
}
}
} else if opts.AutoChecksum.CanMergeCRC() {
crc, err := opts.AutoChecksum.FullObjectChecksum(allParts)
if err == nil {
opts.UserMetadata = map[string]string{
opts.AutoChecksum.KeyCapitalized(): crc.Encoded(),
amzChecksumMode: ChecksumFullObjectMode.String(),
opts.AutoChecksum.Key(): crc.Encoded(),
amzChecksumMode: ChecksumFullObjectMode.String(),
}
}
}

View file

@ -1970,7 +1970,7 @@ func testPutObjectWithChecksums() {
// initialize logging params
startTime := time.Now()
testName := getFuncName()
function := "PutObject(bucketName, objectName, reader,size, opts)"
function := "PutObject(bucketName, objectName, reader, size, opts)"
args := map[string]interface{}{
"bucketName": "",
"objectName": "",
@ -1982,7 +1982,7 @@ func testPutObjectWithChecksums() {
return
}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
return
@ -2037,6 +2037,10 @@ func testPutObjectWithChecksums() {
h := test.cs.Hasher()
h.Reset()
if test.cs.IsSet() {
meta["x-amz-checksum-algorithm"] = test.cs.String()
}
// Test with a bad CRC - we haven't called h.Write(b), so this is a checksum of empty data
meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil))
args["metadata"] = meta
@ -2323,7 +2327,7 @@ func testPutObjectWithTrailingChecksums() {
}
// Test PutObject with custom checksums.
func testPutMultipartObjectWithChecksums(trailing bool) {
func testPutMultipartObjectWithChecksums() {
// initialize logging params
startTime := time.Now()
testName := getFuncName()
@ -2331,7 +2335,7 @@ func testPutMultipartObjectWithChecksums(trailing bool) {
args := map[string]interface{}{
"bucketName": "",
"objectName": "",
"opts": fmt.Sprintf("minio.PutObjectOptions{UserMetadata: metadata, Trailing: %v}", trailing),
"opts": "minio.PutObjectOptions{UserMetadata: metadata, Trailing: true}",
}
if !isFullMode() {
@ -2339,7 +2343,7 @@ func testPutMultipartObjectWithChecksums(trailing bool) {
return
}
c, err := NewClient(ClientConfig{TrailingHeaders: trailing})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
return
@ -2433,12 +2437,8 @@ func testPutMultipartObjectWithChecksums(trailing bool) {
h.Reset()
want := hashMultiPart(b, partSize, test.cs)
var cs minio.ChecksumType
rd := io.Reader(io.NopCloser(bytes.NewReader(b)))
if trailing {
cs = test.cs
rd = bytes.NewReader(b)
}
rd := bytes.NewReader(b)
cs := test.cs
// Set correct CRC.
args["section"] = "PutObject"
@ -2447,7 +2447,6 @@ func testPutMultipartObjectWithChecksums(trailing bool) {
DisableMultipart: false,
UserMetadata: nil,
PartSize: partSize,
AutoChecksum: test.cs,
Checksum: cs,
})
if err != nil {
@ -2589,11 +2588,10 @@ func testTrailingChecksums() {
return
}
hashMultiPart := func(b []byte, partSize int, hasher hash.Hash) string {
hashMultiPart := func(b []byte, partSize int, hasher hash.Hash) (oparts []minio.ObjectPart) {
r := bytes.NewReader(b)
tmp := make([]byte, partSize)
parts := 0
var all []byte
for {
n, err := io.ReadFull(r, tmp)
if err != nil && err != io.ErrUnexpectedEOF {
@ -2605,14 +2603,16 @@ func testTrailingChecksums() {
parts++
hasher.Reset()
hasher.Write(tmp[:n])
all = append(all, hasher.Sum(nil)...)
oparts = append(oparts, minio.ObjectPart{
PartNumber: parts,
Size: int64(n),
ChecksumCRC32C: base64.StdEncoding.EncodeToString(hasher.Sum(nil)),
})
if err != nil {
break
}
}
hasher.Reset()
hasher.Write(all)
return fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), parts)
return oparts
}
defer cleanupBucket(bucketName, c)
tests := []struct {
@ -2636,6 +2636,7 @@ func testTrailingChecksums() {
DisableMultipart: false,
UserMetadata: nil,
PartSize: 5 << 20,
Checksum: minio.ChecksumFullObjectCRC32C,
},
},
{
@ -2647,6 +2648,7 @@ func testTrailingChecksums() {
DisableMultipart: false,
UserMetadata: nil,
PartSize: 6_645_654, // Rather arbitrary size
Checksum: minio.ChecksumFullObjectCRC32C,
},
},
{
@ -2658,6 +2660,7 @@ func testTrailingChecksums() {
DisableMultipart: false,
UserMetadata: nil,
PartSize: 5 << 20,
Checksum: minio.ChecksumFullObjectCRC32C,
},
},
{
@ -2669,6 +2672,7 @@ func testTrailingChecksums() {
DisableMultipart: false,
UserMetadata: nil,
PartSize: 6_645_654, // Rather arbitrary size
Checksum: minio.ChecksumFullObjectCRC32C,
},
},
}
@ -2696,7 +2700,14 @@ func testTrailingChecksums() {
reader.Close()
h := test.hasher
h.Reset()
test.ChecksumCRC32C = hashMultiPart(b, int(test.PO.PartSize), test.hasher)
parts := hashMultiPart(b, int(test.PO.PartSize), test.hasher)
cksum, err := minio.ChecksumFullObjectCRC32C.FullObjectChecksum(parts)
if err != nil {
logError(testName, function, args, startTime, "", "checksum calculation failed", err)
return
}
test.ChecksumCRC32C = cksum.Encoded()
// Set correct CRC.
resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), test.PO)
@ -4172,7 +4183,7 @@ func testFPutObjectMultipart() {
"opts": "",
}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
return
@ -5586,6 +5597,161 @@ func testPresignedPostPolicyWrongFile() {
logSuccess(testName, function, args, startTime)
}
// testPresignedPostPolicyEmptyFileName tests that an empty file name in the presigned post policy
func testPresignedPostPolicyEmptyFileName() {
// initialize logging params
startTime := time.Now()
testName := getFuncName()
function := "PresignedPostPolicy(policy)"
args := map[string]interface{}{
"policy": "",
}
c, err := NewClient(ClientConfig{})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
return
}
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
// Make a new bucket in 'us-east-1' (source bucket).
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
if err != nil {
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
return
}
defer cleanupBucket(bucketName, c)
// Generate 33K of data.
reader := getDataReader("datafile-33-kB")
defer reader.Close()
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
// Azure requires the key to not start with a number
metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user")
metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "")
buf, err := io.ReadAll(reader)
if err != nil {
logError(testName, function, args, startTime, "", "ReadAll failed", err)
return
}
policy := minio.NewPostPolicy()
policy.SetBucket(bucketName)
policy.SetKey(objectName)
policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days
policy.SetContentType("binary/octet-stream")
policy.SetContentLengthRange(10, 1024*1024)
policy.SetUserMetadata(metadataKey, metadataValue)
policy.SetContentEncoding("gzip")
// Add CRC32C
checksum := minio.ChecksumCRC32C.ChecksumBytes(buf)
err = policy.SetChecksum(checksum)
if err != nil {
logError(testName, function, args, startTime, "", "SetChecksum failed", err)
return
}
args["policy"] = policy.String()
presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy)
if err != nil {
logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err)
return
}
var formBuf bytes.Buffer
writer := multipart.NewWriter(&formBuf)
for k, v := range formData {
writer.WriteField(k, v)
}
// Get a 33KB file to upload and test if set post policy works
filePath := getMintDataDirFilePath("datafile-33-kB")
if filePath == "" {
// Make a temp file with 33 KB data.
file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest")
if err != nil {
logError(testName, function, args, startTime, "", "TempFile creation failed", err)
return
}
if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil {
logError(testName, function, args, startTime, "", "Copy failed", err)
return
}
if err = file.Close(); err != nil {
logError(testName, function, args, startTime, "", "File Close failed", err)
return
}
filePath = file.Name()
}
// add file to post request
f, err := os.Open(filePath)
defer f.Close()
if err != nil {
logError(testName, function, args, startTime, "", "File open failed", err)
return
}
w, err := writer.CreateFormFile("", filePath)
if err != nil {
logError(testName, function, args, startTime, "", "CreateFormFile failed", err)
return
}
_, err = io.Copy(w, f)
if err != nil {
logError(testName, function, args, startTime, "", "Copy failed", err)
return
}
writer.Close()
httpClient := &http.Client{
// Setting a sensible time out of 30secs to wait for response
// headers. Request is pro-actively canceled after 30secs
// with no response.
Timeout: 30 * time.Second,
Transport: createHTTPTransport(),
}
args["url"] = presignedPostPolicyURL.String()
req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes()))
if err != nil {
logError(testName, function, args, startTime, "", "Http request failed", err)
return
}
req.Header.Set("Content-Type", writer.FormDataContentType())
// make post request with correct form data
res, err := httpClient.Do(req)
if err != nil {
logError(testName, function, args, startTime, "", "Http request failed", err)
return
}
defer res.Body.Close()
if res.StatusCode != http.StatusBadRequest {
logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status))
return
}
body, err := io.ReadAll(res.Body)
if err != nil {
logError(testName, function, args, startTime, "", "ReadAll failed", err)
return
}
if !strings.Contains(string(body), "MalformedPOSTRequest") {
logError(testName, function, args, startTime, "", "Invalid error from server", errors.New(string(body)))
}
logSuccess(testName, function, args, startTime)
}
// Tests copy object
func testCopyObject() {
// initialize logging params
@ -11560,8 +11726,11 @@ func testPutObjectMetadataNonUSASCIIV2() {
}
for k, v := range metadata {
if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") {
continue
}
if st.Metadata.Get(http.CanonicalHeaderKey("X-Amz-Meta-"+k)) != v {
logError(testName, function, args, startTime, "", "Expected upload object metadata "+k+": "+v+" but got "+st.Metadata.Get("X-Amz-Meta-"+k), err)
logError(testName, function, args, startTime, "", "Expected upload object metadata "+k+": "+v+" but got "+st.Metadata.Get(http.CanonicalHeaderKey("X-Amz-Meta-"+k)), err)
return
}
}
@ -14069,8 +14238,7 @@ func main() {
testUserMetadataCopyingV2()
testPutObjectWithChecksums()
testPutObjectWithTrailingChecksums()
testPutMultipartObjectWithChecksums(false)
testPutMultipartObjectWithChecksums(true)
testPutMultipartObjectWithChecksums()
testPutObject0ByteV2()
testPutObjectMetadataNonUSASCIIV2()
testPutObjectNoLengthV2()
@ -14098,6 +14266,7 @@ func main() {
testGetObjectReadAtWhenEOFWasReached()
testPresignedPostPolicy()
testPresignedPostPolicyWrongFile()
testPresignedPostPolicyEmptyFileName()
testCopyObject()
testComposeObjectErrorCases()
testCompose10KSources()

View file

@ -84,7 +84,7 @@ func (hr *hookReader) Read(b []byte) (n int, err error) {
// reports the data read from the source to the hook.
func newHook(source, hook io.Reader) io.Reader {
if hook == nil {
return source
return &hookReader{source: source}
}
return &hookReader{
source: source,

View file

@ -125,6 +125,7 @@ func (i *STSCertificateIdentity) RetrieveWithCredContext(cc *CredContext) (Value
queryValues := url.Values{}
queryValues.Set("Action", "AssumeRoleWithCertificate")
queryValues.Set("Version", STSVersion)
queryValues.Set("DurationSeconds", strconv.FormatUint(uint64(livetime.Seconds()), 10))
if i.TokenRevokeType != "" {
queryValues.Set("TokenRevokeType", i.TokenRevokeType)
}
@ -134,10 +135,6 @@ func (i *STSCertificateIdentity) RetrieveWithCredContext(cc *CredContext) (Value
if err != nil {
return Value{}, err
}
if req.Form == nil {
req.Form = url.Values{}
}
req.Form.Add("DurationSeconds", strconv.FormatUint(uint64(livetime.Seconds()), 10))
client := i.Client
if client == nil {

View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package utils
package peeker
import (
"bytes"

View file

@ -417,7 +417,7 @@ func (p PostPolicy) String() string {
// marshalJSON - Provides Marshaled JSON in bytes.
func (p PostPolicy) marshalJSON() []byte {
expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"`
expirationStr := `"expiration":"` + p.expiration.UTC().Format(expirationDateFormat) + `"`
var conditionsStr string
conditions := []string{}
for _, po := range p.conditions {