mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-29 17:12:26 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[bugfix] s3 media uploaded without content-type (#3353)

Browse source 
* update go-storage dependency, for S3Storage manually call PutObject() so we can set content-type

* update calls to PutFile() to include the contentType
This commit is contained in:
kim 2024-09-26 12:43:10 +00:00 • committed by GitHub
commit 53ee6aef08
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 433 additions and 210 deletions
Download patch file Download diff file Expand all files Collapse all files

253
vendor/github.com/minio/minio-go/v7/functional_tests.go generated vendored
View file

@ -83,7 +83,7 @@ func createHTTPTransport() (transport *http.Transport) {
return nil
}
if mustParseBool(os.Getenv(skipCERTValidation)) {
if mustParseBool(os.Getenv(enableHTTPS)) && mustParseBool(os.Getenv(skipCERTValidation)) {
transport.TLSClientConfig.InsecureSkipVerify = true
}
@ -2334,7 +2334,7 @@ func testPutObjectWithChecksums() {
}
// Test PutObject with custom checksums.
func testPutMultipartObjectWithChecksums() {
func testPutObjectWithTrailingChecksums() {
// initialize logging params
startTime := time.Now()
testName := getFuncName()
@ -2342,7 +2342,7 @@ func testPutMultipartObjectWithChecksums() {
args := map[string]interface{}{
"bucketName": "",
"objectName": "",
"opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}",
"opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress, TrailChecksum: xxx}",
}
if !isFullMode() {
@ -2356,9 +2356,201 @@ func testPutMultipartObjectWithChecksums() {
// Instantiate new minio client object.
c, err := minio.New(os.Getenv(serverEndpoint),
&minio.Options{
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
Transport: createHTTPTransport(),
Secure: mustParseBool(os.Getenv(enableHTTPS)),
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
Transport: createHTTPTransport(),
Secure: mustParseBool(os.Getenv(enableHTTPS)),
TrailingHeaders: true,
})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
return
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c.SetAppInfo("MinIO-go-FunctionalTest", appVersion)
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
args["bucketName"] = bucketName
// Make a new bucket.
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
if err != nil {
logError(testName, function, args, startTime, "", "Make bucket failed", err)
return
}
defer cleanupBucket(bucketName, c)
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
{cs: minio.ChecksumSHA1},
{cs: minio.ChecksumSHA256},
}
for _, test := range tests {
function := "PutObject(bucketName, objectName, reader,size, opts)"
bufSize := dataFileMap["datafile-10-kB"]
// Save the data
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
args["objectName"] = objectName
cmpChecksum := func(got, want string) {
if want != got {
logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got))
return
}
}
meta := map[string]string{}
reader := getDataReader("datafile-10-kB")
b, err := io.ReadAll(reader)
if err != nil {
logError(testName, function, args, startTime, "", "Read failed", err)
return
}
h := test.cs.Hasher()
h.Reset()
// Test with Wrong CRC.
args["metadata"] = meta
args["range"] = "false"
args["checksum"] = test.cs.String()
resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{
DisableMultipart: true,
DisableContentSha256: true,
UserMetadata: meta,
Checksum: test.cs,
})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
h.Write(b)
meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil))
cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"])
cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
function = "GetObject(...)"
r, err := c.GetObject(context.Background(), bucketName, objectName, gopts)
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
st, err := r.Stat()
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
}
cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"])
cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
return
}
if err := r.Close(); err != nil {
logError(testName, function, args, startTime, "", "Object Close failed", err)
return
}
if err := r.Close(); err == nil {
logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err)
return
}
function = "GetObject( Range...)"
args["range"] = "true"
err = gopts.SetRange(100, 1000)
if err != nil {
logError(testName, function, args, startTime, "", "SetRange failed", err)
return
}
r, err = c.GetObject(context.Background(), bucketName, objectName, gopts)
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
b, err = io.ReadAll(r)
if err != nil {
logError(testName, function, args, startTime, "", "Read failed", err)
return
}
st, err = r.Stat()
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
}
// Range requests should return empty checksums...
cmpChecksum(st.ChecksumSHA256, "")
cmpChecksum(st.ChecksumSHA1, "")
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
function = "GetObjectAttributes(...)"
s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err)
return
}
cmpChecksum(s.Checksum.ChecksumSHA256, meta["x-amz-checksum-sha256"])
cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
delete(args, "range")
delete(args, "metadata")
}
logSuccess(testName, function, args, startTime)
}
// Test PutObject with custom checksums.
func testPutMultipartObjectWithChecksums(trailing bool) {
// initialize logging params
startTime := time.Now()
testName := getFuncName()
function := "PutObject(bucketName, objectName, reader,size, opts)"
args := map[string]interface{}{
"bucketName": "",
"objectName": "",
"opts": fmt.Sprintf("minio.PutObjectOptions{UserMetadata: metadata, Progress: progress Checksum: %v}", trailing),
}
if !isFullMode() {
logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs")
return
}
// Seed random based on current time.
rand.Seed(time.Now().Unix())
// Instantiate new minio client object.
c, err := minio.New(os.Getenv(serverEndpoint),
&minio.Options{
Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""),
Transport: createHTTPTransport(),
Secure: mustParseBool(os.Getenv(enableHTTPS)),
TrailingHeaders: trailing,
})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
@ -2445,14 +2637,20 @@ func testPutMultipartObjectWithChecksums() {
h.Reset()
want := hashMultiPart(b, partSize, test.cs.Hasher())
var cs minio.ChecksumType
rd := io.Reader(io.NopCloser(bytes.NewReader(b)))
if trailing {
cs = test.cs
rd = bytes.NewReader(b)
}
// Set correct CRC.
resp, err := c.PutObject(context.Background(), bucketName, objectName, io.NopCloser(bytes.NewReader(b)), int64(bufSize), minio.PutObjectOptions{
resp, err := c.PutObject(context.Background(), bucketName, objectName, rd, int64(bufSize), minio.PutObjectOptions{
DisableContentSha256: true,
DisableMultipart: false,
UserMetadata: nil,
PartSize: partSize,
AutoChecksum: test.cs,
Checksum: cs,
})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
@ -2982,6 +3180,7 @@ func testGetObjectAttributes() {
testFiles[i].UploadInfo, err = c.PutObject(context.Background(), v.Bucket, v.Object, reader, int64(bufSize), minio.PutObjectOptions{
ContentType: v.ContentType,
SendContentMd5: v.SendContentMd5,
Checksum: minio.ChecksumCRC32C,
})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
@ -3063,7 +3262,7 @@ func testGetObjectAttributes() {
test: objectAttributesTestOptions{
TestFileName: "file1",
StorageClass: "STANDARD",
HasFullChecksum: false,
HasFullChecksum: true,
},
}
@ -3152,9 +3351,10 @@ func testGetObjectAttributesSSECEncryption() {
info, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{
ContentType: "content/custom",
SendContentMd5: true,
SendContentMd5: false,
ServerSideEncryption: sse,
PartSize: uint64(bufSize) / 2,
Checksum: minio.ChecksumCRC32C,
})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
@ -3174,9 +3374,9 @@ func testGetObjectAttributesSSECEncryption() {
ETag: info.ETag,
NumberOfParts: 2,
ObjectSize: int(info.Size),
HasFullChecksum: false,
HasFullChecksum: true,
HasParts: true,
HasPartChecksums: false,
HasPartChecksums: true,
})
if err != nil {
logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed", err)
@ -5594,18 +5794,12 @@ func testPresignedPostPolicy() {
}
writer.Close()
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
if err != nil {
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
return
}
httpClient := &http.Client{
// Setting a sensible time out of 30secs to wait for response
// headers. Request is pro-actively canceled after 30secs
// with no response.
Timeout: 30 * time.Second,
Transport: transport,
Transport: createHTTPTransport(),
}
args["url"] = presignedPostPolicyURL.String()
@ -7519,7 +7713,7 @@ func testFunctional() {
return
}
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
transport := createHTTPTransport()
if err != nil {
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
return
@ -12450,18 +12644,12 @@ func testFunctionalV2() {
return
}
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
if err != nil {
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
return
}
httpClient := &http.Client{
// Setting a sensible time out of 30secs to wait for response
// headers. Request is pro-actively canceled after 30secs
// with no response.
Timeout: 30 * time.Second,
Transport: transport,
Transport: createHTTPTransport(),
}
req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil)
@ -13556,14 +13744,9 @@ func testCors() {
bucketURL := c.EndpointURL().String() + "/" + bucketName + "/"
objectURL := bucketURL + objectName
transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
if err != nil {
logError(testName, function, args, startTime, "", "DefaultTransport failed", err)
return
}
httpClient := &http.Client{
Timeout: 30 * time.Second,
Transport: transport,
Transport: createHTTPTransport(),
}
errStrAccessForbidden := `<Error><Code>AccessForbidden</Code><Message>CORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted`
@ -14757,7 +14940,9 @@ func main() {
testCompose10KSourcesV2()
testUserMetadataCopyingV2()
testPutObjectWithChecksums()
testPutMultipartObjectWithChecksums()
testPutObjectWithTrailingChecksums()
testPutMultipartObjectWithChecksums(false)
testPutMultipartObjectWithChecksums(true)
testPutObject0ByteV2()
testPutObjectNoLengthV2()
testPutObjectsUnknownV2()