[chore] Bump github.com/minio/minio-go/v7 from 7.0.37 to 7.0.43 (#983)

Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.37 to 7.0.43.
- [Release notes](https://github.com/minio/minio-go/releases)
- [Commits](https://github.com/minio/minio-go/compare/v7.0.37...v7.0.43)

---
updated-dependencies:
- dependency-name: github.com/minio/minio-go/v7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
dependabot[bot] 2022-11-07 11:20:43 +01:00 committed by GitHub
commit 459a5c8d96
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
44 changed files with 1031 additions and 336 deletions

View file

@ -0,0 +1,225 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2022 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package signer
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"strconv"
"strings"
"time"
)
// getUnsignedChunkLength - calculates the length of chunk metadata
func getUnsignedChunkLength(chunkDataSize int64) int64 {
return int64(len(fmt.Sprintf("%x", chunkDataSize))) +
crlfLen +
chunkDataSize +
crlfLen
}
// getUSStreamLength - calculates the length of the overall stream (data + metadata)
func getUSStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
if dataLen <= 0 {
return 0
}
chunksCount := int64(dataLen / chunkSize)
remainingBytes := int64(dataLen % chunkSize)
streamLen := int64(0)
streamLen += chunksCount * getUnsignedChunkLength(chunkSize)
if remainingBytes > 0 {
streamLen += getUnsignedChunkLength(remainingBytes)
}
streamLen += getUnsignedChunkLength(0)
if len(trailers) > 0 {
for name, placeholder := range trailers {
if len(placeholder) > 0 {
streamLen += int64(len(name) + len(trailerKVSeparator) + len(placeholder[0]) + 1)
}
}
streamLen += crlfLen
}
return streamLen
}
// prepareStreamingRequest - prepares a request with appropriate
// headers before computing the seed signature.
func prepareUSStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) {
req.TransferEncoding = []string{"aws-chunked"}
if sessionToken != "" {
req.Header.Set("X-Amz-Security-Token", sessionToken)
}
req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat))
// Set content length with streaming signature for each chunk included.
req.ContentLength = getUSStreamLength(dataLen, int64(payloadChunkSize), req.Trailer)
}
// StreamingUSReader implements chunked upload signature as a reader on
// top of req.Body's ReaderCloser chunk header;data;... repeat
type StreamingUSReader struct {
contentLen int64 // Content-Length from req header
baseReadCloser io.ReadCloser // underlying io.Reader
bytesRead int64 // bytes read from underlying io.Reader
buf bytes.Buffer // holds signed chunk
chunkBuf []byte // holds raw data read from req Body
chunkBufLen int // no. of bytes read so far into chunkBuf
done bool // done reading the underlying reader to EOF
chunkNum int
totalChunks int
lastChunkSize int
trailer http.Header
}
// writeChunk - signs a chunk read from s.baseReader of chunkLen size.
func (s *StreamingUSReader) writeChunk(chunkLen int, addCrLf bool) {
s.buf.WriteString(strconv.FormatInt(int64(chunkLen), 16) + "\r\n")
// Write chunk data into streaming buffer
s.buf.Write(s.chunkBuf[:chunkLen])
// Write the chunk trailer.
if addCrLf {
s.buf.Write([]byte("\r\n"))
}
// Reset chunkBufLen for next chunk read.
s.chunkBufLen = 0
s.chunkNum++
}
// addSignedTrailer - adds a trailer with the provided headers,
// then signs a chunk and adds it to output.
func (s *StreamingUSReader) addTrailer(h http.Header) {
olen := len(s.chunkBuf)
s.chunkBuf = s.chunkBuf[:0]
for k, v := range h {
s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...)
}
s.buf.Write(s.chunkBuf)
s.buf.WriteString("\r\n\r\n")
// Reset chunkBufLen for next chunk read.
s.chunkBuf = s.chunkBuf[:olen]
s.chunkBufLen = 0
s.chunkNum++
}
// StreamingUnsignedV4 - provides chunked upload
func StreamingUnsignedV4(req *http.Request, sessionToken string, dataLen int64, reqTime time.Time) *http.Request {
// Set headers needed for streaming signature.
prepareUSStreamingRequest(req, sessionToken, dataLen, reqTime)
if req.Body == nil {
req.Body = ioutil.NopCloser(bytes.NewReader([]byte("")))
}
stReader := &StreamingUSReader{
baseReadCloser: req.Body,
chunkBuf: make([]byte, payloadChunkSize),
contentLen: dataLen,
chunkNum: 1,
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
lastChunkSize: int(dataLen % payloadChunkSize),
}
if len(req.Trailer) > 0 {
stReader.trailer = req.Trailer
// Remove...
req.Trailer = nil
}
req.Body = stReader
return req
}
// Read - this method performs chunk upload signature providing a
// io.Reader interface.
func (s *StreamingUSReader) Read(buf []byte) (int, error) {
switch {
// After the last chunk is read from underlying reader, we
// never re-fill s.buf.
case s.done:
// s.buf will be (re-)filled with next chunk when has lesser
// bytes than asked for.
case s.buf.Len() < len(buf):
s.chunkBufLen = 0
for {
n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:])
// Usually we validate `err` first, but in this case
// we are validating n > 0 for the following reasons.
//
// 1. n > 0, err is one of io.EOF, nil (near end of stream)
// A Reader returning a non-zero number of bytes at the end
// of the input stream may return either err == EOF or err == nil
//
// 2. n == 0, err is io.EOF (actual end of stream)
//
// Callers should always process the n > 0 bytes returned
// before considering the error err.
if n1 > 0 {
s.chunkBufLen += n1
s.bytesRead += int64(n1)
if s.chunkBufLen == payloadChunkSize ||
(s.chunkNum == s.totalChunks-1 &&
s.chunkBufLen == s.lastChunkSize) {
// Sign the chunk and write it to s.buf.
s.writeChunk(s.chunkBufLen, true)
break
}
}
if err != nil {
if err == io.EOF {
// No more data left in baseReader - last chunk.
// Done reading the last chunk from baseReader.
s.done = true
// bytes read from baseReader different than
// content length provided.
if s.bytesRead != s.contentLen {
return 0, fmt.Errorf("http: ContentLength=%d with Body length %d", s.contentLen, s.bytesRead)
}
// Sign the chunk and write it to s.buf.
s.writeChunk(0, len(s.trailer) == 0)
if len(s.trailer) > 0 {
// Trailer must be set now.
s.addTrailer(s.trailer)
}
break
}
return 0, err
}
}
}
return s.buf.Read(buf)
}
// Close - this method makes underlying io.ReadCloser's Close method available.
func (s *StreamingUSReader) Close() error {
return s.baseReadCloser.Close()
}

View file

@ -32,13 +32,17 @@ import (
// Reference for constants used below -
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
const (
streamingSignAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
streamingPayloadHdr = "AWS4-HMAC-SHA256-PAYLOAD"
emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
payloadChunkSize = 64 * 1024
chunkSigConstLen = 17 // ";chunk-signature="
signatureStrLen = 64 // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2"
crlfLen = 2 // CRLF
streamingSignAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
streamingSignTrailerAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
streamingPayloadHdr = "AWS4-HMAC-SHA256-PAYLOAD"
streamingTrailerHdr = "AWS4-HMAC-SHA256-TRAILER"
emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
payloadChunkSize = 64 * 1024
chunkSigConstLen = 17 // ";chunk-signature="
signatureStrLen = 64 // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2"
crlfLen = 2 // CRLF
trailerKVSeparator = ":"
trailerSignature = "x-amz-trailer-signature"
)
// Request headers to be ignored while calculating seed signature for
@ -60,7 +64,7 @@ func getSignedChunkLength(chunkDataSize int64) int64 {
}
// getStreamLength - calculates the length of the overall stream (data + metadata)
func getStreamLength(dataLen, chunkSize int64) int64 {
func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
if dataLen <= 0 {
return 0
}
@ -73,6 +77,15 @@ func getStreamLength(dataLen, chunkSize int64) int64 {
streamLen += getSignedChunkLength(remainingBytes)
}
streamLen += getSignedChunkLength(0)
if len(trailers) > 0 {
for name, placeholder := range trailers {
if len(placeholder) > 0 {
streamLen += int64(len(name) + len(trailerKVSeparator) + len(placeholder[0]) + 1)
}
}
streamLen += int64(len(trailerSignature)+len(trailerKVSeparator)) + signatureStrLen + crlfLen + crlfLen
}
return streamLen
}
@ -91,18 +104,41 @@ func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData [
return strings.Join(stringToSignParts, "\n")
}
// buildTrailerChunkStringToSign - returns the string to sign given chunk data
// and previous signature.
func buildTrailerChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string {
stringToSignParts := []string{
streamingTrailerHdr,
t.Format(iso8601DateFormat),
getScope(region, t, ServiceTypeS3),
previousSig,
hex.EncodeToString(sum256(chunkData)),
}
return strings.Join(stringToSignParts, "\n")
}
// prepareStreamingRequest - prepares a request with appropriate
// headers before computing the seed signature.
func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) {
// Set x-amz-content-sha256 header.
req.Header.Set("X-Amz-Content-Sha256", streamingSignAlgorithm)
if len(req.Trailer) == 0 {
req.Header.Set("X-Amz-Content-Sha256", streamingSignAlgorithm)
} else {
req.Header.Set("X-Amz-Content-Sha256", streamingSignTrailerAlgorithm)
for k := range req.Trailer {
req.Header.Add("X-Amz-Trailer", strings.ToLower(k))
}
req.TransferEncoding = []string{"aws-chunked"}
}
if sessionToken != "" {
req.Header.Set("X-Amz-Security-Token", sessionToken)
}
req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat))
// Set content length with streaming signature for each chunk included.
req.ContentLength = getStreamLength(dataLen, int64(payloadChunkSize))
req.ContentLength = getStreamLength(dataLen, int64(payloadChunkSize), req.Trailer)
req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLen, 10))
}
@ -122,6 +158,16 @@ func buildChunkSignature(chunkData []byte, reqTime time.Time, region,
return getSignature(signingKey, chunkStringToSign)
}
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
func buildTrailerChunkSignature(chunkData []byte, reqTime time.Time, region,
previousSignature, secretAccessKey string,
) string {
chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region,
previousSignature, chunkData)
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
return getSignature(signingKey, chunkStringToSign)
}
// getSeedSignature - returns the seed signature for a given request.
func (s *StreamingReader) setSeedSignature(req *http.Request) {
// Get canonical request
@ -156,10 +202,11 @@ type StreamingReader struct {
chunkNum int
totalChunks int
lastChunkSize int
trailer http.Header
}
// signChunk - signs a chunk read from s.baseReader of chunkLen size.
func (s *StreamingReader) signChunk(chunkLen int) {
func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) {
// Compute chunk signature for next header
signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime,
s.region, s.prevSignature, s.secretAccessKey)
@ -175,13 +222,40 @@ func (s *StreamingReader) signChunk(chunkLen int) {
s.buf.Write(s.chunkBuf[:chunkLen])
// Write the chunk trailer.
s.buf.Write([]byte("\r\n"))
if addCrLf {
s.buf.Write([]byte("\r\n"))
}
// Reset chunkBufLen for next chunk read.
s.chunkBufLen = 0
s.chunkNum++
}
// addSignedTrailer - adds a trailer with the provided headers,
// then signs a chunk and adds it to output.
func (s *StreamingReader) addSignedTrailer(h http.Header) {
olen := len(s.chunkBuf)
s.chunkBuf = s.chunkBuf[:0]
for k, v := range h {
s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...)
}
// Compute chunk signature
signature := buildTrailerChunkSignature(s.chunkBuf, s.reqTime,
s.region, s.prevSignature, s.secretAccessKey)
// For next chunk signature computation
s.prevSignature = signature
s.buf.Write(s.chunkBuf)
s.buf.WriteString("\r\n" + trailerSignature + trailerKVSeparator + signature + "\r\n\r\n")
// Reset chunkBufLen for next chunk read.
s.chunkBuf = s.chunkBuf[:olen]
s.chunkBufLen = 0
s.chunkNum++
}
// setStreamingAuthHeader - builds and sets authorization header value
// for streaming signature.
func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) {
@ -222,6 +296,11 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
lastChunkSize: int(dataLen % payloadChunkSize),
}
if len(req.Trailer) > 0 {
stReader.trailer = req.Trailer
// Remove...
req.Trailer = nil
}
// Add the request headers required for chunk upload signing.
@ -272,7 +351,7 @@ func (s *StreamingReader) Read(buf []byte) (int, error) {
(s.chunkNum == s.totalChunks-1 &&
s.chunkBufLen == s.lastChunkSize) {
// Sign the chunk and write it to s.buf.
s.signChunk(s.chunkBufLen)
s.signChunk(s.chunkBufLen, true)
break
}
}
@ -289,7 +368,11 @@ func (s *StreamingReader) Read(buf []byte) (int, error) {
}
// Sign the chunk and write it to s.buf.
s.signChunk(0)
s.signChunk(0, len(s.trailer) == 0)
if len(s.trailer) > 0 {
// Trailer must be set now.
s.addSignedTrailer(s.trailer)
}
break
}
return 0, err

View file

@ -162,11 +162,12 @@ func SignV2(req http.Request, accessKeyID, secretAccessKey string, virtualHost b
// From the Amazon docs:
//
// StringToSign = HTTP-Verb + "\n" +
// Content-Md5 + "\n" +
// Content-Type + "\n" +
// Expires + "\n" +
// CanonicalizedProtocolHeaders +
// CanonicalizedResource;
//
// Content-Md5 + "\n" +
// Content-Type + "\n" +
// Expires + "\n" +
// CanonicalizedProtocolHeaders +
// CanonicalizedResource;
func preStringToSignV2(req http.Request, virtualHost bool) string {
buf := new(bytes.Buffer)
// Write standard headers.
@ -189,11 +190,12 @@ func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) {
// From the Amazon docs:
//
// StringToSign = HTTP-Verb + "\n" +
// Content-Md5 + "\n" +
// Content-Type + "\n" +
// Date + "\n" +
// CanonicalizedProtocolHeaders +
// CanonicalizedResource;
//
// Content-Md5 + "\n" +
// Content-Type + "\n" +
// Date + "\n" +
// CanonicalizedProtocolHeaders +
// CanonicalizedResource;
func stringToSignV2(req http.Request, virtualHost bool) string {
buf := new(bytes.Buffer)
// Write standard headers.
@ -281,8 +283,9 @@ var resourceList = []string{
// From the Amazon docs:
//
// CanonicalizedResource = [ "/" + Bucket ] +
// <HTTP-Request-URI, from the protocol name up to the query string> +
// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
//
// <HTTP-Request-URI, from the protocol name up to the query string> +
// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, virtualHost bool) {
// Save request URL.
requestURL := req.URL

View file

@ -42,7 +42,6 @@ const (
ServiceTypeSTS = "sts"
)
//
// Excerpts from @lsegal -
// https:/github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
//
@ -57,7 +56,6 @@ const (
// * Accept-Encoding
// Some S3 servers like Hitachi Content Platform do not honor this header for signature
// calculation.
//
var v4IgnoredHeaders = map[string]bool{
"Accept-Encoding": true,
"Authorization": true,
@ -177,12 +175,13 @@ func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string {
// getCanonicalRequest generate a canonical request of style.
//
// canonicalRequest =
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
//
// <HTTPMethod>\n
// <CanonicalURI>\n
// <CanonicalQueryString>\n
// <CanonicalHeaders>\n
// <SignedHeaders>\n
// <HashedPayload>
func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool, hashedPayload string) string {
req.URL.RawQuery = strings.ReplaceAll(req.URL.Query().Encode(), "+", "%20")
canonicalRequest := strings.Join([]string{
@ -264,11 +263,11 @@ func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, l
// SignV4STS - signature v4 for STS request.
func SignV4STS(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, "", location, ServiceTypeSTS)
return signV4(req, accessKeyID, secretAccessKey, "", location, ServiceTypeSTS, nil)
}
// Internal function called for different service types.
func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request {
func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string, trailer http.Header) *http.Request {
// Signature calculation is not needed for anonymous credentials.
if accessKeyID == "" || secretAccessKey == "" {
return &req
@ -285,6 +284,15 @@ func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati
req.Header.Set("X-Amz-Security-Token", sessionToken)
}
if len(trailer) > 0 {
for k := range trailer {
req.Header.Add("X-Amz-Trailer", strings.ToLower(k))
}
req.TransferEncoding = []string{"aws-chunked"}
req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(req.ContentLength, 10))
}
hashedPayload := getHashedPayload(req)
if serviceType == ServiceTypeSTS {
// Content sha256 header is not sent with the request
@ -322,11 +330,22 @@ func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, locati
auth := strings.Join(parts, ", ")
req.Header.Set("Authorization", auth)
if len(trailer) > 0 {
// Use custom chunked encoding.
req.Trailer = trailer
return StreamingUnsignedV4(&req, sessionToken, req.ContentLength, time.Now().UTC())
}
return &req
}
// SignV4 sign the request before Do(), in accordance with
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html.
func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3)
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3, nil)
}
// SignV4Trailer sign the request before Do(), in accordance with
// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
func SignV4Trailer(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3, trailer)
}

View file

@ -19,10 +19,9 @@ package signer
import (
"crypto/hmac"
"crypto/sha256"
"net/http"
"strings"
"github.com/minio/sha256-simd"
)
// unsignedPayload - value to be set to X-Amz-Content-Sha256 header when