c76ad94492
This commit adds a build step to travis to auto-delete unstable archives older than 14 days (our regular release schedule) from Azure via ci.go purge. The commit also pulls in the latest Azure storage code, also switching over from the old import path (github.com/Azure/azure-sdk-for-go) to the new split one (github.com/Azure/azure-storage-go).
480 lines
13 KiB
Go
480 lines
13 KiB
Go
// Package storage provides clients for Microsoft Azure Storage Services.
|
|
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"net/url"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/Azure/go-autorest/autorest/azure"
|
|
)
|
|
|
|
const (
|
|
// DefaultBaseURL is the domain name used for storage requests in the
|
|
// public cloud when a default client is created.
|
|
DefaultBaseURL = "core.windows.net"
|
|
|
|
// DefaultAPIVersion is the Azure Storage API version string used when a
|
|
// basic client is created.
|
|
DefaultAPIVersion = "2016-05-31"
|
|
|
|
defaultUseHTTPS = true
|
|
|
|
// StorageEmulatorAccountName is the fixed storage account used by Azure Storage Emulator
|
|
StorageEmulatorAccountName = "devstoreaccount1"
|
|
|
|
// StorageEmulatorAccountKey is the the fixed storage account used by Azure Storage Emulator
|
|
StorageEmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
|
|
|
blobServiceName = "blob"
|
|
tableServiceName = "table"
|
|
queueServiceName = "queue"
|
|
fileServiceName = "file"
|
|
|
|
storageEmulatorBlob = "127.0.0.1:10000"
|
|
storageEmulatorTable = "127.0.0.1:10002"
|
|
storageEmulatorQueue = "127.0.0.1:10001"
|
|
|
|
userAgentHeader = "User-Agent"
|
|
)
|
|
|
|
// Client is the object that needs to be constructed to perform
|
|
// operations on the storage account.
|
|
type Client struct {
|
|
// HTTPClient is the http.Client used to initiate API
|
|
// requests. If it is nil, http.DefaultClient is used.
|
|
HTTPClient *http.Client
|
|
|
|
accountName string
|
|
accountKey []byte
|
|
useHTTPS bool
|
|
UseSharedKeyLite bool
|
|
baseURL string
|
|
apiVersion string
|
|
userAgent string
|
|
}
|
|
|
|
type storageResponse struct {
|
|
statusCode int
|
|
headers http.Header
|
|
body io.ReadCloser
|
|
}
|
|
|
|
type odataResponse struct {
|
|
storageResponse
|
|
odata odataErrorMessage
|
|
}
|
|
|
|
// AzureStorageServiceError contains fields of the error response from
|
|
// Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx
|
|
// Some fields might be specific to certain calls.
|
|
type AzureStorageServiceError struct {
|
|
Code string `xml:"Code"`
|
|
Message string `xml:"Message"`
|
|
AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"`
|
|
QueryParameterName string `xml:"QueryParameterName"`
|
|
QueryParameterValue string `xml:"QueryParameterValue"`
|
|
Reason string `xml:"Reason"`
|
|
StatusCode int
|
|
RequestID string
|
|
}
|
|
|
|
type odataErrorMessageMessage struct {
|
|
Lang string `json:"lang"`
|
|
Value string `json:"value"`
|
|
}
|
|
|
|
type odataErrorMessageInternal struct {
|
|
Code string `json:"code"`
|
|
Message odataErrorMessageMessage `json:"message"`
|
|
}
|
|
|
|
type odataErrorMessage struct {
|
|
Err odataErrorMessageInternal `json:"odata.error"`
|
|
}
|
|
|
|
// UnexpectedStatusCodeError is returned when a storage service responds with neither an error
|
|
// nor with an HTTP status code indicating success.
|
|
type UnexpectedStatusCodeError struct {
|
|
allowed []int
|
|
got int
|
|
}
|
|
|
|
func (e UnexpectedStatusCodeError) Error() string {
|
|
s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) }
|
|
|
|
got := s(e.got)
|
|
expected := []string{}
|
|
for _, v := range e.allowed {
|
|
expected = append(expected, s(v))
|
|
}
|
|
return fmt.Sprintf("storage: status code from service response is %s; was expecting %s", got, strings.Join(expected, " or "))
|
|
}
|
|
|
|
// Got is the actual status code returned by Azure.
|
|
func (e UnexpectedStatusCodeError) Got() int {
|
|
return e.got
|
|
}
|
|
|
|
// NewBasicClient constructs a Client with given storage service name and
|
|
// key.
|
|
func NewBasicClient(accountName, accountKey string) (Client, error) {
|
|
if accountName == StorageEmulatorAccountName {
|
|
return NewEmulatorClient()
|
|
}
|
|
return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS)
|
|
}
|
|
|
|
// NewBasicClientOnSovereignCloud constructs a Client with given storage service name and
|
|
// key in the referenced cloud.
|
|
func NewBasicClientOnSovereignCloud(accountName, accountKey string, env azure.Environment) (Client, error) {
|
|
if accountName == StorageEmulatorAccountName {
|
|
return NewEmulatorClient()
|
|
}
|
|
return NewClient(accountName, accountKey, env.StorageEndpointSuffix, DefaultAPIVersion, defaultUseHTTPS)
|
|
}
|
|
|
|
//NewEmulatorClient contructs a Client intended to only work with Azure
|
|
//Storage Emulator
|
|
func NewEmulatorClient() (Client, error) {
|
|
return NewClient(StorageEmulatorAccountName, StorageEmulatorAccountKey, DefaultBaseURL, DefaultAPIVersion, false)
|
|
}
|
|
|
|
// NewClient constructs a Client. This should be used if the caller wants
|
|
// to specify whether to use HTTPS, a specific REST API version or a custom
|
|
// storage endpoint than Azure Public Cloud.
|
|
func NewClient(accountName, accountKey, blobServiceBaseURL, apiVersion string, useHTTPS bool) (Client, error) {
|
|
var c Client
|
|
if accountName == "" {
|
|
return c, fmt.Errorf("azure: account name required")
|
|
} else if accountKey == "" {
|
|
return c, fmt.Errorf("azure: account key required")
|
|
} else if blobServiceBaseURL == "" {
|
|
return c, fmt.Errorf("azure: base storage service url required")
|
|
}
|
|
|
|
key, err := base64.StdEncoding.DecodeString(accountKey)
|
|
if err != nil {
|
|
return c, fmt.Errorf("azure: malformed storage account key: %v", err)
|
|
}
|
|
|
|
c = Client{
|
|
accountName: accountName,
|
|
accountKey: key,
|
|
useHTTPS: useHTTPS,
|
|
baseURL: blobServiceBaseURL,
|
|
apiVersion: apiVersion,
|
|
UseSharedKeyLite: false,
|
|
}
|
|
c.userAgent = c.getDefaultUserAgent()
|
|
return c, nil
|
|
}
|
|
|
|
func (c Client) getDefaultUserAgent() string {
|
|
return fmt.Sprintf("Go/%s (%s-%s) Azure-SDK-For-Go/%s storage-dataplane/%s",
|
|
runtime.Version(),
|
|
runtime.GOARCH,
|
|
runtime.GOOS,
|
|
sdkVersion,
|
|
c.apiVersion,
|
|
)
|
|
}
|
|
|
|
// AddToUserAgent adds an extension to the current user agent
|
|
func (c *Client) AddToUserAgent(extension string) error {
|
|
if extension != "" {
|
|
c.userAgent = fmt.Sprintf("%s %s", c.userAgent, extension)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("Extension was empty, User Agent stayed as %s", c.userAgent)
|
|
}
|
|
|
|
// protectUserAgent is used in funcs that include extraheaders as a parameter.
|
|
// It prevents the User-Agent header to be overwritten, instead if it happens to
|
|
// be present, it gets added to the current User-Agent. Use it before getStandardHeaders
|
|
func (c *Client) protectUserAgent(extraheaders map[string]string) map[string]string {
|
|
if v, ok := extraheaders[userAgentHeader]; ok {
|
|
c.AddToUserAgent(v)
|
|
delete(extraheaders, userAgentHeader)
|
|
}
|
|
return extraheaders
|
|
}
|
|
|
|
func (c Client) getBaseURL(service string) string {
|
|
scheme := "http"
|
|
if c.useHTTPS {
|
|
scheme = "https"
|
|
}
|
|
host := ""
|
|
if c.accountName == StorageEmulatorAccountName {
|
|
switch service {
|
|
case blobServiceName:
|
|
host = storageEmulatorBlob
|
|
case tableServiceName:
|
|
host = storageEmulatorTable
|
|
case queueServiceName:
|
|
host = storageEmulatorQueue
|
|
}
|
|
} else {
|
|
host = fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseURL)
|
|
}
|
|
|
|
u := &url.URL{
|
|
Scheme: scheme,
|
|
Host: host}
|
|
return u.String()
|
|
}
|
|
|
|
func (c Client) getEndpoint(service, path string, params url.Values) string {
|
|
u, err := url.Parse(c.getBaseURL(service))
|
|
if err != nil {
|
|
// really should not be happening
|
|
panic(err)
|
|
}
|
|
|
|
// API doesn't accept path segments not starting with '/'
|
|
if !strings.HasPrefix(path, "/") {
|
|
path = fmt.Sprintf("/%v", path)
|
|
}
|
|
|
|
if c.accountName == StorageEmulatorAccountName {
|
|
path = fmt.Sprintf("/%v%v", StorageEmulatorAccountName, path)
|
|
}
|
|
|
|
u.Path = path
|
|
u.RawQuery = params.Encode()
|
|
return u.String()
|
|
}
|
|
|
|
// GetBlobService returns a BlobStorageClient which can operate on the blob
|
|
// service of the storage account.
|
|
func (c Client) GetBlobService() BlobStorageClient {
|
|
b := BlobStorageClient{
|
|
client: c,
|
|
}
|
|
b.client.AddToUserAgent(blobServiceName)
|
|
b.auth = sharedKey
|
|
if c.UseSharedKeyLite {
|
|
b.auth = sharedKeyLite
|
|
}
|
|
return b
|
|
}
|
|
|
|
// GetQueueService returns a QueueServiceClient which can operate on the queue
|
|
// service of the storage account.
|
|
func (c Client) GetQueueService() QueueServiceClient {
|
|
q := QueueServiceClient{
|
|
client: c,
|
|
}
|
|
q.client.AddToUserAgent(queueServiceName)
|
|
q.auth = sharedKey
|
|
if c.UseSharedKeyLite {
|
|
q.auth = sharedKeyLite
|
|
}
|
|
return q
|
|
}
|
|
|
|
// GetTableService returns a TableServiceClient which can operate on the table
|
|
// service of the storage account.
|
|
func (c Client) GetTableService() TableServiceClient {
|
|
t := TableServiceClient{
|
|
client: c,
|
|
}
|
|
t.client.AddToUserAgent(tableServiceName)
|
|
t.auth = sharedKeyForTable
|
|
if c.UseSharedKeyLite {
|
|
t.auth = sharedKeyLiteForTable
|
|
}
|
|
return t
|
|
}
|
|
|
|
// GetFileService returns a FileServiceClient which can operate on the file
|
|
// service of the storage account.
|
|
func (c Client) GetFileService() FileServiceClient {
|
|
f := FileServiceClient{
|
|
client: c,
|
|
}
|
|
f.client.AddToUserAgent(fileServiceName)
|
|
f.auth = sharedKey
|
|
if c.UseSharedKeyLite {
|
|
f.auth = sharedKeyLite
|
|
}
|
|
return f
|
|
}
|
|
|
|
func (c Client) getStandardHeaders() map[string]string {
|
|
return map[string]string{
|
|
userAgentHeader: c.userAgent,
|
|
"x-ms-version": c.apiVersion,
|
|
"x-ms-date": currentTimeRfc1123Formatted(),
|
|
}
|
|
}
|
|
|
|
func (c Client) exec(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*storageResponse, error) {
|
|
headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
req, err := http.NewRequest(verb, url, body)
|
|
if err != nil {
|
|
return nil, errors.New("azure/storage: error creating request: " + err.Error())
|
|
}
|
|
|
|
if clstr, ok := headers["Content-Length"]; ok {
|
|
// content length header is being signed, but completely ignored by golang.
|
|
// instead we have to use the ContentLength property on the request struct
|
|
// (see https://golang.org/src/net/http/request.go?s=18140:18370#L536 and
|
|
// https://golang.org/src/net/http/transfer.go?s=1739:2467#L49)
|
|
req.ContentLength, err = strconv.ParseInt(clstr, 10, 64)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
for k, v := range headers {
|
|
req.Header.Add(k, v)
|
|
}
|
|
|
|
httpClient := c.HTTPClient
|
|
if httpClient == nil {
|
|
httpClient = http.DefaultClient
|
|
}
|
|
resp, err := httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
statusCode := resp.StatusCode
|
|
if statusCode >= 400 && statusCode <= 505 {
|
|
var respBody []byte
|
|
respBody, err = readAndCloseBody(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
requestID := resp.Header.Get("x-ms-request-id")
|
|
if len(respBody) == 0 {
|
|
// no error in response body, might happen in HEAD requests
|
|
err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID)
|
|
} else {
|
|
// response contains storage service error object, unmarshal
|
|
storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, requestID)
|
|
if err != nil { // error unmarshaling the error response
|
|
err = errIn
|
|
}
|
|
err = storageErr
|
|
}
|
|
return &storageResponse{
|
|
statusCode: resp.StatusCode,
|
|
headers: resp.Header,
|
|
body: ioutil.NopCloser(bytes.NewReader(respBody)), /* restore the body */
|
|
}, err
|
|
}
|
|
|
|
return &storageResponse{
|
|
statusCode: resp.StatusCode,
|
|
headers: resp.Header,
|
|
body: resp.Body}, nil
|
|
}
|
|
|
|
func (c Client) execInternalJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) {
|
|
headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
req, err := http.NewRequest(verb, url, body)
|
|
for k, v := range headers {
|
|
req.Header.Add(k, v)
|
|
}
|
|
|
|
httpClient := c.HTTPClient
|
|
if httpClient == nil {
|
|
httpClient = http.DefaultClient
|
|
}
|
|
|
|
resp, err := httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
respToRet := &odataResponse{}
|
|
respToRet.body = resp.Body
|
|
respToRet.statusCode = resp.StatusCode
|
|
respToRet.headers = resp.Header
|
|
|
|
statusCode := resp.StatusCode
|
|
if statusCode >= 400 && statusCode <= 505 {
|
|
var respBody []byte
|
|
respBody, err = readAndCloseBody(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(respBody) == 0 {
|
|
// no error in response body, might happen in HEAD requests
|
|
err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, resp.Header.Get("x-ms-request-id"))
|
|
return respToRet, err
|
|
}
|
|
// try unmarshal as odata.error json
|
|
err = json.Unmarshal(respBody, &respToRet.odata)
|
|
return respToRet, err
|
|
}
|
|
|
|
return respToRet, nil
|
|
}
|
|
|
|
func readAndCloseBody(body io.ReadCloser) ([]byte, error) {
|
|
defer body.Close()
|
|
out, err := ioutil.ReadAll(body)
|
|
if err == io.EOF {
|
|
err = nil
|
|
}
|
|
return out, err
|
|
}
|
|
|
|
func serviceErrFromXML(body []byte, statusCode int, requestID string) (AzureStorageServiceError, error) {
|
|
var storageErr AzureStorageServiceError
|
|
if err := xml.Unmarshal(body, &storageErr); err != nil {
|
|
return storageErr, err
|
|
}
|
|
storageErr.StatusCode = statusCode
|
|
storageErr.RequestID = requestID
|
|
return storageErr, nil
|
|
}
|
|
|
|
func serviceErrFromStatusCode(code int, status string, requestID string) AzureStorageServiceError {
|
|
return AzureStorageServiceError{
|
|
StatusCode: code,
|
|
Code: status,
|
|
RequestID: requestID,
|
|
Message: "no response body was available for error status code",
|
|
}
|
|
}
|
|
|
|
func (e AzureStorageServiceError) Error() string {
|
|
return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s, QueryParameterName=%s, QueryParameterValue=%s",
|
|
e.StatusCode, e.Code, e.Message, e.RequestID, e.QueryParameterName, e.QueryParameterValue)
|
|
}
|
|
|
|
// checkRespCode returns UnexpectedStatusError if the given response code is not
|
|
// one of the allowed status codes; otherwise nil.
|
|
func checkRespCode(respCode int, allowed []int) error {
|
|
for _, v := range allowed {
|
|
if respCode == v {
|
|
return nil
|
|
}
|
|
}
|
|
return UnexpectedStatusCodeError{allowed, respCode}
|
|
}
|