Repository: libcloud Updated Branches: refs/heads/trunk afc1aefb2 -> a1ebd1448
Fixed upload/download streams Signed-off-by: Rick van de Loo <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/90399682 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/90399682 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/90399682 Branch: refs/heads/trunk Commit: 9039968249cba20a546e5d1eb54ad2efbfa79f43 Parents: afc1aef Author: Michael Perel <[email protected]> Authored: Thu Aug 2 13:29:46 2018 -0400 Committer: Rick van de Loo <[email protected]> Committed: Wed Aug 8 03:58:31 2018 +0200 ---------------------------------------------------------------------- libcloud/storage/drivers/azure_blobs.py | 27 ++++++++++++++++++++++---- libcloud/test/storage/test_azure_blobs.py | 4 +++- 2 files changed, 26 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/90399682/libcloud/storage/drivers/azure_blobs.py ---------------------------------------------------------------------- diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py index a53f6ac..ae49a1d 100644 --- a/libcloud/storage/drivers/azure_blobs.py +++ b/libcloud/storage/drivers/azure_blobs.py @@ -579,11 +579,14 @@ class AzureBlobsStorageDriver(StorageDriver): @inherits: :class:`StorageDriver.download_object_as_stream` """ obj_path = self._get_object_path(obj.container, obj.name) - response = self.connection.request(obj_path, raw=True, data=None) + response = self.connection.request(obj_path, method='GET', + stream=True, raw=True) + iterator = response.iter_content(AZURE_CHUNK_SIZE) - return self._get_object(obj=obj, callback=read_in_chunks, + return self._get_object(obj=obj, + callback=read_in_chunks, response=response, - callback_kwargs={'iterator': response.response, + callback_kwargs={'iterator': iterator, 'chunk_size': chunk_size}, success_status_code=httplib.OK) @@ -810,11 +813,27 @@ class AzureBlobsStorageDriver(StorageDriver): if ex_blob_type is None: ex_blob_type = self.ex_blob_type + """ + Azure requires that for page blobs that a maximum size that the page + can grow to. For block blobs, it is required that the Content-Length + header be set. The size of the block blob will be the total size of + the stream minus the current position, so in this case + ex_page_blob_size should be 0 (and will be checked in + self._check_values). + Source: + https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob + """ self._check_values(ex_blob_type, ex_page_blob_size) + if ex_blob_type == "BlockBlob": + iterator.seek(0, os.SEEK_END) + blob_size = iterator.tell() + iterator.seek(0) + else: + blob_size = ex_page_blob_size return self._put_object(container=container, object_name=object_name, - object_size=ex_page_blob_size, + object_size=blob_size, extra=extra, verify_hash=verify_hash, blob_type=ex_blob_type, use_lease=ex_use_lease, http://git-wip-us.apache.org/repos/asf/libcloud/blob/90399682/libcloud/test/storage/test_azure_blobs.py ---------------------------------------------------------------------- diff --git a/libcloud/test/storage/test_azure_blobs.py b/libcloud/test/storage/test_azure_blobs.py index 69bdebb..aa09c66 100644 --- a/libcloud/test/storage/test_azure_blobs.py +++ b/libcloud/test/storage/test_azure_blobs.py @@ -647,7 +647,9 @@ class AzureBlobsTests(unittest.TestCase): stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) - self.assertTrue(hasattr(stream, '__iter__')) + + consumed_stream = ''.join(chunk.decode('utf-8') for chunk in stream) + self.assertEqual(len(consumed_stream), obj.size) def test_upload_object_invalid_ex_blob_type(self): # Invalid hash is detected on the amazon side and BAD_REQUEST is
