Added QCOW2 virtual size checking for S3. - Cleaned up S3TemplateDownloader - Created static QCOW2 utils class. - Reformatted some parts of DownloadManagerImpl
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/1c6378ec Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/1c6378ec Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/1c6378ec Branch: refs/heads/master Commit: 1c6378ec0056e8c75990a4a0c15e99b2df162a75 Parents: 1a02773 Author: Boris Schrijver <bo...@pcextreme.nl> Authored: Wed Sep 9 17:53:35 2015 +0200 Committer: Wido den Hollander <w...@widodh.nl> Committed: Fri Sep 11 14:57:32 2015 +0200 ---------------------------------------------------------------------- .../storage/template/S3TemplateDownloader.java | 169 ++++++++++++------- .../storage/template/DownloadManagerImpl.java | 87 +++++++--- .../src/main/java/com/cloud/utils/S3Utils.java | 16 ++ .../com/cloud/utils/storage/QCOW2Utils.java | 60 +++++++ 4 files changed, 244 insertions(+), 88 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1c6378ec/core/src/com/cloud/storage/template/S3TemplateDownloader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java index ec44d8d..ac47dec 100644 --- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java +++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java @@ -27,6 +27,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.Date; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.commons.httpclient.ChunkedInputStream; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Header; @@ -50,10 +52,6 @@ import com.amazonaws.services.s3.model.ProgressEvent; import com.amazonaws.services.s3.model.ProgressListener; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.StorageClass; - -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; - import com.cloud.agent.api.storage.Proxy; import com.cloud.agent.api.to.S3TO; import com.cloud.utils.Pair; @@ -61,46 +59,48 @@ import com.cloud.utils.S3Utils; import com.cloud.utils.UriUtils; /** - * Download a template file using HTTP - * + * Download a template file using HTTP(S) */ public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader { - public static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName()); + private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName()); private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); private String downloadUrl; private String installPath; private String s3Key; private String fileName; - public TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED; - public String errorString = " "; - private long remoteSize = 0; - public long downloadTime = 0; - public long totalBytes; + private String fileExtension; + private String errorString = " "; + + private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED; + private ResourceType resourceType = ResourceType.TEMPLATE; private final HttpClient client; + private final HttpMethodRetryHandler myretryhandler; private GetMethod request; - private boolean resume = false; private DownloadCompleteCallback completionCallback; - private S3TO s3; - private boolean inited = true; + private S3TO s3to; + private long remoteSize = 0; + private long downloadTime = 0; + private long totalBytes; private long maxTemplateSizeInByte; - private ResourceType resourceType = ResourceType.TEMPLATE; - private final HttpMethodRetryHandler myretryhandler; - public S3TemplateDownloader(S3TO storageLayer, String downloadUrl, String installPath, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, - String password, Proxy proxy, ResourceType resourceType) { - s3 = storageLayer; + private boolean resume = false; + private boolean inited = true; + + public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback, + long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { + this.s3to = s3to; this.downloadUrl = downloadUrl; this.installPath = installPath; - status = TemplateDownloader.Status.NOT_STARTED; + this.status = TemplateDownloader.Status.NOT_STARTED; this.resourceType = resourceType; - maxTemplateSizeInByte = maxTemplateSizeInBytes; + this.maxTemplateSizeInByte = maxTemplateSizeInBytes; - totalBytes = 0; - client = new HttpClient(s_httpClientManager); + this.totalBytes = 0; + this.client = new HttpClient(s_httpClientManager); - myretryhandler = new HttpMethodRetryHandler() { + this.myretryhandler = new HttpMethodRetryHandler() { @Override public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) { if (executionCount >= 2) { @@ -128,6 +128,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp Pair<String, Integer> hostAndPort = UriUtils.validateUrl(downloadUrl); fileName = StringUtils.substringAfterLast(downloadUrl, "/"); + fileExtension = StringUtils.substringAfterLast(fileName, "."); if (proxy != null) { client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort()); @@ -139,8 +140,10 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp if ((user != null) && (password != null)) { client.getParams().setAuthenticationPreemptive(true); Credentials defaultcreds = new UsernamePasswordCredentials(user, password); - client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); - s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second()); + client.getState().setCredentials( + new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); + s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + + ":" + hostAndPort.second()); } else { s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second()); } @@ -160,11 +163,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp @Override public long download(boolean resume, DownloadCompleteCallback callback) { switch (status) { - case ABORTED: - case UNRECOVERABLE_ERROR: - case DOWNLOAD_FINISHED: - return 0; - default: + case ABORTED: + case UNRECOVERABLE_ERROR: + case DOWNLOAD_FINISHED: + return 0; + default: } @@ -215,10 +218,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp contentType = contentTypeHeader.getValue(); } - InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream()) : new ChunkedInputStream(request.getResponseBodyAsStream()); + InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream()) + : new ChunkedInputStream(request.getResponseBodyAsStream()); - s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3.getBucketName() + " remoteSize=" + remoteSize + " , max size=" + - maxTemplateSizeInByte); + s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName() + + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte); Date start = new Date(); // compute s3 key @@ -230,9 +234,9 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp if (contentType != null) { metadata.setContentType(contentType); } - PutObjectRequest putObjectRequest = new PutObjectRequest(s3.getBucketName(), s3Key, in, metadata); + PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata); // check if RRS is enabled - if (s3.getEnableRRS()) { + if (s3to.getEnableRRS()) { putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy); } // register progress listenser @@ -257,14 +261,15 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp }); - if (!s3.getSingleUpload(remoteSize)) { + if (!s3to.getSingleUpload(remoteSize)) { // use TransferManager to do multipart upload - S3Utils.mputObject(s3, putObjectRequest); + S3Utils.mputObject(s3to, putObjectRequest); } else { // single part upload, with 5GB limit in Amazon - S3Utils.putObject(s3, putObjectRequest); - while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR && - status != TemplateDownloader.Status.ABORTED) { + S3Utils.putObject(s3to, putObjectRequest); + while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED + && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR + && status != TemplateDownloader.Status.ABORTED) { // wait for completion } } @@ -324,32 +329,59 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp return totalBytes; } + /** + * Returns an InputStream only when the status is DOWNLOAD_FINISHED. + * + * The caller of this method must close the InputStream to prevent resource leaks! + * + * @return S3ObjectInputStream of the object. + */ + public InputStream getS3ObjectInputStream() { + // Check if the download is finished + if (status != Status.DOWNLOAD_FINISHED) { + return null; + } + + return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key); + } + + public void cleanupAfterError() { + if (status != Status.UNRECOVERABLE_ERROR) { + s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry."); + return; + } + + s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key); + + S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key); + } + @Override @SuppressWarnings("fallthrough") public boolean stopDownload() { switch (getStatus()) { - case IN_PROGRESS: - if (request != null) { - request.abort(); - } - status = TemplateDownloader.Status.ABORTED; - return true; - case UNKNOWN: - case NOT_STARTED: - case RECOVERABLE_ERROR: - case UNRECOVERABLE_ERROR: - case ABORTED: - status = TemplateDownloader.Status.ABORTED; - case DOWNLOAD_FINISHED: - try { - S3Utils.deleteObject(s3, s3.getBucketName(), s3Key); - } catch (Exception ex) { - // ignore delete exception if it is not there - } - return true; + case IN_PROGRESS: + if (request != null) { + request.abort(); + } + status = TemplateDownloader.Status.ABORTED; + return true; + case UNKNOWN: + case NOT_STARTED: + case RECOVERABLE_ERROR: + case UNRECOVERABLE_ERROR: + case ABORTED: + status = TemplateDownloader.Status.ABORTED; + case DOWNLOAD_FINISHED: + try { + S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key); + } catch (Exception ex) { + // ignore delete exception if it is not there + } + return true; - default: - return true; + default: + return true; } } @@ -359,7 +391,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp return 0; } - return (int)(100.0 * totalBytes / remoteSize); + return (int) (100.0 * totalBytes / remoteSize); } @Override @@ -417,4 +449,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp return resourceType; } -} + public long getTotalBytes() { + return totalBytes; + } + + public String getFileExtension() { + return fileExtension; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1c6378ec/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 25c0887..2e4bb74 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -41,13 +41,12 @@ import java.util.concurrent.Executors; import javax.ejb.Local; import javax.naming.ConfigurationException; -import org.apache.log4j.Logger; - import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; import org.apache.cloudstack.storage.resource.SecondaryStorageResource; +import org.apache.log4j.Logger; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.Proxy; @@ -67,9 +66,9 @@ import com.cloud.storage.template.Processor; import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.storage.template.QCOW2Processor; import com.cloud.storage.template.RawImageProcessor; -import com.cloud.storage.template.TARProcessor; import com.cloud.storage.template.S3TemplateDownloader; import com.cloud.storage.template.ScpTemplateDownloader; +import com.cloud.storage.template.TARProcessor; import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.template.TemplateDownloader; import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback; @@ -83,6 +82,7 @@ import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +import com.cloud.utils.storage.QCOW2Utils; @Local(value = DownloadManager.class) public class DownloadManagerImpl extends ManagerBase implements DownloadManager { @@ -129,10 +129,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager this.tmpltName = tmpltName; this.format = format; this.hvm = hvm; - description = descr; - checksum = cksum; + this.description = descr; + this.checksum = cksum; this.installPathPrefix = installPathPrefix; - templatesize = 0; + this.templatesize = 0; this.id = id; this.resourceType = resourceType; } @@ -276,11 +276,27 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager threadPool.execute(td); break; case DOWNLOAD_FINISHED: - if (!(td instanceof S3TemplateDownloader)) { - // we currently only create template.properties for NFS by - // running some post download script + if(td instanceof S3TemplateDownloader) { + // For S3 and Swift, which are considered "remote", + // as in the file cannot be accessed locally, + // we run the postRemoteDownload() method. + td.setDownloadError("Download success, starting install "); + String result = postRemoteDownload(jobId); + if (result != null) { + s_logger.error("Failed post download install: " + result); + td.setStatus(Status.UNRECOVERABLE_ERROR); + td.setDownloadError("Failed post download install: " + result); + ((S3TemplateDownloader) td).cleanupAfterError(); + } else { + td.setStatus(Status.POST_DOWNLOAD_FINISHED); + td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); + } + } + else { + // For other TemplateDownloaders where files are locally available, + // we run the postLocalDownload() method. td.setDownloadError("Download success, starting install "); - String result = postDownload(jobId); + String result = postLocalDownload(jobId); if (result != null) { s_logger.error("Failed post download script: " + result); td.setStatus(Status.UNRECOVERABLE_ERROR); @@ -289,17 +305,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager td.setStatus(Status.POST_DOWNLOAD_FINISHED); td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); } - } else { - // for s3 and swift, we skip post download step and just set - // status to trigger callback. - td.setStatus(Status.POST_DOWNLOAD_FINISHED); - // set template size for S3 - S3TemplateDownloader std = (S3TemplateDownloader)td; - long size = std.totalBytes; - DownloadJob dnld = jobs.get(jobId); - dnld.setTemplatesize(size); - dnld.setTemplatePhysicalSize(size); - dnld.setTmpltPath(std.getDownloadLocalPath()); // update template path to include file name. } dj.cleanup(); break; @@ -339,12 +344,48 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } /** - * Post download activity (install and cleanup). Executed in context of + * Post remote download activity (install and cleanup). Executed in context of the downloader thread. + */ + private String postRemoteDownload(String jobId) { + String result = null; + DownloadJob dnld = jobs.get(jobId); + S3TemplateDownloader td = (S3TemplateDownloader)dnld.getTemplateDownloader(); + + if (td.getFileExtension().equalsIgnoreCase("QCOW2")) { + // The QCOW2 is the only format with a header, + // and as such can be easily read. + + try { + InputStream inputStream = td.getS3ObjectInputStream(); + + dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream)); + + inputStream.close(); + } + catch (IOException e) { + result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage(); + } + + } + else { + // For the other formats, both the virtual + // and actual file size are set the same. + dnld.setTemplatesize(td.getTotalBytes()); + } + + dnld.setTemplatePhysicalSize(td.getTotalBytes()); + dnld.setTmpltPath(td.getDownloadLocalPath()); + + return result; + } + + /** + * Post local download activity (install and cleanup). Executed in context of * downloader thread * * @throws IOException */ - private String postDownload(String jobId) { + private String postLocalDownload(String jobId) { DownloadJob dnld = jobs.get(jobId); TemplateDownloader td = dnld.getTemplateDownloader(); String resourcePath = dnld.getInstallPathPrefix(); // path with mount http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1c6378ec/utils/src/main/java/com/cloud/utils/S3Utils.java ---------------------------------------------------------------------- diff --git a/utils/src/main/java/com/cloud/utils/S3Utils.java b/utils/src/main/java/com/cloud/utils/S3Utils.java index 6efe76b..c07db33 100644 --- a/utils/src/main/java/com/cloud/utils/S3Utils.java +++ b/utils/src/main/java/com/cloud/utils/S3Utils.java @@ -62,6 +62,7 @@ import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.Upload; @@ -256,6 +257,21 @@ public final class S3Utils { } + // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak. + public static S3ObjectInputStream getObjectStream(final ClientOptions clientOptions, final String bucketName, final String key) { + + assert clientOptions != null; + assert !isBlank(bucketName); + assert !isBlank(key); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName)); + } + + return acquireClient(clientOptions).getObject(bucketName, key).getObjectContent(); + + } + @SuppressWarnings("unchecked") public static File getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File targetDirectory, final FileNamingStrategy namingStrategy) { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1c6378ec/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java ---------------------------------------------------------------------- diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java new file mode 100644 index 0000000..3e08bd6 --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -0,0 +1,60 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.storage; + +import java.io.IOException; +import java.io.InputStream; + +import com.cloud.utils.NumbersUtil; + +public final class QCOW2Utils { + private static final int VIRTUALSIZE_HEADER_LOCATION = 24; + private static final int VIRTUALSIZE_HEADER_LENGTH = 8; + + /** + * Private constructor -> This utility class cannot be instantiated. + */ + private QCOW2Utils() {} + + /** + * @return the header location of the virtual size field. + */ + public static int getVirtualSizeHeaderLocation() { + return VIRTUALSIZE_HEADER_LOCATION; + } + + /** + * @param inputStream The QCOW2 object in stream format. + * @return The virtual size of the QCOW2 object. + */ + public static long getVirtualSize(InputStream inputStream) throws IOException { + byte[] bytes = new byte[VIRTUALSIZE_HEADER_LENGTH]; + + if (inputStream.skip(VIRTUALSIZE_HEADER_LOCATION) != VIRTUALSIZE_HEADER_LOCATION) { + throw new IOException("Unable to skip to the virtual size header"); + } + + if (inputStream.read(bytes) != VIRTUALSIZE_HEADER_LENGTH) { + throw new IOException("Unable to properly read the size"); + } + + return NumbersUtil.bytesToLong(bytes); + } +} \ No newline at end of file
