CLOUDSTACK-9677: Adding storage policy support for swift as secondary…

api/src/com/cloud/agent/api/to/SwiftTO.java

@@ -26,17 +26,19 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
 
     String userName;
     String key;
+    String storagePolicy;
     private static final String pathSeparator = "/";
 
     public SwiftTO() {
     }
 
-    public SwiftTO(Long id, String url, String account, String userName, String key) {
+    public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) {
         this.id = id;
         this.url = url;
         this.account = account;
         this.userName = userName;
         this.key = key;
+        this.storagePolicy = storagePolicy;
     }
 
     public Long getId() {
@@ -63,6 +65,11 @@ public String getKey() {
         return key;
     }
 
+    @Override
+    public String getStoragePolicy() {
+        return this.storagePolicy;
+    }
+
     @Override
     public DataStoreRole getRole() {
         return DataStoreRole.Image;

api/src/org/apache/cloudstack/api/ApiConstants.java

@@ -287,6 +287,7 @@ public class ApiConstants {
     public static final String STATE = "state";
     public static final String STATUS = "status";
     public static final String STORAGE_TYPE = "storagetype";
+    public static final String STORAGE_POLICY = "storagepolicy";
     public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
     public static final String STORAGE_CAPABILITIES = "storagecapabilities";
     public static final String SYSTEM_VM_TYPE = "systemvmtype";

plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java

@@ -18,23 +18,24 @@
  */
 package com.cloud.hypervisor.xenserver.resource;
 
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.to.DataObjectType;
-import com.cloud.agent.api.to.DataStoreTO;
-import com.cloud.agent.api.to.DataTO;
-import com.cloud.agent.api.to.DiskTO;
-import com.cloud.agent.api.to.NfsTO;
-import com.cloud.agent.api.to.S3TO;
-import com.cloud.agent.api.to.SwiftTO;
-import com.cloud.exception.InternalErrorException;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
-import com.cloud.storage.DataStoreRole;
-import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.ImageFormat;
-import com.cloud.storage.resource.StorageProcessor;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.storage.S3.ClientOptions;
+import static com.cloud.utils.ReflectUtil.flattenProperties;
+import static com.google.common.collect.Lists.newArrayList;
+
+import java.io.File;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.log4j.Logger;
+import org.apache.xmlrpc.XmlRpcException;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.xensource.xenapi.Connection;
 import com.xensource.xenapi.SR;
 import com.xensource.xenapi.Types;
@@ -44,6 +45,7 @@
 import com.xensource.xenapi.VBD;
 import com.xensource.xenapi.VDI;
 import com.xensource.xenapi.VM;
+
 import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
 import org.apache.cloudstack.storage.command.AttachAnswer;
 import org.apache.cloudstack.storage.command.AttachCommand;
@@ -65,20 +67,24 @@
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.log4j.Logger;
-import org.apache.xmlrpc.XmlRpcException;
-
-import java.io.File;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
 
-import static com.cloud.utils.ReflectUtil.flattenProperties;
-import static com.google.common.collect.Lists.newArrayList;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.to.DataObjectType;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.DataTO;
+import com.cloud.agent.api.to.DiskTO;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.exception.InternalErrorException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.resource.StorageProcessor;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.storage.S3.ClientOptions;
 
 public class XenServerStorageProcessor implements StorageProcessor {
     private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
@@ -914,20 +920,55 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) {
 
     private boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI,
             final int wait) {
-        String result = null;
+
+        List<String> params = getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
         try {
-            result =
-                    hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username",
-                            swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString());
-            if (result != null && result.equals("true")) {
-                return true;
-            }
+            String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
+            return BooleanUtils.toBoolean(result);
         } catch (final Exception e) {
             s_logger.warn("swift upload failed due to " + e.toString(), e);
         }
         return false;
     }
 
+    @VisibleForTesting
+    List<String> getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename, Boolean isISCSI) {
+        // ORDER IS IMPORTANT
+        List<String> params = new ArrayList<>();
+
+        //operation
+        params.add("op");
+        params.add("upload");
+
+        //auth
+        params.add("url");
+        params.add(swift.getUrl());
+        params.add("account");
+        params.add(swift.getAccount());
+        params.add("username");
+        params.add(swift.getUserName());
+        params.add("key");
+        params.add(swift.getKey());
+
+        // object info
+        params.add("container");
+        params.add(container);
+        params.add("ldir");
+        params.add(ldir);
+        params.add("lfilename");
+        params.add(lfilename);
+        params.add("isISCSI");
+        params.add(isISCSI.toString());
+
+        if (swift.getStoragePolicy() != null) {
+            params.add("storagepolicy");
+            params.add(swift.getStoragePolicy());
+        }
+
+        return params;
+    }
+
     protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
 
         // If anybody modifies the formatting below again, I'll skin them

plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.hypervisor.xenserver.resource;
+
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.cloud.agent.api.to.SwiftTO;
+
+public class XenServerStorageProcessorTest {
+    @Test
+    public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() {
+        CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
+        XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
+
+        SwiftTO swift = Mockito.mock(SwiftTO.class);
+        when(swift.getStoragePolicy()).thenReturn(null);
+
+        String container = "sample-container-name";
+        String ldir = "sample-ldir";
+        String lfilename = "sample-lfilename";
+        Boolean isISCSI = true;
+
+        List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
+        // make sure the params not null and has correct number of items in it
+        Assert.assertNotNull("params is null", params);
+        Assert.assertTrue("Expected param list size is 18 but it was" + params.size(), params.size() == 18);
+
+        // check the order of params
+        Assert.assertEquals("unexpected param.", "op", params.get(0));
+        Assert.assertEquals("unexpected param.", "upload", params.get(1));
+        Assert.assertEquals("unexpected param.", "url", params.get(2));
+        Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
+        Assert.assertEquals("unexpected param.", "account", params.get(4));
+        Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
+        Assert.assertEquals("unexpected param.", "username", params.get(6));
+        Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
+        Assert.assertEquals("unexpected param.", "key", params.get(8));
+        Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
+        Assert.assertEquals("unexpected param.", "container", params.get(10));
+        Assert.assertEquals("unexpected param.", container, params.get(11));
+        Assert.assertEquals("unexpected param.", "ldir", params.get(12));
+        Assert.assertEquals("unexpected param.", ldir, params.get(13));
+        Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
+        Assert.assertEquals("unexpected param.", lfilename, params.get(15));
+        Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
+        Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
+    }
+
+    @Test
+    public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() {
+        CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
+        XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
+
+        SwiftTO swift = Mockito.mock(SwiftTO.class);
+        when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy");
+
+        String container = "sample-container-name";
+        String ldir = "sample-ldir";
+        String lfilename = "sample-lfilename";
+        Boolean isISCSI = true;
+
+        List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
+        // make sure the params not null and has correct number of items in it
+        Assert.assertNotNull("params is null", params);
+        Assert.assertTrue("Expected param list size is 20 but it was" + params.size(), params.size() == 20);
+
+        // check the order of params
+        Assert.assertEquals("unexpected param.", "op", params.get(0));
+        Assert.assertEquals("unexpected param.", "upload", params.get(1));
+        Assert.assertEquals("unexpected param.", "url", params.get(2));
+        Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
+        Assert.assertEquals("unexpected param.", "account", params.get(4));
+        Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
+        Assert.assertEquals("unexpected param.", "username", params.get(6));
+        Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
+        Assert.assertEquals("unexpected param.", "key", params.get(8));
+        Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
+        Assert.assertEquals("unexpected param.", "container", params.get(10));
+        Assert.assertEquals("unexpected param.", container, params.get(11));
+        Assert.assertEquals("unexpected param.", "ldir", params.get(12));
+        Assert.assertEquals("unexpected param.", ldir, params.get(13));
+        Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
+        Assert.assertEquals("unexpected param.", lfilename, params.get(15));
+        Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
+        Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
+        Assert.assertEquals("unexpected param.", "storagepolicy", params.get(18));
+        Assert.assertEquals("unexpected param.", "sample-storagepolicy", params.get(19));
+    }
+}

plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java

@@ -68,7 +68,7 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
     public DataStoreTO getStoreTO(DataStore store) {
         ImageStoreImpl imgStore = (ImageStoreImpl)store;
         Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
-        return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
+        return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
     }
 
     @Override

scripts/storage/secondary/swift

@@ -1473,8 +1473,9 @@ post [options] [container] [object]
     Updates meta information for the account, container, or object depending on
     the args given. If the container is not found, it will be created
     automatically; but this is not true for accounts and objects. Containers
-    also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
-    or --meta option is allowed on all and used to define the user meta data
+    also allow the -r (or --read-acl) and -w (or --write-acl) options.
+    The --storage-policy will set a storage policy to the container if the container does not exist.
+    The -m or --meta option is allowed on all and used to define the user meta data
     items to set in the form Name:Value. This option can be repeated. Example:
     post -m Color:Blue -m Size:Large'''.strip('\n')
 
@@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue):
     parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
         help='Sets a meta data item with the syntax name:value. This option '
         'may be repeated. Example: -m Color:Blue -m Size:Large')
+    parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+         help='Sets a storage policy to the container if the container does not exist')
     (options, args) = parse_args(parser, args)
     args = args[1:]
     if (options.read_acl or options.write_acl or options.sync_to or
@@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue):
             headers['X-Container-Sync-To'] = options.sync_to
         if options.sync_key is not None:
             headers['X-Container-Sync-Key'] = options.sync_key
+        if options.storage_policy is not None:
+             headers['X-Storage-Policy'] = options.storage_policy
         try:
             conn.post_container(args[0], headers=headers)
         except ClientException, err:
@@ -1558,7 +1563,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
     Uploads to the given container the files and directories specified by the
     remaining args. -c or --changed is an option that will only upload files
     that have changed since the last upload. -S <size> or --segment-size <size>
-    and --leave-segments are options as well (see --help for more).
+    and --leave-segments are options as well (see --help for more). --storage-policy
+     Sets a storage policy to the container if the container does not exist.
 '''.strip('\n')
 
 
@@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue):
         dest='leave_segments', default=False, help='Indicates that you want '
         'the older segments of manifest objects left alone (in the case of '
         'overwrites)')
+    parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+         help='Sets a storage policy to the container if the container does not exist')
     (options, args) = parse_args(parser, args)
     args = args[1:]
     if len(args) < 2:
@@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue):
     # permissions, so we'll ignore any error. If there's really a problem,
     # it'll surface on the first object PUT.
     try:
-        conn.put_container(args[0])
+        container_headers = {}
+        if options.storage_policy is not None:
+            container_headers['X-Storage-Policy'] = options.storage_policy
+        conn.put_container(args[0],headers=container_headers)
         if options.segment_size is not None:
-            conn.put_container(args[0] + '_segments')
+            conn.put_container(args[0] + '_segments',headers=container_headers)
     except Exception:
         pass
     try: