From d8e798655eb61f84478bd21d96179b11943c4abb Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 22 Mar 2018 13:48:23 +0530 Subject: [PATCH 01/14] CLOUDSTACK-10333: Secure Live VM Migration for KVM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This extends securing of KVM hosts to securing of libvirt on KVM host as well for TLS enabled live VM migration. To simplify implementation securing of host implies that both host and libvirtd processes are secured with management server's CA plugin issued certificates. Based on whether keystore and certificates files are available at /etc/cloudstack/agent, the KVM agent determines whether to use TLS or TCP based uris for live VM migration. It is also enforced that a secured host will allow live VM migration to/from other secured host, and an unsecured hosts will allow live VM migration to/from other unsecured host only. Post upgrade the KVM agent on startup will expose its security state (secured detail is sent as true or false) to the managements server that gets saved in host_details for the host. This host detail can be accesed via the listHosts response, and in the UI unsecured KVM hosts will show up with the host state of ‘unsecured’. Further, a button has been added that allows admins to provision/renew certificates to KVM hosts and can be used to secure any unsecured KVM host. The `cloudstack-setup-agent` was modified to accept a new flag `-s` which will reconfigure libvirtd with following settings: listen_tcp=0 listen_tls=1 tcp_port="16509" tls_port="16514" auth_tcp="none" auth_tls="none" key_file = "/etc/pki/libvirt/private/serverkey.pem" cert_file = "/etc/pki/libvirt/servercert.pem" ca_file = "/etc/pki/CA/cacert.pem" For a connected KVM host agent, when the certificate are renewed/provisioned a background task is scheduled that waits until all of the agent tasks finish after which libvirt process is restarted and finally the agent is restarted via AgentShell. There are no API or DB changes. Signed-off-by: Rohit Yadav --- agent/bindir/cloud-setup-agent.in | 12 +- agent/src/com/cloud/agent/Agent.java | 148 +++++++++++++++--- agent/src/com/cloud/agent/AgentShell.java | 7 +- agent/src/com/cloud/agent/IAgentShell.java | 5 + .../VirtualRoutingResource.java | 18 +-- .../ca/PostCertificateRenewalCommand.java | 34 ++++ .../ca/SetupCertificateCommand.java | 6 +- debian/cloudstack-agent.postinst | 8 + packaging/centos63/cloud.spec | 8 + packaging/centos7/cloud.spec | 8 + .../ca/provider/RootCAProvider.java | 2 +- .../resource/LibvirtComputingResource.java | 27 +++- .../wrapper/LibvirtMigrateCommandWrapper.java | 21 ++- .../LibvirtModifyTargetsCommandWrapper.java | 2 +- ...tPostCertificateRenewalCommandWrapper.java | 52 ++++++ .../LibvirtMigrateCommandWrapperTest.java | 21 +++ python/lib/cloud_utils.py | 4 +- python/lib/cloudutils/serviceConfig.py | 52 +++--- scripts/util/keystore-cert-import | 12 ++ .../discoverer/LibvirtServerDiscoverer.java | 32 ++-- .../apache/cloudstack/ca/CAManagerImpl.java | 3 +- ui/css/cloudstack3.css | 2 + ui/l10n/en.js | 2 + ui/scripts/system.js | 51 +++++- .../main/java/com/cloud/utils/nio/Link.java | 8 +- .../java/com/cloud/utils/script/Script.java | 2 +- .../com/cloud/utils/ssh/SSHCmdHelper.java | 4 +- .../utils/security/KeyStoreUtils.java | 36 +++-- 28 files changed, 472 insertions(+), 115 deletions(-) create mode 100644 core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java create mode 100644 plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java diff --git a/agent/bindir/cloud-setup-agent.in b/agent/bindir/cloud-setup-agent.in index 8d2b91961aef..3c6203c2d340 100755 --- a/agent/bindir/cloud-setup-agent.in +++ b/agent/bindir/cloud-setup-agent.in @@ -26,6 +26,7 @@ from cloudutils.configFileOps import configFileOps from cloudutils.globalEnv import globalEnv from cloudutils.networkConfig import networkConfig from cloudutils.syscfg import sysConfigFactory +from cloudutils.serviceConfig import configureLibvirtConfig from optparse import OptionParser @@ -100,6 +101,7 @@ if __name__ == '__main__': parser.add_option("-c", "--cluster", dest="cluster", help="cluster id") parser.add_option("-t", "--hypervisor", default="kvm", dest="hypervisor", help="hypervisor type") parser.add_option("-g", "--guid", dest="guid", help="guid") + parser.add_option("-s", action="store_true", default=False, dest="secure", help="Secure and enable TLS for libvirtd") parser.add_option("--pubNic", dest="pubNic", help="Public traffic interface") parser.add_option("--prvNic", dest="prvNic", help="Private traffic interface") parser.add_option("--guestNic", dest="guestNic", help="Guest traffic interface") @@ -110,6 +112,12 @@ if __name__ == '__main__': glbEnv.bridgeType = bridgeType (options, args) = parser.parse_args() + + if not options.auto and options.secure: + configureLibvirtConfig(True) + print "Libvirtd with TLS configured" + sys.exit(0) + if options.auto is None: userInputs = getUserInputs() glbEnv.mgtSvr = userInputs[0] @@ -138,7 +146,9 @@ if __name__ == '__main__': glbEnv.nics.append(options.prvNic) glbEnv.nics.append(options.pubNic) glbEnv.nics.append(options.guestNic) - + + glbEnv.secure = options.secure + print "Starting to configure your system:" syscfg = sysConfigFactory.getSysConfigFactory(glbEnv) try: diff --git a/agent/src/com/cloud/agent/Agent.java b/agent/src/com/cloud/agent/Agent.java index 32112540c1c3..74b5205df08c 100644 --- a/agent/src/com/cloud/agent/Agent.java +++ b/agent/src/com/cloud/agent/Agent.java @@ -42,6 +42,7 @@ import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate; import org.apache.cloudstack.agent.lb.SetupMSListAnswer; import org.apache.cloudstack.agent.lb.SetupMSListCommand; +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; import org.apache.cloudstack.ca.SetupCertificateAnswer; import org.apache.cloudstack.ca.SetupCertificateCommand; import org.apache.cloudstack.ca.SetupKeyStoreCommand; @@ -68,6 +69,7 @@ import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.exception.AgentControlChannelException; +import com.cloud.host.Host; import com.cloud.resource.ServerResource; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.StringUtils; @@ -127,6 +129,7 @@ public int value() { Long _id; Timer _timer = new Timer("Agent Timer"); + Timer certTimer; Timer hostLBTimer; List _watchList = new ArrayList(); @@ -140,9 +143,11 @@ public int value() { long _startupWait = _startupWaitDefault; boolean _reconnectAllowed = true; //For time sentitive task, e.g. PingTask - private final ThreadPoolExecutor _ugentTaskPool; + ThreadPoolExecutor _ugentTaskPool; ExecutorService _executor; + Thread _shutdownThread = new ShutdownThread(this); + private String _keystoreSetupPath; private String _keystoreCertImportPath; @@ -153,7 +158,7 @@ public Agent(final IAgentShell shell) { _connection = new NioClient("Agent", _shell.getNextHost(), _shell.getPort(), _shell.getWorkers(), this); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -192,7 +197,7 @@ public Agent(final IAgentShell shell, final int localAgentId, final ServerResour // ((NioClient)_connection).setBindAddress(_shell.getPrivateIp()); s_logger.debug("Adding shutdown hook"); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -245,14 +250,14 @@ public void start() { throw new CloudRuntimeException("Unable to start the resource: " + _resource.getName()); } - _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreSetupScript); + _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_SETUP_SCRIPT); if (_keystoreSetupPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreSetupScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_SETUP_SCRIPT)); } - _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreImportScript); + _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_IMPORT_SCRIPT); if (_keystoreCertImportPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreImportScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_IMPORT_SCRIPT)); } try { @@ -274,6 +279,19 @@ public void start() { } } _shell.updateConnectedHost(); + + // In case of software based restart, GC to remove old instances + _executor.submit(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(2000L); + } catch (final InterruptedException ignored) { + } finally { + System.gc(); + } + } + }); } public void stop(final String reason, final String detail) { @@ -298,6 +316,7 @@ public void stop(final String reason, final String detail) { } _connection.stop(); _connection = null; + _link = null; } if (_resource != null) { @@ -305,7 +324,34 @@ public void stop(final String reason, final String detail) { _resource = null; } - _ugentTaskPool.shutdownNow(); + if (_startup != null) { + _startup = null; + } + + if (_ugentTaskPool != null) { + _ugentTaskPool.shutdownNow(); + _ugentTaskPool = null; + } + + if (_executor != null) { + _executor.shutdown(); + _executor = null; + } + + if (_timer != null) { + _timer.cancel(); + _timer = null; + } + + if (hostLBTimer != null) { + hostLBTimer.cancel(); + hostLBTimer = null; + } + + if (certTimer != null) { + certTimer.cancel(); + certTimer = null; + } } public Long getId() { @@ -318,6 +364,15 @@ public void setId(final Long id) { _shell.setPersistentProperty(getResourceName(), "id", Long.toString(id)); } + private synchronized void scheduleServicesRestartTask() { + if (certTimer != null) { + certTimer.cancel(); + certTimer.purge(); + } + certTimer = new Timer("Certificate Renewal Timer"); + certTimer.schedule(new PostCertificateRenewalTask(this), 5000L); + } + private synchronized void scheduleHostLBCheckerTask(final long checkInterval) { if (hostLBTimer != null) { hostLBTimer.cancel(); @@ -578,6 +633,9 @@ protected void processRequest(final Request request, final Link link) { answer = setupAgentKeystore((SetupKeyStoreCommand) cmd); } else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) { answer = setupAgentCertificate((SetupCertificateCommand) cmd); + if (Host.Type.Routing.equals(_resource.getType())) { + scheduleServicesRestartTask(); + } } else if (cmd instanceof SetupDirectDownloadCertificate) { answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd); } else if (cmd instanceof SetupMSListCommand) { @@ -641,7 +699,7 @@ private Answer setupDirectDownloadCertificate(SetupDirectDownloadCertificate cmd return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; String cerFile = agentFile.getParent() + "/" + certificateName + ".cer"; Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile)); @@ -666,13 +724,13 @@ public Answer setupAgentKeystore(final SetupKeyStoreCommand cmd) { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; - final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCsrFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.CSR_FILENAME; - String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.passphrasePropertyName); + String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (Strings.isNullOrEmpty(storedPassword)) { storedPassword = keyStorePassword; - _shell.setPersistentProperty(null, KeyStoreUtils.passphrasePropertyName, storedPassword); + _shell.setPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY, storedPassword); } Script script = new Script(true, _keystoreSetupPath, 60000, s_logger); @@ -706,10 +764,10 @@ private Answer setupAgentCertificate(final SetupCertificateCommand cmd) { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; - final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCertFile; - final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultPrivateKeyFile; - final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCaCertFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME; + final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.PKEY_FILENAME; + final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.CACERT_FILENAME; try { FileUtils.writeStringToFile(new File(certFile), certificate, Charset.defaultCharset()); @@ -722,7 +780,7 @@ private Answer setupAgentCertificate(final SetupCertificateCommand cmd) { Script script = new Script(true, _keystoreCertImportPath, 60000, s_logger); script.add(agentFile.getAbsolutePath()); script.add(keyStoreFile); - script.add(KeyStoreUtils.agentMode); + script.add(KeyStoreUtils.AGENT_MODE); script.add(certFile); script.add(""); script.add(caCertFile); @@ -1072,6 +1130,60 @@ public void doTask(final Task task) throws TaskExecutionException { } } + /** + * Task stops the current agent and launches a new agent + * when there are no outstanding jobs in the agent's task queue + */ + public class PostCertificateRenewalTask extends ManagedContextTimerTask { + + private Agent agent; + + public PostCertificateRenewalTask(final Agent agent) { + this.agent = agent; + } + + @Override + protected void runInContext() { + while (true) { + try { + if (_inProgress.get() == 0) { + s_logger.debug("Running post certificate renewal task to restart services."); + + // Let the resource perform any post certificate renewal cleanups + _resource.executeRequest(new PostCertificateRenewalCommand()); + + IAgentShell shell = agent._shell; + ServerResource resource = agent._resource.getClass().newInstance(); + + // Stop current agent + agent.cancelTasks(); + agent._reconnectAllowed = false; + Runtime.getRuntime().removeShutdownHook(agent._shutdownThread); + agent.stop(ShutdownCommand.Requested, "Restarting due to new X509 certificates"); + + // Nullify references for GC + agent._shell = null; + agent._watchList = null; + agent._shutdownThread = null; + agent._controlListeners = null; + agent = null; + + // Start a new agent instance + shell.launchNewAgent(resource); + return; + } + if (s_logger.isTraceEnabled()) { + s_logger.debug("Other tasks are in progress, will retry post certificate renewal command after few seconds"); + } + Thread.sleep(5000); + } catch (final Exception e) { + s_logger.warn("Failed to execute post certificate renewal command:", e); + break; + } + } + } + } + public class PreferredHostCheckerTask extends ManagedContextTimerTask { @Override diff --git a/agent/src/com/cloud/agent/AgentShell.java b/agent/src/com/cloud/agent/AgentShell.java index 13b6c65a3519..01654ac9caa9 100644 --- a/agent/src/com/cloud/agent/AgentShell.java +++ b/agent/src/com/cloud/agent/AgentShell.java @@ -419,7 +419,7 @@ private void launchAgentFromClassInfo(String resourceClassNames) throws Configur final Constructor constructor = impl.getDeclaredConstructor(); constructor.setAccessible(true); ServerResource resource = (ServerResource)constructor.newInstance(); - launchAgent(getNextAgentId(), resource); + launchNewAgent(resource); } catch (final ClassNotFoundException e) { throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString()); } catch (final SecurityException e) { @@ -447,9 +447,10 @@ private void launchAgentFromTypeInfo() throws ConfigurationException { s_logger.trace("Launching agent based on type=" + typeInfo); } - private void launchAgent(int localAgentId, ServerResource resource) throws ConfigurationException { + public void launchNewAgent(ServerResource resource) throws ConfigurationException { // we don't track agent after it is launched for now - Agent agent = new Agent(this, localAgentId, resource); + _agents.clear(); + Agent agent = new Agent(this, getNextAgentId(), resource); _agents.add(agent); agent.start(); } diff --git a/agent/src/com/cloud/agent/IAgentShell.java b/agent/src/com/cloud/agent/IAgentShell.java index 5b52cee63615..5d389a07041b 100644 --- a/agent/src/com/cloud/agent/IAgentShell.java +++ b/agent/src/com/cloud/agent/IAgentShell.java @@ -19,6 +19,9 @@ import java.util.Map; import java.util.Properties; +import javax.naming.ConfigurationException; + +import com.cloud.resource.ServerResource; import com.cloud.utils.backoff.BackoffAlgorithm; public interface IAgentShell { @@ -66,4 +69,6 @@ public interface IAgentShell { void updateConnectedHost(); String getConnectedHost(); + + void launchNewAgent(ServerResource resource) throws ConfigurationException; } diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index d7c3d56f9b9b..0ffe8cc0ea2f 100644 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -157,11 +157,11 @@ private Answer execute(final SetupKeyStoreCommand cmd) { "/usr/local/cloud/systemvm/conf/%s " + "%s %d " + "/usr/local/cloud/systemvm/conf/%s", - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_FILENAME, cmd.getKeystorePassword(), cmd.getValidityDays(), - KeyStoreUtils.defaultCsrFile); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreSetupScript, args); + KeyStoreUtils.CSR_FILENAME); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_SETUP_SCRIPT, args); return new SetupKeystoreAnswer(result.getDetails()); } @@ -171,15 +171,15 @@ private Answer execute(final SetupCertificateCommand cmd) { "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\"", - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, cmd.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, cmd.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, cmd.getEncodedPrivateKey()); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreImportScript, args); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_IMPORT_SCRIPT, args); return new SetupCertificateAnswer(result.isSuccess()); } diff --git a/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java b/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java new file mode 100644 index 000000000000..12df6196128d --- /dev/null +++ b/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.ca; + +import com.cloud.agent.api.Command; + +public class PostCertificateRenewalCommand extends Command { + + public PostCertificateRenewalCommand() { + super(); + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java b/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java index 1cd31509d392..7727282bcee4 100644 --- a/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java +++ b/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java @@ -82,15 +82,15 @@ public String getCaCertificates() { } public String getEncodedPrivateKey() { - return privateKey.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return privateKey.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCertificate() { - return certificate.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return certificate.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCaCertificates() { - return caCertificates.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return caCertificates.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public boolean isHandleByAgent() { diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index a9b8b687fab1..15ccdfa5eafc 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -50,6 +50,14 @@ case "$1" in mkdir /etc/libvirt/hooks fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu + + # Enable libvirt TLS if host is secured + if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then + /usr/bin/cloudstack-setup-agent -s + /sbin/service libvirt-bin restart + /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + /sbin/iptables-save > /etc/iptables/rules.v4 + fi ;; esac diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 898118fa96df..221cd86bd4be 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -493,6 +493,14 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi +# Enable libvirt TLS if host is secured +if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then + /usr/bin/cloudstack-setup-agent -s + /sbin/service libvirtd restart + /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + /sbin/service iptables save +fi + %preun usage /sbin/service cloudstack-usage stop || true if [ "$1" == "0" ] ; then diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index f16858a4a8f1..07ef3c44b407 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -437,6 +437,14 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi +# Enable libvirt TLS if host is secured +if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then + /usr/bin/cloudstack-setup-agent -s + /sbin/service libvirtd restart + /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + /sbin/service iptables save +fi + %pre usage id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -c "CloudStack unprivileged user" \ -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true diff --git a/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java index 4a3585ac4e26..6584b35861a0 100644 --- a/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -241,7 +241,7 @@ public KeyStore getManagementKeyStore() throws KeyStoreException { @Override public char[] getKeyStorePassphrase() { - return KeyStoreUtils.defaultKeystorePassphrase; + return KeyStoreUtils.DEFAULT_KS_PASSPHRASE; } ///////////////////////////////////////////////// diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index dd039e54263b..d7c1b497b4d4 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -41,11 +41,19 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; + import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; +import org.apache.cloudstack.utils.linux.CPUStat; +import org.apache.cloudstack.utils.linux.MemStat; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; @@ -68,14 +76,6 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import com.google.common.base.Strings; - -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; -import org.apache.cloudstack.utils.linux.CPUStat; -import org.apache.cloudstack.utils.linux.MemStat; -import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -168,6 +168,7 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VmDetailConstants; +import com.google.common.base.Strings; /** * LibvirtComputingResource execute requests on the computing/routing host using @@ -239,6 +240,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected long _hypervisorLibvirtVersion; protected long _hypervisorQemuVersion; protected String _hypervisorPath; + protected String _hostDistro; protected String _networkDirectSourceMode; protected String _networkDirectDevice; protected String _sysvmISOPath; @@ -2599,11 +2601,16 @@ public StartupCommand[] initialize() { fillNetworkInformation(cmd); _privateIp = cmd.getPrivateIpAddress(); cmd.getHostDetails().putAll(getVersionStrings()); + cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(KeyStoreUtils.isHostSecured()).toLowerCase()); cmd.setPool(_pool); cmd.setCluster(_clusterId); cmd.setGatewayIpAddress(_localGateway); cmd.setIqn(getIqn()); + if (cmd.getHostDetails().containsKey("Host.OS")) { + _hostDistro = cmd.getHostDetails().get("Host.OS"); + } + StartupStorageCommand sscmd = null; try { @@ -3777,4 +3784,8 @@ public void restoreVMSnapshotMetadata(Domain dm, String vmName, List { @@ -80,9 +80,17 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage, command.isAutoConvergence(), vmName, command.getDestinationIp()); @@ -203,6 +211,9 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. } catch (final LibvirtException e) { s_logger.debug("Can't migrate domain: " + e.getMessage()); result = e.getMessage(); + if (result.startsWith("unable to connect to server") && result.endsWith("refused")) { + result = String.format("Migration was refused connection to destination: %s. Please check libvirt configuration compatibility on the source and destination hosts.", destinationUri); + } } catch (final InterruptedException e) { s_logger.debug("Interrupted while migrating domain: " + e.getMessage()); result = e.getMessage(); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java index 627d4b7beb6d..724caad3f227 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java @@ -38,7 +38,7 @@ @ResourceWrapper(handles = ModifyTargetsCommand.class) public final class LibvirtModifyTargetsCommandWrapper extends CommandWrapper { - private static final Logger s_logger = Logger.getLogger(LibvirtMigrateCommandWrapper.class); + private static final Logger s_logger = Logger.getLogger(LibvirtModifyTargetsCommandWrapper.class); @Override public Answer execute(final ModifyTargetsCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java new file mode 100644 index 000000000000..df89d2470dda --- /dev/null +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; +import org.apache.cloudstack.ca.SetupCertificateAnswer; +import org.apache.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = PostCertificateRenewalCommand.class) +public final class LibvirtPostCertificateRenewalCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtPostCertificateRenewalCommandWrapper.class); + + @Override + public Answer execute(final PostCertificateRenewalCommand command, final LibvirtComputingResource serverResource) { + s_logger.info("Restarting libvirt after certificate provisioning/renewal"); + if (command != null) { + final int timeout = 30000; + Script script = new Script(true, "service", timeout, s_logger); + if ("Ubuntu".equals(serverResource.getHostDistro()) || "Debian".equals(serverResource.getHostDistro())) { + script.add("libvirt-bin"); + } else { + script.add("libvirtd"); + } + script.add("restart"); + script.execute(); + return new SetupCertificateAnswer(true); + } + return new SetupCertificateAnswer(false); + } +} diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index ed13cb25f0c8..19848a1f9406 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -18,10 +18,14 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.junit.Test; +import com.cloud.utils.exception.CloudRuntimeException; + public class LibvirtMigrateCommandWrapperTest { String fullfile = "\n" + @@ -303,4 +307,21 @@ public void testReplaceFqdnForVNCInDesc() { final String result = lw.replaceIpForVNCInDescFile(xmlDesc, targetIp); assertTrue("transformation does not live up to expectation:\n" + result, expectedXmlDesc.equals(result)); } + + @Test + public void testMigrationUri() { + final String ip = "10.1.1.1"; + LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); + if (KeyStoreUtils.isHostSecured()) { + assertEquals(lw.createMigrationURI(ip), String.format("qemu+tls://%s/system", ip)); + } else { + assertEquals(lw.createMigrationURI(ip), String.format("qemu+tcp://%s/system", ip)); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testMigrationUriException() { + LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); + lw.createMigrationURI(null); + } } diff --git a/python/lib/cloud_utils.py b/python/lib/cloud_utils.py index 43c93c876316..3c69c5652b14 100644 --- a/python/lib/cloud_utils.py +++ b/python/lib/cloud_utils.py @@ -802,9 +802,9 @@ def done(self): rule = "-p tcp -m tcp --dport 16509 -j ACCEPT" if rule in iptablessave().stdout: return True return False - + def execute(self): - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() if distro in (Fedora , CentOS, RHEL6): for p in ports: iptables("-I","INPUT","1","-p","tcp","--dport",p,'-j','ACCEPT') o = service.iptables.save() ; print o.stdout + o.stderr diff --git a/python/lib/cloudutils/serviceConfig.py b/python/lib/cloudutils/serviceConfig.py index c9c019c5abe5..3b9f6127e6bd 100755 --- a/python/lib/cloudutils/serviceConfig.py +++ b/python/lib/cloudutils/serviceConfig.py @@ -471,6 +471,23 @@ def restore(self): logging.debug(formatExceptionInfo()) return False +def configureLibvirtConfig(tls_enabled = True, cfg = None): + cfo = configFileOps("/etc/libvirt/libvirtd.conf", cfg) + if tls_enabled: + cfo.addEntry("listen_tcp", "0") + cfo.addEntry("listen_tls", "1") + cfo.addEntry("key_file", "\"/etc/pki/libvirt/private/serverkey.pem\"") + cfo.addEntry("cert_file", "\"/etc/pki/libvirt/servercert.pem\"") + cfo.addEntry("ca_file", "\"/etc/pki/CA/cacert.pem\"") + else: + cfo.addEntry("listen_tcp", "1") + cfo.addEntry("listen_tls", "0") + cfo.addEntry("tcp_port", "\"16509\"") + cfo.addEntry("tls_port", "\"16514\"") + cfo.addEntry("auth_tcp", "\"none\"") + cfo.addEntry("auth_tls", "\"none\"") + cfo.save() + class libvirtConfigRedhat(serviceCfgBase): def __init__(self, syscfg): super(libvirtConfigRedhat, self).__init__(syscfg) @@ -478,12 +495,7 @@ def __init__(self, syscfg): def config(self): try: - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\"") - cfo.addEntry("auth_tcp", "\"none\"") - cfo.addEntry("listen_tls", "0") - cfo.save() + configureLibvirtConfig(self.syscfg.env.secure, self) cfo = configFileOps("/etc/sysconfig/libvirtd", self) cfo.addEntry("export CGROUP_DAEMON", "'cpu:/virt'") @@ -515,24 +527,16 @@ def __init__(self, syscfg): super(libvirtConfigUbuntu, self).__init__(syscfg) self.serviceName = "Libvirt" - def setupLiveMigration(self): - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\""); - cfo.addEntry("auth_tcp", "\"none\""); - cfo.addEntry("listen_tls", "0") - cfo.save() - - if os.path.exists("/etc/init/libvirt-bin.conf"): - cfo = configFileOps("/etc/init/libvirt-bin.conf", self) - cfo.replace_line("exec /usr/sbin/libvirtd","exec /usr/sbin/libvirtd -d -l") - else: - cfo = configFileOps("/etc/default/libvirt-bin", self) - cfo.replace_or_add_line("libvirtd_opts=","libvirtd_opts='-l -d'") - def config(self): try: - self.setupLiveMigration() + configureLibvirtConfig(self.syscfg.env.secure, self) + if os.path.exists("/etc/init/libvirt-bin.conf"): + cfo = configFileOps("/etc/init/libvirt-bin.conf", self) + cfo.replace_line("exec /usr/sbin/libvirtd","exec /usr/sbin/libvirtd -d -l") + else: + cfo = configFileOps("/etc/default/libvirt-bin", self) + cfo.replace_or_add_line("libvirtd_opts=","libvirtd_opts='-l -d'") + filename = "/etc/libvirt/qemu.conf" @@ -564,7 +568,7 @@ def __init__(self, syscfg): def config(self): try: - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() for p in ports: bash("ufw allow %s"%p) bash("ufw allow proto tcp from any to any port 5900:6100") @@ -624,7 +628,7 @@ def restore(self): class firewallConfigAgent(firewallConfigBase): def __init__(self, syscfg): super(firewallConfigAgent, self).__init__(syscfg) - self.ports = "22 16509 5900:6100 49152:49216".split() + self.ports = "22 16509 16514 5900:6100 49152:49216".split() if syscfg.env.distribution.getVersion() == "CentOS": self.rules = ["-D FORWARD -j RH-Firewall-1-INPUT"] else: diff --git a/scripts/util/keystore-cert-import b/scripts/util/keystore-cert-import index 67ce3400345d..eeaae7c83550 100755 --- a/scripts/util/keystore-cert-import +++ b/scripts/util/keystore-cert-import @@ -28,6 +28,7 @@ PRIVKEY=$(echo "$9" | tr '^' '\n' | tr '~' ' ') ALIAS="cloud" SYSTEM_FILE="/var/cache/cloud/cmdline" +LIBVIRTD_FILE="/etc/libvirt/libvirtd.conf" # Find keystore password KS_PASS=$(sed -n '/keystore.passphrase/p' "$PROPS_FILE" 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null) @@ -78,6 +79,17 @@ fi rm -f "$NEW_KS_FILE.p12" mv -f "$NEW_KS_FILE" "$KS_FILE" +# Secure libvirtd on cert import +if [ -f "$LIBVIRTD_FILE" ]; then + mkdir -p /etc/pki/libvirt/private + ln -sf /etc/cloudstack/agent/cloud.ca.crt /etc/pki/CA/cacert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/clientcert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/servercert.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/clientkey.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/serverkey.pem + cloudstack-setup-agent -s > /dev/null +fi + # Update ca-certs if we're in systemvm if [ -f "$SYSTEM_FILE" ]; then mkdir -p /usr/local/share/ca-certificates/cloudstack diff --git a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index ef69fdcb4366..0b8b40b1b9b0 100644 --- a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -18,6 +18,7 @@ import java.net.InetAddress; import java.net.URI; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -141,11 +142,6 @@ public boolean processTimeout(long agentId, long seq) { } private void setupAgentSecurity(final Connection sshConnection, final String agentIp, final String agentHostname) { - if (!caManager.canProvisionCertificates()) { - s_logger.warn("Cannot secure agent communication because configure CA plugin cannot provision client certificate"); - return; - } - if (sshConnection == null) { throw new CloudRuntimeException("Cannot secure agent communication because ssh connection is invalid for host ip=" + agentIp); } @@ -161,17 +157,17 @@ private void setupAgentSecurity(final Connection sshConnection, final String age "/etc/cloudstack/agent/%s " + "%s %d " + "/etc/cloudstack/agent/%s", - KeyStoreUtils.keyStoreSetupScript, - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_SETUP_SCRIPT, + KeyStoreUtils.KS_FILENAME, PasswordGenerator.generateRandomPassword(16), validityPeriod, - KeyStoreUtils.defaultCsrFile)); + KeyStoreUtils.CSR_FILENAME)); if (!keystoreSetupResult.isSuccess()) { throw new CloudRuntimeException("Failed to setup keystore on the KVM host: " + agentIp); } - final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Collections.singletonList(agentHostname), Collections.singletonList(agentIp), null, null); + final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Arrays.asList(agentHostname, agentIp), Collections.singletonList(agentIp), null, null); if (certificate == null || certificate.getClientCertificate() == null) { throw new CloudRuntimeException("Failed to issue certificates for KVM host agent: " + agentIp); } @@ -184,14 +180,14 @@ private void setupAgentSecurity(final Connection sshConnection, final String age "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\"", - KeyStoreUtils.keyStoreImportScript, - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_IMPORT_SCRIPT, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, certificateCommand.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, certificateCommand.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, certificateCommand.getEncodedPrivateKey())); if (setupCertResult != null && !setupCertResult.isSuccess()) { @@ -288,9 +284,13 @@ private void setupAgentSecurity(final Connection sshConnection, final String age kvmGuestNic = (kvmPublicNic != null) ? kvmPublicNic : kvmPrivateNic; } + if (!caManager.canProvisionCertificates()) { + throw new CloudRuntimeException("Configured CA plugin cannot provision X509 certificate(s), failing to add host due to security insufficiency."); + } + setupAgentSecurity(sshConnection, agentIp, hostname); - String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a"; + String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a -s "; parameters += " --pubNic=" + kvmPublicNic; parameters += " --prvNic=" + kvmPrivateNic; diff --git a/server/src/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/org/apache/cloudstack/ca/CAManagerImpl.java index 3a904315fe16..23a0379305b8 100644 --- a/server/src/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/org/apache/cloudstack/ca/CAManagerImpl.java @@ -27,7 +27,6 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -191,7 +190,7 @@ public boolean provisionCertificate(final Host host, final Boolean reconnect, fi if (Strings.isNullOrEmpty(csr)) { return false; } - final Certificate certificate = issueCertificate(csr, Collections.singletonList(host.getName()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); + final Certificate certificate = issueCertificate(csr, Arrays.asList(host.getName(), host.getPrivateIpAddress()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); return deployCertificate(host, certificate, reconnect, null); } catch (final AgentUnavailableException | OperationTimedoutException e) { LOG.error("Host/agent is not available or operation timed out, failed to setup keystore and generate CSR for host/agent id=" + host.getId() + ", due to: ", e); diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index 519778ba59ba..bab92d0c7d03 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -12612,11 +12612,13 @@ div.ui-dialog div.autoscaler div.field-group div.form-container form div.form-it background-position: -101px -647px; } +.secureKVMHost .icon, .resetPassword .icon, .changePassword .icon { background-position: -68px -30px; } +.secureKVMHost:hover .icon, .resetPassword:hover .icon, .changePassword:hover .icon { background-position: -68px -612px; diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 7b19946b1ed6..745deb11ca24 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -270,6 +270,7 @@ var dictionary = {"ICMP.code":"ICMP Code", "label.action.restore.instance.processing":"Restoring Instance....", "label.action.revert.snapshot":"Revert to Snapshot", "label.action.revert.snapshot.processing":"Reverting to Snapshot...", +"label.action.secure.host":"Provision Host Security Keys", "label.action.start.instance":"Start Instance", "label.action.start.instance.processing":"Starting Instance....", "label.action.start.router":"Start Router", @@ -1910,6 +1911,7 @@ var dictionary = {"ICMP.code":"ICMP Code", "message.action.reset.password.warning":"Your instance must be stopped before attempting to change its current password.", "message.action.restore.instance":"Please confirm that you want to restore this instance.", "message.action.revert.snapshot":"Please confirm that you want to revert the owning volume to this snapshot.", +"message.action.secure.host":"This will restart the host agent and libvirtd process after applying new X509 certificates, please confirm?", "message.action.start.instance":"Please confirm that you want to start this instance.", "message.action.start.router":"Please confirm that you want to start this router.", "message.action.start.systemvm":"Please confirm that you want to start this system VM.", diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 07520df1d463..63f549ad2925 100755 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -9200,6 +9200,11 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } + }); } @@ -15713,7 +15718,8 @@ 'Down': 'off', 'Disconnected': 'off', 'Alert': 'off', - 'Error': 'off' + 'Error': 'off', + 'Unsecure': 'warning' } }, powerstate: { @@ -15761,6 +15767,10 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } }); } @@ -16530,6 +16540,40 @@ } }, + secureKVMHost: { + label: 'label.action.secure.host', + action: function(args) { + var data = { + hostid: args.context.hosts[0].id + }; + $.ajax({ + url: createURL('provisionCertificate'), + data: data, + async: true, + success: function(json) { + args.response.success({ + _custom: { + jobId: json.provisioncertificateresponse.jobid, + getActionFilter: function () { + return hostActionfilter; + } + } + }); + } + }); + }, + messages: { + confirm: function (args) { + return 'message.action.secure.host'; + }, + notification: function (args) { + return 'label.action.secure.host'; + } + }, + notification: { + poll: pollAsyncJobResult + } + }, enableMaintenanceMode: { label: 'label.action.enable.maintenance.mode', @@ -21968,6 +22012,11 @@ if (jsonObj.state != "Disconnected") allowedActions.push("forceReconnect"); + + if (jsonObj.hypervisor == "KVM") { + allowedActions.push("secureKVMHost"); + } + } else if (jsonObj.resourcestate == "ErrorInMaintenance") { allowedActions.push("edit"); allowedActions.push("enableMaintenanceMode"); diff --git a/utils/src/main/java/com/cloud/utils/nio/Link.java b/utils/src/main/java/com/cloud/utils/nio/Link.java index 25f6662c5225..658244087709 100644 --- a/utils/src/main/java/com/cloud/utils/nio/Link.java +++ b/utils/src/main/java/com/cloud/utils/nio/Link.java @@ -379,7 +379,7 @@ public static SSLEngine initServerSSLEngine(final CAService caService, final Str return caService.createSSLEngine(sslContext, clientAddress); } s_logger.error("CA service is not configured, by-passing CA manager to create SSL engine"); - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; final KeyStore ks = loadKeyStore(NioConnection.class.getResourceAsStream("/cloud.keystore"), passphrase); final KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509"); final TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509"); @@ -409,11 +409,11 @@ public static SSLContext initManagementSSLContext(final CAService caService) thr } public static SSLContext initClientSSLContext() throws GeneralSecurityException, IOException { - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; File confFile = PropertiesUtil.findConfigFile("agent.properties"); if (confFile != null) { s_logger.info("Conf file found: " + confFile.getAbsolutePath()); - final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.passphrasePropertyName); + final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (pass != null) { passphrase = pass.toCharArray(); } @@ -421,7 +421,7 @@ public static SSLContext initClientSSLContext() throws GeneralSecurityException, InputStream stream = null; if (confFile != null) { - final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; + final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; if (new File(keystorePath).exists()) { stream = new FileInputStream(keystorePath); } diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index 01f18bda2d2e..b73371ac7154 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -203,7 +203,7 @@ public String execute(OutputInterpreter interpreter) { String[] command = _command.toArray(new String[_command.size()]); if (_logger.isDebugEnabled()) { - _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.defaultKeystoreFile)[0]); + _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0]); } try { diff --git a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java index 10407b65642e..5324cdcbc18f 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java @@ -139,7 +139,7 @@ public static SSHCmdResult sshExecuteCmdWithResult(com.trilead.ssh2.Connection s } public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshConnection, String cmd) throws SshException { - s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0]); + s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0]); Session sshSession = null; try { sshSession = sshConnection.openSession(); @@ -202,7 +202,7 @@ public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshC final SSHCmdResult result = new SSHCmdResult(-1, sbStdoutResult.toString(), sbStdErrResult.toString()); if (!Strings.isNullOrEmpty(result.getStdOut()) || !Strings.isNullOrEmpty(result.getStdErr())) { - s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); + s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); } // exit status delivery might get delayed diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java b/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java index 8259d77d4ff0..12bf89ea41af 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java @@ -19,23 +19,31 @@ package org.apache.cloudstack.utils.security; +import java.io.File; + +import com.cloud.utils.PropertiesUtil; + public class KeyStoreUtils { + public static final String KS_SETUP_SCRIPT = "keystore-setup"; + public static final String KS_IMPORT_SCRIPT = "keystore-cert-import"; + public static final String KS_PASSPHRASE_PROPERTY = "keystore.passphrase"; - public static String defaultTmpKeyStoreFile = "/tmp/tmp.jks"; - public static String defaultKeystoreFile = "cloud.jks"; - public static String defaultPrivateKeyFile = "cloud.key"; - public static String defaultCsrFile = "cloud.csr"; - public static String defaultCertFile = "cloud.crt"; - public static String defaultCaCertFile = "cloud.ca.crt"; - public static char[] defaultKeystorePassphrase = "vmops.com".toCharArray(); + public static final String PKEY_FILENAME = "cloud.key"; + public static final String CSR_FILENAME = "cloud.csr"; + public static final String CERT_FILENAME = "cloud.crt"; + public static final String CACERT_FILENAME = "cloud.ca.crt"; + public static final String KS_FILENAME = "cloud.jks"; + public static final char[] DEFAULT_KS_PASSPHRASE = "vmops.com".toCharArray(); - public static String certNewlineEncoder = "^"; - public static String certSpaceEncoder = "~"; + public static final String CERT_NEWLINE_ENCODER = "^"; + public static final String CERT_SPACE_ENCODER = "~"; - public static String keyStoreSetupScript = "keystore-setup"; - public static String keyStoreImportScript = "keystore-cert-import"; - public static String passphrasePropertyName = "keystore.passphrase"; + public static final String SSH_MODE = "ssh"; + public static final String AGENT_MODE = "agent"; + public static final String SECURED = "secured"; - public static String sshMode = "ssh"; - public static String agentMode = "agent"; + public static boolean isHostSecured() { + final File confFile = PropertiesUtil.findConfigFile("agent.properties"); + return confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + CERT_FILENAME).exists(); + } } From 9e6bd8aedb7e3a555bcb64e9220b9facb5aef879 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 23 Mar 2018 15:45:47 +0530 Subject: [PATCH 02/14] Don't force libvirt reconfig during upgrade Signed-off-by: Rohit Yadav --- debian/cloudstack-agent.postinst | 11 +++++------ packaging/centos63/cloud.spec | 10 ++++------ packaging/centos7/cloud.spec | 10 ++++------ .../kvm/resource/LibvirtComputingResource.java | 16 +++++++++++++++- .../wrapper/LibvirtMigrateCommandWrapper.java | 6 +++--- .../LibvirtMigrateCommandWrapperTest.java | 4 ++-- .../cloudstack/utils/security/KeyStoreUtils.java | 11 +++++++---- 7 files changed, 40 insertions(+), 28 deletions(-) diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index 15ccdfa5eafc..673e7f9d6007 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -51,13 +51,12 @@ case "$1" in fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu - # Enable libvirt TLS if host is secured - if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then - /usr/bin/cloudstack-setup-agent -s - /sbin/service libvirt-bin restart - /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - /sbin/iptables-save > /etc/iptables/rules.v4 + # Enable TLS enabled VM migration for libvirtd + if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then + iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + iptables-save > /etc/iptables/rules.v4 fi + ;; esac diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 221cd86bd4be..d25c28a6d693 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -493,12 +493,10 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi -# Enable libvirt TLS if host is secured -if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then - /usr/bin/cloudstack-setup-agent -s - /sbin/service libvirtd restart - /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - /sbin/service iptables save +# Enable TLS enabled VM migration for libvirtd +if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then + iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + iptables-save > /etc/iptables/rules.v4 fi %preun usage diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index 07ef3c44b407..b499866f1363 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -437,12 +437,10 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi -# Enable libvirt TLS if host is secured -if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then - /usr/bin/cloudstack-setup-agent -s - /sbin/service libvirtd restart - /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - /sbin/service iptables save +# Enable TLS enabled VM migration for libvirtd +if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then + iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + iptables-save > /etc/iptables/rules.v4 fi %pre usage diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index d7c1b497b4d4..a32b60d692c1 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2601,7 +2601,7 @@ public StartupCommand[] initialize() { fillNetworkInformation(cmd); _privateIp = cmd.getPrivateIpAddress(); cmd.getHostDetails().putAll(getVersionStrings()); - cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(KeyStoreUtils.isHostSecured()).toLowerCase()); + cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(isHostSecured()).toLowerCase()); cmd.setPool(_pool); cmd.setCluster(_clusterId); cmd.setGatewayIpAddress(_localGateway); @@ -3788,4 +3788,18 @@ public long getTotalMemory() { public String getHostDistro() { return _hostDistro; } + + public boolean isHostSecured() { + // Test for host certificates + final File confFile = PropertiesUtil.findConfigFile(KeyStoreUtils.AGENT_PROPSFILE); + boolean certExists = confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists(); + // Test for libvirt TLS configuration + boolean libvirtTlsEnabled = true; + try { + new Connect(String.format("qemu+tls://%s/system", _privateIp)); + } catch (final LibvirtException ignored) { + libvirtTlsEnabled = false; + } + return certExists && libvirtTlsEnabled; + } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 79db62368f28..005c937c9e8f 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -80,17 +80,17 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper Date: Fri, 23 Mar 2018 16:09:20 +0530 Subject: [PATCH 03/14] Fix build issues Signed-off-by: Rohit Yadav --- .../kvm/resource/LibvirtComputingResource.java | 10 ++++++---- .../wrapper/LibvirtMigrateCommandWrapperTest.java | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index a32b60d692c1..fc5e5395b87f 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3792,14 +3792,16 @@ public String getHostDistro() { public boolean isHostSecured() { // Test for host certificates final File confFile = PropertiesUtil.findConfigFile(KeyStoreUtils.AGENT_PROPSFILE); - boolean certExists = confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists(); + if (confFile == null || !confFile.exists() || !new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists()) { + return false; + } + // Test for libvirt TLS configuration - boolean libvirtTlsEnabled = true; try { new Connect(String.format("qemu+tls://%s/system", _privateIp)); } catch (final LibvirtException ignored) { - libvirtTlsEnabled = false; + return false; } - return certExists && libvirtTlsEnabled; + return true; } } diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index 7cfa0eb74eb7..da71e40c30f0 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -312,16 +312,17 @@ public void testReplaceFqdnForVNCInDesc() { public void testMigrationUri() { final String ip = "10.1.1.1"; LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); - if (new LibvirtComputingResource().isHostSecured()) { - assertEquals(lw.createMigrationURI(ip), String.format("qemu+tls://%s/system", ip)); + LibvirtComputingResource lcr = new LibvirtComputingResource(); + if (lcr.isHostSecured()) { + assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tls://%s/system", ip)); } else { - assertEquals(lw.createMigrationURI(ip), String.format("qemu+tcp://%s/system", ip)); + assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tcp://%s/system", ip)); } } @Test(expected = CloudRuntimeException.class) public void testMigrationUriException() { LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); - lw.createMigrationURI(null); + lw.createMigrationURI(null, new LibvirtComputingResource()); } } From 88be41eb74a63096cae986a189d8234823cd3188 Mon Sep 17 00:00:00 2001 From: Boris Date: Mon, 26 Mar 2018 13:44:18 +0300 Subject: [PATCH 04/14] Add marvin tests to cover secured and non-secured hosts vm migration --- test/integration/smoke/test_vm_life_cycle.py | 321 ++++++++++++++++++- 1 file changed, 317 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 3504849786ba..3ad02ed7b461 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -21,9 +21,11 @@ from marvin.cloudstackAPI import (recoverVirtualMachine, destroyVirtualMachine, attachIso, - detachIso) -from marvin.lib.utils import (cleanup_resources, - validateList) + detachIso, + provisionCertificate, + updateConfiguration) +from marvin.lib.utils import * + from marvin.lib.base import (Account, ServiceOffering, VirtualMachine, @@ -33,11 +35,13 @@ Configurations) from marvin.lib.common import (get_domain, get_zone, - get_template) + get_template, + list_hosts) from marvin.codes import FAILED, PASS from nose.plugins.attrib import attr #Import System modules import time +import re _multiprocess_shared_ = True class TestDeployVM(cloudstackTestCase): @@ -781,3 +785,312 @@ def test_10_attachAndDetach_iso(self): "Check if ISO is detached from virtual machine" ) return + +class TestSecuredVmMigration(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestSecuredVmMigration, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + + # Get Zone, Domain and templates + domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ + cls.management_ip = cls.config.__dict__["mgtSvr"][0].__dict__["mgtSvrIp"] + + template = get_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"] + ) + if template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + cls.services["iso1"]["zoneid"] = cls.zone.id + + # Create VMs, NAT Rules etc + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) + + cls.small_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["small"] + ) + + cls._cleanup = [ + cls.small_offering, + cls.account + ] + + @classmethod + def tearDownClass(cls): + + cls.apiclient = super(TestSecuredVmMigration, cls).getClsTestClient().getApiClient() + try: + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + self.updateConfiguration("ca.plugin.root.auth.strictness", "false") + self.make_all_hosts_secure() + + + if self.hypervisor.lower() not in ["kvm"]: + self.skipTest("Secured migration is not supported on other than KVM") + + def tearDown(self): + self.make_all_hosts_secure() + + try: + #Clean up, terminate the created ISOs + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_01_secured_vm_migration(self): + """Test secured VM migration""" + + # Validate the following + # 1. Environment has enough hosts for migration + # 2. DeployVM on suitable host (with another host in the cluster) + # 3. Migrate the VM and assert migration successful + + hosts = self.get_hosts() + + secured_hosts = [] + + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + if len(secured_hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") + + origin_host = secured_hosts[0] + + self.vm_to_migrate = self.deploy_vm(origin_host) + + target_host = self.get_target_host(secured='true', virtualmachineid=self.vm_to_migrate.id) + + self.migrate_and_check(origin_host, target_host, proto='tls') + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_02_not_secured_vm_migration(self): + """Test Non-secured VM Migration + """ + #self.skipTest() + # Validate the following + # 1. Prepare 2 hosts to run in non-secured more + # 2. DeployVM on suitable host (with another host in the cluster) + # 3. Migrate the VM and assert migration successful + hosts = self.get_hosts() + for host in hosts: + self.make_unsecure_connection(host) + + non_secured_hosts = [] + + hosts = self.get_hosts() + + for host in hosts: + if host.details.secured == 'false': + non_secured_hosts.append(host) + + if len(non_secured_hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") + origin_host = non_secured_hosts[0] + + self.vm_to_migrate = self.deploy_vm(origin_host) + + target_host = self.get_target_host(secured='false', virtualmachineid=self.vm_to_migrate.id) + + self.migrate_and_check(origin_host, target_host, proto='tcp') + + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_03_secured_to_nonsecured_vm_migration(self): + """Test destroy Virtual Machine + """ + + # Validate the following + # 1. Makes one of the hosts non-secured + # 2. Deploys a VM to a Secured host + # 3. Migrates the VM to the non-secured host and assers the migration is via TCP. + + hosts = self.get_hosts() + + non_secured_host = self.make_unsecure_connection(hosts[0]) + + secured_hosts = [] + hosts = self.get_hosts() + + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + self.vm_to_migrate = self.deploy_vm(secured_hosts[0]) + try: + self.migrate_and_check(origin_host=secured_hosts[0], destination_host=non_secured_host, proto='tcp') + except Exception: + pass + else: self.fail("Migration succeed, instead it should fail") + + + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_04_nonsecured_to_secured_vm_migration(self): + """Test Non-secured VM Migration + """ + + # Validate the following + # 1. Makes one of the hosts non-secured + # 2. Deploys a VM to the non-secured host + # 3. Migrates the VM to the secured host and assers the migration is via TCP. + hosts = self.get_hosts() + + non_secured_host = self.make_unsecure_connection(hosts[0]) + + secured_hosts = [] + + hosts = self.get_hosts() + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + self.vm_to_migrate = self.deploy_vm(non_secured_host) + + try: + self.migrate_and_check(origin_host=non_secured_host, destination_host=secured_hosts[0], proto='tcp') + except Exception: + pass + else: self.fail("Migration succeed, instead it should fail") + + return + + def get_target_host(self, secured, virtualmachineid): + target_hosts = Host.listForMigration(self.apiclient, + virtualmachineid=virtualmachineid) + for host in target_hosts: + h = list_hosts(self.apiclient,type='Routing', id=host.id)[0] + if h.details.secured == secured: + return h + + cloudstackTestCase.skipTest(self, "No target hosts available, skipping test.") + + def check_migration_protocol(self, protocol, host): + resp = SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("cat /var/log/cloudstack/agent/agent.log | grep Live | tail -1") + + if protocol not in resp[0]: + print 'It failed' + cloudstackTestCase.fail(self, "Migration protocol was not as expected: '" + protocol + "\n" + "Instead we got: " + resp[0]) + pass + + def make_unsecure_connection(self, host): + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("rm -f /etc/cloudstack/agent/cloud*") + + guid = re.sub('\-LibvirtComputingResource$', '', self.dbclient.execute("select guid from host where host.uuid = '"+host.id+"';")[0][0]) + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute(" cloudstack-setup-agent -m "+self.management_ip+" -z 1 -p 1 -c 1 -g "+guid+" -a --pubNic=cloudbr1 --prvNic=cloudbr0 --guestNic=cloudbr1 --hypervisor=kvm") + + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("service cloudstack-agent restart") + + self.check_connection(host=host, secured='false') + return host + + def make_all_hosts_secure(self): + hosts = Host.list( + self.apiclient, + zoneid=self.zone.id, + type='Routing' + ) + for host in hosts: + cmd = provisionCertificate.provisionCertificateCmd() + cmd.hostid = host.id + self.apiclient.updateConfiguration(cmd) + + for host in hosts: + self.check_connection(secured='true', host=host) + + pass + + def get_hosts(self): + + hosts = Host.list( + self.apiclient, + zoneid=self.zone.id, + type='Routing' + ) + self.assertEqual(validateList(hosts)[0], PASS, "hosts list validation failed") + return hosts + + def deploy_vm(self, origin_host): + return VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id, + mode=self.services["mode"], + hostid=origin_host.id + ) + + def check_connection(self, secured, host, retries=5, interval=5): + + while retries > -1: + time.sleep(interval) + host = Host.list( + self.apiclient, + zoneid=self.zone.id, + hostid=host.id, + type='Routing' + )[0] + if host.details.secured != secured: + if retries >= 0: + retries = retries - 1 + continue + else: + return + + raise Exception("Host communication is not as expected: " + secured + + ". Instead it's: " + host.details.secured) + + def migrate_and_check(self, origin_host, destination_host, proto): + + self.vm_to_migrate.migrate(self.apiclient, hostid=destination_host.id) + + self.check_migration_protocol(protocol=proto, host=origin_host) + + vm_response = VirtualMachine.list(self.apiclient, id=self.vm_to_migrate.id)[0] + + self.assertEqual(vm_response.hostid, destination_host.id, "Check destination hostID of migrated VM") + pass + + def updateConfiguration(self, name, value): + cmd = updateConfiguration.updateConfigurationCmd() + cmd.name = name + cmd.value = value + self.apiclient.updateConfiguration(cmd) \ No newline at end of file From 0cb76e25cfd79d90ce1527480aa942152be633bf Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 22 Mar 2018 13:48:23 +0530 Subject: [PATCH 05/14] CLOUDSTACK-10333: Secure Live VM Migration for KVM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This extends securing of KVM hosts to securing of libvirt on KVM host as well for TLS enabled live VM migration. To simplify implementation securing of host implies that both host and libvirtd processes are secured with management server's CA plugin issued certificates. Based on whether keystore and certificates files are available at /etc/cloudstack/agent, the KVM agent determines whether to use TLS or TCP based uris for live VM migration. It is also enforced that a secured host will allow live VM migration to/from other secured host, and an unsecured hosts will allow live VM migration to/from other unsecured host only. Post upgrade the KVM agent on startup will expose its security state (secured detail is sent as true or false) to the managements server that gets saved in host_details for the host. This host detail can be accesed via the listHosts response, and in the UI unsecured KVM hosts will show up with the host state of ‘unsecured’. Further, a button has been added that allows admins to provision/renew certificates to KVM hosts and can be used to secure any unsecured KVM host. The `cloudstack-setup-agent` was modified to accept a new flag `-s` which will reconfigure libvirtd with following settings: listen_tcp=0 listen_tls=1 tcp_port="16509" tls_port="16514" auth_tcp="none" auth_tls="none" key_file = "/etc/pki/libvirt/private/serverkey.pem" cert_file = "/etc/pki/libvirt/servercert.pem" ca_file = "/etc/pki/CA/cacert.pem" For a connected KVM host agent, when the certificate are renewed/provisioned a background task is scheduled that waits until all of the agent tasks finish after which libvirt process is restarted and finally the agent is restarted via AgentShell. There are no API or DB changes. Signed-off-by: Rohit Yadav --- agent/bindir/cloud-setup-agent.in | 12 +- agent/src/com/cloud/agent/Agent.java | 148 +++++++++++++++--- agent/src/com/cloud/agent/AgentShell.java | 7 +- agent/src/com/cloud/agent/IAgentShell.java | 5 + .../VirtualRoutingResource.java | 18 +-- .../ca/PostCertificateRenewalCommand.java | 34 ++++ .../ca/SetupCertificateCommand.java | 6 +- debian/cloudstack-agent.postinst | 8 + packaging/centos63/cloud.spec | 8 + packaging/centos7/cloud.spec | 8 + .../ca/provider/RootCAProvider.java | 2 +- .../resource/LibvirtComputingResource.java | 27 +++- .../wrapper/LibvirtMigrateCommandWrapper.java | 21 ++- .../LibvirtModifyTargetsCommandWrapper.java | 2 +- ...tPostCertificateRenewalCommandWrapper.java | 52 ++++++ .../LibvirtMigrateCommandWrapperTest.java | 21 +++ python/lib/cloud_utils.py | 4 +- python/lib/cloudutils/serviceConfig.py | 52 +++--- scripts/util/keystore-cert-import | 12 ++ .../discoverer/LibvirtServerDiscoverer.java | 32 ++-- .../apache/cloudstack/ca/CAManagerImpl.java | 3 +- ui/css/cloudstack3.css | 2 + ui/l10n/en.js | 2 + ui/scripts/system.js | 51 +++++- .../main/java/com/cloud/utils/nio/Link.java | 8 +- .../java/com/cloud/utils/script/Script.java | 2 +- .../com/cloud/utils/ssh/SSHCmdHelper.java | 4 +- .../utils/security/KeyStoreUtils.java | 36 +++-- 28 files changed, 472 insertions(+), 115 deletions(-) create mode 100644 core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java create mode 100644 plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java diff --git a/agent/bindir/cloud-setup-agent.in b/agent/bindir/cloud-setup-agent.in index 8d2b91961aef..3c6203c2d340 100755 --- a/agent/bindir/cloud-setup-agent.in +++ b/agent/bindir/cloud-setup-agent.in @@ -26,6 +26,7 @@ from cloudutils.configFileOps import configFileOps from cloudutils.globalEnv import globalEnv from cloudutils.networkConfig import networkConfig from cloudutils.syscfg import sysConfigFactory +from cloudutils.serviceConfig import configureLibvirtConfig from optparse import OptionParser @@ -100,6 +101,7 @@ if __name__ == '__main__': parser.add_option("-c", "--cluster", dest="cluster", help="cluster id") parser.add_option("-t", "--hypervisor", default="kvm", dest="hypervisor", help="hypervisor type") parser.add_option("-g", "--guid", dest="guid", help="guid") + parser.add_option("-s", action="store_true", default=False, dest="secure", help="Secure and enable TLS for libvirtd") parser.add_option("--pubNic", dest="pubNic", help="Public traffic interface") parser.add_option("--prvNic", dest="prvNic", help="Private traffic interface") parser.add_option("--guestNic", dest="guestNic", help="Guest traffic interface") @@ -110,6 +112,12 @@ if __name__ == '__main__': glbEnv.bridgeType = bridgeType (options, args) = parser.parse_args() + + if not options.auto and options.secure: + configureLibvirtConfig(True) + print "Libvirtd with TLS configured" + sys.exit(0) + if options.auto is None: userInputs = getUserInputs() glbEnv.mgtSvr = userInputs[0] @@ -138,7 +146,9 @@ if __name__ == '__main__': glbEnv.nics.append(options.prvNic) glbEnv.nics.append(options.pubNic) glbEnv.nics.append(options.guestNic) - + + glbEnv.secure = options.secure + print "Starting to configure your system:" syscfg = sysConfigFactory.getSysConfigFactory(glbEnv) try: diff --git a/agent/src/com/cloud/agent/Agent.java b/agent/src/com/cloud/agent/Agent.java index 32112540c1c3..74b5205df08c 100644 --- a/agent/src/com/cloud/agent/Agent.java +++ b/agent/src/com/cloud/agent/Agent.java @@ -42,6 +42,7 @@ import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate; import org.apache.cloudstack.agent.lb.SetupMSListAnswer; import org.apache.cloudstack.agent.lb.SetupMSListCommand; +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; import org.apache.cloudstack.ca.SetupCertificateAnswer; import org.apache.cloudstack.ca.SetupCertificateCommand; import org.apache.cloudstack.ca.SetupKeyStoreCommand; @@ -68,6 +69,7 @@ import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.exception.AgentControlChannelException; +import com.cloud.host.Host; import com.cloud.resource.ServerResource; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.StringUtils; @@ -127,6 +129,7 @@ public int value() { Long _id; Timer _timer = new Timer("Agent Timer"); + Timer certTimer; Timer hostLBTimer; List _watchList = new ArrayList(); @@ -140,9 +143,11 @@ public int value() { long _startupWait = _startupWaitDefault; boolean _reconnectAllowed = true; //For time sentitive task, e.g. PingTask - private final ThreadPoolExecutor _ugentTaskPool; + ThreadPoolExecutor _ugentTaskPool; ExecutorService _executor; + Thread _shutdownThread = new ShutdownThread(this); + private String _keystoreSetupPath; private String _keystoreCertImportPath; @@ -153,7 +158,7 @@ public Agent(final IAgentShell shell) { _connection = new NioClient("Agent", _shell.getNextHost(), _shell.getPort(), _shell.getWorkers(), this); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -192,7 +197,7 @@ public Agent(final IAgentShell shell, final int localAgentId, final ServerResour // ((NioClient)_connection).setBindAddress(_shell.getPrivateIp()); s_logger.debug("Adding shutdown hook"); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -245,14 +250,14 @@ public void start() { throw new CloudRuntimeException("Unable to start the resource: " + _resource.getName()); } - _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreSetupScript); + _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_SETUP_SCRIPT); if (_keystoreSetupPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreSetupScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_SETUP_SCRIPT)); } - _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreImportScript); + _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_IMPORT_SCRIPT); if (_keystoreCertImportPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreImportScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_IMPORT_SCRIPT)); } try { @@ -274,6 +279,19 @@ public void start() { } } _shell.updateConnectedHost(); + + // In case of software based restart, GC to remove old instances + _executor.submit(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(2000L); + } catch (final InterruptedException ignored) { + } finally { + System.gc(); + } + } + }); } public void stop(final String reason, final String detail) { @@ -298,6 +316,7 @@ public void stop(final String reason, final String detail) { } _connection.stop(); _connection = null; + _link = null; } if (_resource != null) { @@ -305,7 +324,34 @@ public void stop(final String reason, final String detail) { _resource = null; } - _ugentTaskPool.shutdownNow(); + if (_startup != null) { + _startup = null; + } + + if (_ugentTaskPool != null) { + _ugentTaskPool.shutdownNow(); + _ugentTaskPool = null; + } + + if (_executor != null) { + _executor.shutdown(); + _executor = null; + } + + if (_timer != null) { + _timer.cancel(); + _timer = null; + } + + if (hostLBTimer != null) { + hostLBTimer.cancel(); + hostLBTimer = null; + } + + if (certTimer != null) { + certTimer.cancel(); + certTimer = null; + } } public Long getId() { @@ -318,6 +364,15 @@ public void setId(final Long id) { _shell.setPersistentProperty(getResourceName(), "id", Long.toString(id)); } + private synchronized void scheduleServicesRestartTask() { + if (certTimer != null) { + certTimer.cancel(); + certTimer.purge(); + } + certTimer = new Timer("Certificate Renewal Timer"); + certTimer.schedule(new PostCertificateRenewalTask(this), 5000L); + } + private synchronized void scheduleHostLBCheckerTask(final long checkInterval) { if (hostLBTimer != null) { hostLBTimer.cancel(); @@ -578,6 +633,9 @@ protected void processRequest(final Request request, final Link link) { answer = setupAgentKeystore((SetupKeyStoreCommand) cmd); } else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) { answer = setupAgentCertificate((SetupCertificateCommand) cmd); + if (Host.Type.Routing.equals(_resource.getType())) { + scheduleServicesRestartTask(); + } } else if (cmd instanceof SetupDirectDownloadCertificate) { answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd); } else if (cmd instanceof SetupMSListCommand) { @@ -641,7 +699,7 @@ private Answer setupDirectDownloadCertificate(SetupDirectDownloadCertificate cmd return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; String cerFile = agentFile.getParent() + "/" + certificateName + ".cer"; Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile)); @@ -666,13 +724,13 @@ public Answer setupAgentKeystore(final SetupKeyStoreCommand cmd) { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; - final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCsrFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + final String csrFile = agentFile.getParent() + "/" + KeyStoreUtils.CSR_FILENAME; - String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.passphrasePropertyName); + String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (Strings.isNullOrEmpty(storedPassword)) { storedPassword = keyStorePassword; - _shell.setPersistentProperty(null, KeyStoreUtils.passphrasePropertyName, storedPassword); + _shell.setPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY, storedPassword); } Script script = new Script(true, _keystoreSetupPath, 60000, s_logger); @@ -706,10 +764,10 @@ private Answer setupAgentCertificate(final SetupCertificateCommand cmd) { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; - final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCertFile; - final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultPrivateKeyFile; - final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultCaCertFile; + final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + final String certFile = agentFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME; + final String privateKeyFile = agentFile.getParent() + "/" + KeyStoreUtils.PKEY_FILENAME; + final String caCertFile = agentFile.getParent() + "/" + KeyStoreUtils.CACERT_FILENAME; try { FileUtils.writeStringToFile(new File(certFile), certificate, Charset.defaultCharset()); @@ -722,7 +780,7 @@ private Answer setupAgentCertificate(final SetupCertificateCommand cmd) { Script script = new Script(true, _keystoreCertImportPath, 60000, s_logger); script.add(agentFile.getAbsolutePath()); script.add(keyStoreFile); - script.add(KeyStoreUtils.agentMode); + script.add(KeyStoreUtils.AGENT_MODE); script.add(certFile); script.add(""); script.add(caCertFile); @@ -1072,6 +1130,60 @@ public void doTask(final Task task) throws TaskExecutionException { } } + /** + * Task stops the current agent and launches a new agent + * when there are no outstanding jobs in the agent's task queue + */ + public class PostCertificateRenewalTask extends ManagedContextTimerTask { + + private Agent agent; + + public PostCertificateRenewalTask(final Agent agent) { + this.agent = agent; + } + + @Override + protected void runInContext() { + while (true) { + try { + if (_inProgress.get() == 0) { + s_logger.debug("Running post certificate renewal task to restart services."); + + // Let the resource perform any post certificate renewal cleanups + _resource.executeRequest(new PostCertificateRenewalCommand()); + + IAgentShell shell = agent._shell; + ServerResource resource = agent._resource.getClass().newInstance(); + + // Stop current agent + agent.cancelTasks(); + agent._reconnectAllowed = false; + Runtime.getRuntime().removeShutdownHook(agent._shutdownThread); + agent.stop(ShutdownCommand.Requested, "Restarting due to new X509 certificates"); + + // Nullify references for GC + agent._shell = null; + agent._watchList = null; + agent._shutdownThread = null; + agent._controlListeners = null; + agent = null; + + // Start a new agent instance + shell.launchNewAgent(resource); + return; + } + if (s_logger.isTraceEnabled()) { + s_logger.debug("Other tasks are in progress, will retry post certificate renewal command after few seconds"); + } + Thread.sleep(5000); + } catch (final Exception e) { + s_logger.warn("Failed to execute post certificate renewal command:", e); + break; + } + } + } + } + public class PreferredHostCheckerTask extends ManagedContextTimerTask { @Override diff --git a/agent/src/com/cloud/agent/AgentShell.java b/agent/src/com/cloud/agent/AgentShell.java index 13b6c65a3519..01654ac9caa9 100644 --- a/agent/src/com/cloud/agent/AgentShell.java +++ b/agent/src/com/cloud/agent/AgentShell.java @@ -419,7 +419,7 @@ private void launchAgentFromClassInfo(String resourceClassNames) throws Configur final Constructor constructor = impl.getDeclaredConstructor(); constructor.setAccessible(true); ServerResource resource = (ServerResource)constructor.newInstance(); - launchAgent(getNextAgentId(), resource); + launchNewAgent(resource); } catch (final ClassNotFoundException e) { throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString()); } catch (final SecurityException e) { @@ -447,9 +447,10 @@ private void launchAgentFromTypeInfo() throws ConfigurationException { s_logger.trace("Launching agent based on type=" + typeInfo); } - private void launchAgent(int localAgentId, ServerResource resource) throws ConfigurationException { + public void launchNewAgent(ServerResource resource) throws ConfigurationException { // we don't track agent after it is launched for now - Agent agent = new Agent(this, localAgentId, resource); + _agents.clear(); + Agent agent = new Agent(this, getNextAgentId(), resource); _agents.add(agent); agent.start(); } diff --git a/agent/src/com/cloud/agent/IAgentShell.java b/agent/src/com/cloud/agent/IAgentShell.java index 5b52cee63615..5d389a07041b 100644 --- a/agent/src/com/cloud/agent/IAgentShell.java +++ b/agent/src/com/cloud/agent/IAgentShell.java @@ -19,6 +19,9 @@ import java.util.Map; import java.util.Properties; +import javax.naming.ConfigurationException; + +import com.cloud.resource.ServerResource; import com.cloud.utils.backoff.BackoffAlgorithm; public interface IAgentShell { @@ -66,4 +69,6 @@ public interface IAgentShell { void updateConnectedHost(); String getConnectedHost(); + + void launchNewAgent(ServerResource resource) throws ConfigurationException; } diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index d7c3d56f9b9b..0ffe8cc0ea2f 100644 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -157,11 +157,11 @@ private Answer execute(final SetupKeyStoreCommand cmd) { "/usr/local/cloud/systemvm/conf/%s " + "%s %d " + "/usr/local/cloud/systemvm/conf/%s", - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_FILENAME, cmd.getKeystorePassword(), cmd.getValidityDays(), - KeyStoreUtils.defaultCsrFile); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreSetupScript, args); + KeyStoreUtils.CSR_FILENAME); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_SETUP_SCRIPT, args); return new SetupKeystoreAnswer(result.getDetails()); } @@ -171,15 +171,15 @@ private Answer execute(final SetupCertificateCommand cmd) { "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\"", - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, cmd.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, cmd.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, cmd.getEncodedPrivateKey()); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreImportScript, args); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_IMPORT_SCRIPT, args); return new SetupCertificateAnswer(result.isSuccess()); } diff --git a/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java b/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java new file mode 100644 index 000000000000..12df6196128d --- /dev/null +++ b/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.ca; + +import com.cloud.agent.api.Command; + +public class PostCertificateRenewalCommand extends Command { + + public PostCertificateRenewalCommand() { + super(); + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java b/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java index 1cd31509d392..7727282bcee4 100644 --- a/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java +++ b/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java @@ -82,15 +82,15 @@ public String getCaCertificates() { } public String getEncodedPrivateKey() { - return privateKey.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return privateKey.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCertificate() { - return certificate.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return certificate.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCaCertificates() { - return caCertificates.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return caCertificates.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public boolean isHandleByAgent() { diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index a9b8b687fab1..15ccdfa5eafc 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -50,6 +50,14 @@ case "$1" in mkdir /etc/libvirt/hooks fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu + + # Enable libvirt TLS if host is secured + if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then + /usr/bin/cloudstack-setup-agent -s + /sbin/service libvirt-bin restart + /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + /sbin/iptables-save > /etc/iptables/rules.v4 + fi ;; esac diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 898118fa96df..221cd86bd4be 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -493,6 +493,14 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi +# Enable libvirt TLS if host is secured +if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then + /usr/bin/cloudstack-setup-agent -s + /sbin/service libvirtd restart + /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + /sbin/service iptables save +fi + %preun usage /sbin/service cloudstack-usage stop || true if [ "$1" == "0" ] ; then diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index f16858a4a8f1..07ef3c44b407 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -437,6 +437,14 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi +# Enable libvirt TLS if host is secured +if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then + /usr/bin/cloudstack-setup-agent -s + /sbin/service libvirtd restart + /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + /sbin/service iptables save +fi + %pre usage id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -c "CloudStack unprivileged user" \ -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true diff --git a/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java index 4a3585ac4e26..6584b35861a0 100644 --- a/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -241,7 +241,7 @@ public KeyStore getManagementKeyStore() throws KeyStoreException { @Override public char[] getKeyStorePassphrase() { - return KeyStoreUtils.defaultKeystorePassphrase; + return KeyStoreUtils.DEFAULT_KS_PASSPHRASE; } ///////////////////////////////////////////////// diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index dd039e54263b..d7c1b497b4d4 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -41,11 +41,19 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; + import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; +import org.apache.cloudstack.utils.linux.CPUStat; +import org.apache.cloudstack.utils.linux.MemStat; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; @@ -68,14 +76,6 @@ import org.w3c.dom.NodeList; import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import com.google.common.base.Strings; - -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; -import org.apache.cloudstack.utils.linux.CPUStat; -import org.apache.cloudstack.utils.linux.MemStat; -import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -168,6 +168,7 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VmDetailConstants; +import com.google.common.base.Strings; /** * LibvirtComputingResource execute requests on the computing/routing host using @@ -239,6 +240,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected long _hypervisorLibvirtVersion; protected long _hypervisorQemuVersion; protected String _hypervisorPath; + protected String _hostDistro; protected String _networkDirectSourceMode; protected String _networkDirectDevice; protected String _sysvmISOPath; @@ -2599,11 +2601,16 @@ public StartupCommand[] initialize() { fillNetworkInformation(cmd); _privateIp = cmd.getPrivateIpAddress(); cmd.getHostDetails().putAll(getVersionStrings()); + cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(KeyStoreUtils.isHostSecured()).toLowerCase()); cmd.setPool(_pool); cmd.setCluster(_clusterId); cmd.setGatewayIpAddress(_localGateway); cmd.setIqn(getIqn()); + if (cmd.getHostDetails().containsKey("Host.OS")) { + _hostDistro = cmd.getHostDetails().get("Host.OS"); + } + StartupStorageCommand sscmd = null; try { @@ -3777,4 +3784,8 @@ public void restoreVMSnapshotMetadata(Domain dm, String vmName, List { @@ -80,9 +80,17 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage, command.isAutoConvergence(), vmName, command.getDestinationIp()); @@ -203,6 +211,9 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. } catch (final LibvirtException e) { s_logger.debug("Can't migrate domain: " + e.getMessage()); result = e.getMessage(); + if (result.startsWith("unable to connect to server") && result.endsWith("refused")) { + result = String.format("Migration was refused connection to destination: %s. Please check libvirt configuration compatibility on the source and destination hosts.", destinationUri); + } } catch (final InterruptedException e) { s_logger.debug("Interrupted while migrating domain: " + e.getMessage()); result = e.getMessage(); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java index 627d4b7beb6d..724caad3f227 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyTargetsCommandWrapper.java @@ -38,7 +38,7 @@ @ResourceWrapper(handles = ModifyTargetsCommand.class) public final class LibvirtModifyTargetsCommandWrapper extends CommandWrapper { - private static final Logger s_logger = Logger.getLogger(LibvirtMigrateCommandWrapper.class); + private static final Logger s_logger = Logger.getLogger(LibvirtModifyTargetsCommandWrapper.class); @Override public Answer execute(final ModifyTargetsCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java new file mode 100644 index 000000000000..df89d2470dda --- /dev/null +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; +import org.apache.cloudstack.ca.SetupCertificateAnswer; +import org.apache.log4j.Logger; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = PostCertificateRenewalCommand.class) +public final class LibvirtPostCertificateRenewalCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtPostCertificateRenewalCommandWrapper.class); + + @Override + public Answer execute(final PostCertificateRenewalCommand command, final LibvirtComputingResource serverResource) { + s_logger.info("Restarting libvirt after certificate provisioning/renewal"); + if (command != null) { + final int timeout = 30000; + Script script = new Script(true, "service", timeout, s_logger); + if ("Ubuntu".equals(serverResource.getHostDistro()) || "Debian".equals(serverResource.getHostDistro())) { + script.add("libvirt-bin"); + } else { + script.add("libvirtd"); + } + script.add("restart"); + script.execute(); + return new SetupCertificateAnswer(true); + } + return new SetupCertificateAnswer(false); + } +} diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index ed13cb25f0c8..19848a1f9406 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -18,10 +18,14 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.junit.Test; +import com.cloud.utils.exception.CloudRuntimeException; + public class LibvirtMigrateCommandWrapperTest { String fullfile = "\n" + @@ -303,4 +307,21 @@ public void testReplaceFqdnForVNCInDesc() { final String result = lw.replaceIpForVNCInDescFile(xmlDesc, targetIp); assertTrue("transformation does not live up to expectation:\n" + result, expectedXmlDesc.equals(result)); } + + @Test + public void testMigrationUri() { + final String ip = "10.1.1.1"; + LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); + if (KeyStoreUtils.isHostSecured()) { + assertEquals(lw.createMigrationURI(ip), String.format("qemu+tls://%s/system", ip)); + } else { + assertEquals(lw.createMigrationURI(ip), String.format("qemu+tcp://%s/system", ip)); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testMigrationUriException() { + LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); + lw.createMigrationURI(null); + } } diff --git a/python/lib/cloud_utils.py b/python/lib/cloud_utils.py index 43c93c876316..3c69c5652b14 100644 --- a/python/lib/cloud_utils.py +++ b/python/lib/cloud_utils.py @@ -802,9 +802,9 @@ def done(self): rule = "-p tcp -m tcp --dport 16509 -j ACCEPT" if rule in iptablessave().stdout: return True return False - + def execute(self): - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() if distro in (Fedora , CentOS, RHEL6): for p in ports: iptables("-I","INPUT","1","-p","tcp","--dport",p,'-j','ACCEPT') o = service.iptables.save() ; print o.stdout + o.stderr diff --git a/python/lib/cloudutils/serviceConfig.py b/python/lib/cloudutils/serviceConfig.py index c9c019c5abe5..3b9f6127e6bd 100755 --- a/python/lib/cloudutils/serviceConfig.py +++ b/python/lib/cloudutils/serviceConfig.py @@ -471,6 +471,23 @@ def restore(self): logging.debug(formatExceptionInfo()) return False +def configureLibvirtConfig(tls_enabled = True, cfg = None): + cfo = configFileOps("/etc/libvirt/libvirtd.conf", cfg) + if tls_enabled: + cfo.addEntry("listen_tcp", "0") + cfo.addEntry("listen_tls", "1") + cfo.addEntry("key_file", "\"/etc/pki/libvirt/private/serverkey.pem\"") + cfo.addEntry("cert_file", "\"/etc/pki/libvirt/servercert.pem\"") + cfo.addEntry("ca_file", "\"/etc/pki/CA/cacert.pem\"") + else: + cfo.addEntry("listen_tcp", "1") + cfo.addEntry("listen_tls", "0") + cfo.addEntry("tcp_port", "\"16509\"") + cfo.addEntry("tls_port", "\"16514\"") + cfo.addEntry("auth_tcp", "\"none\"") + cfo.addEntry("auth_tls", "\"none\"") + cfo.save() + class libvirtConfigRedhat(serviceCfgBase): def __init__(self, syscfg): super(libvirtConfigRedhat, self).__init__(syscfg) @@ -478,12 +495,7 @@ def __init__(self, syscfg): def config(self): try: - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\"") - cfo.addEntry("auth_tcp", "\"none\"") - cfo.addEntry("listen_tls", "0") - cfo.save() + configureLibvirtConfig(self.syscfg.env.secure, self) cfo = configFileOps("/etc/sysconfig/libvirtd", self) cfo.addEntry("export CGROUP_DAEMON", "'cpu:/virt'") @@ -515,24 +527,16 @@ def __init__(self, syscfg): super(libvirtConfigUbuntu, self).__init__(syscfg) self.serviceName = "Libvirt" - def setupLiveMigration(self): - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\""); - cfo.addEntry("auth_tcp", "\"none\""); - cfo.addEntry("listen_tls", "0") - cfo.save() - - if os.path.exists("/etc/init/libvirt-bin.conf"): - cfo = configFileOps("/etc/init/libvirt-bin.conf", self) - cfo.replace_line("exec /usr/sbin/libvirtd","exec /usr/sbin/libvirtd -d -l") - else: - cfo = configFileOps("/etc/default/libvirt-bin", self) - cfo.replace_or_add_line("libvirtd_opts=","libvirtd_opts='-l -d'") - def config(self): try: - self.setupLiveMigration() + configureLibvirtConfig(self.syscfg.env.secure, self) + if os.path.exists("/etc/init/libvirt-bin.conf"): + cfo = configFileOps("/etc/init/libvirt-bin.conf", self) + cfo.replace_line("exec /usr/sbin/libvirtd","exec /usr/sbin/libvirtd -d -l") + else: + cfo = configFileOps("/etc/default/libvirt-bin", self) + cfo.replace_or_add_line("libvirtd_opts=","libvirtd_opts='-l -d'") + filename = "/etc/libvirt/qemu.conf" @@ -564,7 +568,7 @@ def __init__(self, syscfg): def config(self): try: - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() for p in ports: bash("ufw allow %s"%p) bash("ufw allow proto tcp from any to any port 5900:6100") @@ -624,7 +628,7 @@ def restore(self): class firewallConfigAgent(firewallConfigBase): def __init__(self, syscfg): super(firewallConfigAgent, self).__init__(syscfg) - self.ports = "22 16509 5900:6100 49152:49216".split() + self.ports = "22 16509 16514 5900:6100 49152:49216".split() if syscfg.env.distribution.getVersion() == "CentOS": self.rules = ["-D FORWARD -j RH-Firewall-1-INPUT"] else: diff --git a/scripts/util/keystore-cert-import b/scripts/util/keystore-cert-import index 67ce3400345d..eeaae7c83550 100755 --- a/scripts/util/keystore-cert-import +++ b/scripts/util/keystore-cert-import @@ -28,6 +28,7 @@ PRIVKEY=$(echo "$9" | tr '^' '\n' | tr '~' ' ') ALIAS="cloud" SYSTEM_FILE="/var/cache/cloud/cmdline" +LIBVIRTD_FILE="/etc/libvirt/libvirtd.conf" # Find keystore password KS_PASS=$(sed -n '/keystore.passphrase/p' "$PROPS_FILE" 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null) @@ -78,6 +79,17 @@ fi rm -f "$NEW_KS_FILE.p12" mv -f "$NEW_KS_FILE" "$KS_FILE" +# Secure libvirtd on cert import +if [ -f "$LIBVIRTD_FILE" ]; then + mkdir -p /etc/pki/libvirt/private + ln -sf /etc/cloudstack/agent/cloud.ca.crt /etc/pki/CA/cacert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/clientcert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/servercert.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/clientkey.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/serverkey.pem + cloudstack-setup-agent -s > /dev/null +fi + # Update ca-certs if we're in systemvm if [ -f "$SYSTEM_FILE" ]; then mkdir -p /usr/local/share/ca-certificates/cloudstack diff --git a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index ef69fdcb4366..0b8b40b1b9b0 100644 --- a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -18,6 +18,7 @@ import java.net.InetAddress; import java.net.URI; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -141,11 +142,6 @@ public boolean processTimeout(long agentId, long seq) { } private void setupAgentSecurity(final Connection sshConnection, final String agentIp, final String agentHostname) { - if (!caManager.canProvisionCertificates()) { - s_logger.warn("Cannot secure agent communication because configure CA plugin cannot provision client certificate"); - return; - } - if (sshConnection == null) { throw new CloudRuntimeException("Cannot secure agent communication because ssh connection is invalid for host ip=" + agentIp); } @@ -161,17 +157,17 @@ private void setupAgentSecurity(final Connection sshConnection, final String age "/etc/cloudstack/agent/%s " + "%s %d " + "/etc/cloudstack/agent/%s", - KeyStoreUtils.keyStoreSetupScript, - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_SETUP_SCRIPT, + KeyStoreUtils.KS_FILENAME, PasswordGenerator.generateRandomPassword(16), validityPeriod, - KeyStoreUtils.defaultCsrFile)); + KeyStoreUtils.CSR_FILENAME)); if (!keystoreSetupResult.isSuccess()) { throw new CloudRuntimeException("Failed to setup keystore on the KVM host: " + agentIp); } - final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Collections.singletonList(agentHostname), Collections.singletonList(agentIp), null, null); + final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Arrays.asList(agentHostname, agentIp), Collections.singletonList(agentIp), null, null); if (certificate == null || certificate.getClientCertificate() == null) { throw new CloudRuntimeException("Failed to issue certificates for KVM host agent: " + agentIp); } @@ -184,14 +180,14 @@ private void setupAgentSecurity(final Connection sshConnection, final String age "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\"", - KeyStoreUtils.keyStoreImportScript, - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_IMPORT_SCRIPT, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, certificateCommand.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, certificateCommand.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, certificateCommand.getEncodedPrivateKey())); if (setupCertResult != null && !setupCertResult.isSuccess()) { @@ -288,9 +284,13 @@ private void setupAgentSecurity(final Connection sshConnection, final String age kvmGuestNic = (kvmPublicNic != null) ? kvmPublicNic : kvmPrivateNic; } + if (!caManager.canProvisionCertificates()) { + throw new CloudRuntimeException("Configured CA plugin cannot provision X509 certificate(s), failing to add host due to security insufficiency."); + } + setupAgentSecurity(sshConnection, agentIp, hostname); - String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a"; + String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a -s "; parameters += " --pubNic=" + kvmPublicNic; parameters += " --prvNic=" + kvmPrivateNic; diff --git a/server/src/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/org/apache/cloudstack/ca/CAManagerImpl.java index 3a904315fe16..23a0379305b8 100644 --- a/server/src/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/org/apache/cloudstack/ca/CAManagerImpl.java @@ -27,7 +27,6 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -191,7 +190,7 @@ public boolean provisionCertificate(final Host host, final Boolean reconnect, fi if (Strings.isNullOrEmpty(csr)) { return false; } - final Certificate certificate = issueCertificate(csr, Collections.singletonList(host.getName()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); + final Certificate certificate = issueCertificate(csr, Arrays.asList(host.getName(), host.getPrivateIpAddress()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); return deployCertificate(host, certificate, reconnect, null); } catch (final AgentUnavailableException | OperationTimedoutException e) { LOG.error("Host/agent is not available or operation timed out, failed to setup keystore and generate CSR for host/agent id=" + host.getId() + ", due to: ", e); diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index 519778ba59ba..bab92d0c7d03 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -12612,11 +12612,13 @@ div.ui-dialog div.autoscaler div.field-group div.form-container form div.form-it background-position: -101px -647px; } +.secureKVMHost .icon, .resetPassword .icon, .changePassword .icon { background-position: -68px -30px; } +.secureKVMHost:hover .icon, .resetPassword:hover .icon, .changePassword:hover .icon { background-position: -68px -612px; diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 7b19946b1ed6..745deb11ca24 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -270,6 +270,7 @@ var dictionary = {"ICMP.code":"ICMP Code", "label.action.restore.instance.processing":"Restoring Instance....", "label.action.revert.snapshot":"Revert to Snapshot", "label.action.revert.snapshot.processing":"Reverting to Snapshot...", +"label.action.secure.host":"Provision Host Security Keys", "label.action.start.instance":"Start Instance", "label.action.start.instance.processing":"Starting Instance....", "label.action.start.router":"Start Router", @@ -1910,6 +1911,7 @@ var dictionary = {"ICMP.code":"ICMP Code", "message.action.reset.password.warning":"Your instance must be stopped before attempting to change its current password.", "message.action.restore.instance":"Please confirm that you want to restore this instance.", "message.action.revert.snapshot":"Please confirm that you want to revert the owning volume to this snapshot.", +"message.action.secure.host":"This will restart the host agent and libvirtd process after applying new X509 certificates, please confirm?", "message.action.start.instance":"Please confirm that you want to start this instance.", "message.action.start.router":"Please confirm that you want to start this router.", "message.action.start.systemvm":"Please confirm that you want to start this system VM.", diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 07520df1d463..63f549ad2925 100755 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -9200,6 +9200,11 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } + }); } @@ -15713,7 +15718,8 @@ 'Down': 'off', 'Disconnected': 'off', 'Alert': 'off', - 'Error': 'off' + 'Error': 'off', + 'Unsecure': 'warning' } }, powerstate: { @@ -15761,6 +15767,10 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } }); } @@ -16530,6 +16540,40 @@ } }, + secureKVMHost: { + label: 'label.action.secure.host', + action: function(args) { + var data = { + hostid: args.context.hosts[0].id + }; + $.ajax({ + url: createURL('provisionCertificate'), + data: data, + async: true, + success: function(json) { + args.response.success({ + _custom: { + jobId: json.provisioncertificateresponse.jobid, + getActionFilter: function () { + return hostActionfilter; + } + } + }); + } + }); + }, + messages: { + confirm: function (args) { + return 'message.action.secure.host'; + }, + notification: function (args) { + return 'label.action.secure.host'; + } + }, + notification: { + poll: pollAsyncJobResult + } + }, enableMaintenanceMode: { label: 'label.action.enable.maintenance.mode', @@ -21968,6 +22012,11 @@ if (jsonObj.state != "Disconnected") allowedActions.push("forceReconnect"); + + if (jsonObj.hypervisor == "KVM") { + allowedActions.push("secureKVMHost"); + } + } else if (jsonObj.resourcestate == "ErrorInMaintenance") { allowedActions.push("edit"); allowedActions.push("enableMaintenanceMode"); diff --git a/utils/src/main/java/com/cloud/utils/nio/Link.java b/utils/src/main/java/com/cloud/utils/nio/Link.java index 25f6662c5225..658244087709 100644 --- a/utils/src/main/java/com/cloud/utils/nio/Link.java +++ b/utils/src/main/java/com/cloud/utils/nio/Link.java @@ -379,7 +379,7 @@ public static SSLEngine initServerSSLEngine(final CAService caService, final Str return caService.createSSLEngine(sslContext, clientAddress); } s_logger.error("CA service is not configured, by-passing CA manager to create SSL engine"); - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; final KeyStore ks = loadKeyStore(NioConnection.class.getResourceAsStream("/cloud.keystore"), passphrase); final KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509"); final TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509"); @@ -409,11 +409,11 @@ public static SSLContext initManagementSSLContext(final CAService caService) thr } public static SSLContext initClientSSLContext() throws GeneralSecurityException, IOException { - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; File confFile = PropertiesUtil.findConfigFile("agent.properties"); if (confFile != null) { s_logger.info("Conf file found: " + confFile.getAbsolutePath()); - final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.passphrasePropertyName); + final String pass = PropertiesUtil.loadFromFile(confFile).getProperty(KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (pass != null) { passphrase = pass.toCharArray(); } @@ -421,7 +421,7 @@ public static SSLContext initClientSSLContext() throws GeneralSecurityException, InputStream stream = null; if (confFile != null) { - final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile; + final String keystorePath = confFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; if (new File(keystorePath).exists()) { stream = new FileInputStream(keystorePath); } diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index 01f18bda2d2e..b73371ac7154 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -203,7 +203,7 @@ public String execute(OutputInterpreter interpreter) { String[] command = _command.toArray(new String[_command.size()]); if (_logger.isDebugEnabled()) { - _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.defaultKeystoreFile)[0]); + _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0]); } try { diff --git a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java index 10407b65642e..5324cdcbc18f 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java @@ -139,7 +139,7 @@ public static SSHCmdResult sshExecuteCmdWithResult(com.trilead.ssh2.Connection s } public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshConnection, String cmd) throws SshException { - s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0]); + s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0]); Session sshSession = null; try { sshSession = sshConnection.openSession(); @@ -202,7 +202,7 @@ public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshC final SSHCmdResult result = new SSHCmdResult(-1, sbStdoutResult.toString(), sbStdErrResult.toString()); if (!Strings.isNullOrEmpty(result.getStdOut()) || !Strings.isNullOrEmpty(result.getStdErr())) { - s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); + s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); } // exit status delivery might get delayed diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java b/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java index 8259d77d4ff0..12bf89ea41af 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java @@ -19,23 +19,31 @@ package org.apache.cloudstack.utils.security; +import java.io.File; + +import com.cloud.utils.PropertiesUtil; + public class KeyStoreUtils { + public static final String KS_SETUP_SCRIPT = "keystore-setup"; + public static final String KS_IMPORT_SCRIPT = "keystore-cert-import"; + public static final String KS_PASSPHRASE_PROPERTY = "keystore.passphrase"; - public static String defaultTmpKeyStoreFile = "/tmp/tmp.jks"; - public static String defaultKeystoreFile = "cloud.jks"; - public static String defaultPrivateKeyFile = "cloud.key"; - public static String defaultCsrFile = "cloud.csr"; - public static String defaultCertFile = "cloud.crt"; - public static String defaultCaCertFile = "cloud.ca.crt"; - public static char[] defaultKeystorePassphrase = "vmops.com".toCharArray(); + public static final String PKEY_FILENAME = "cloud.key"; + public static final String CSR_FILENAME = "cloud.csr"; + public static final String CERT_FILENAME = "cloud.crt"; + public static final String CACERT_FILENAME = "cloud.ca.crt"; + public static final String KS_FILENAME = "cloud.jks"; + public static final char[] DEFAULT_KS_PASSPHRASE = "vmops.com".toCharArray(); - public static String certNewlineEncoder = "^"; - public static String certSpaceEncoder = "~"; + public static final String CERT_NEWLINE_ENCODER = "^"; + public static final String CERT_SPACE_ENCODER = "~"; - public static String keyStoreSetupScript = "keystore-setup"; - public static String keyStoreImportScript = "keystore-cert-import"; - public static String passphrasePropertyName = "keystore.passphrase"; + public static final String SSH_MODE = "ssh"; + public static final String AGENT_MODE = "agent"; + public static final String SECURED = "secured"; - public static String sshMode = "ssh"; - public static String agentMode = "agent"; + public static boolean isHostSecured() { + final File confFile = PropertiesUtil.findConfigFile("agent.properties"); + return confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + CERT_FILENAME).exists(); + } } From e8196198718a4e9278aa94960ba7b709fc9914b0 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 23 Mar 2018 15:45:47 +0530 Subject: [PATCH 06/14] Don't force libvirt reconfig during upgrade Signed-off-by: Rohit Yadav --- debian/cloudstack-agent.postinst | 11 +++++------ packaging/centos63/cloud.spec | 10 ++++------ packaging/centos7/cloud.spec | 10 ++++------ .../kvm/resource/LibvirtComputingResource.java | 16 +++++++++++++++- .../wrapper/LibvirtMigrateCommandWrapper.java | 6 +++--- .../LibvirtMigrateCommandWrapperTest.java | 4 ++-- .../cloudstack/utils/security/KeyStoreUtils.java | 11 +++++++---- 7 files changed, 40 insertions(+), 28 deletions(-) diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index 15ccdfa5eafc..673e7f9d6007 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -51,13 +51,12 @@ case "$1" in fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu - # Enable libvirt TLS if host is secured - if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then - /usr/bin/cloudstack-setup-agent -s - /sbin/service libvirt-bin restart - /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - /sbin/iptables-save > /etc/iptables/rules.v4 + # Enable TLS enabled VM migration for libvirtd + if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then + iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + iptables-save > /etc/iptables/rules.v4 fi + ;; esac diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 221cd86bd4be..d25c28a6d693 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -493,12 +493,10 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi -# Enable libvirt TLS if host is secured -if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then - /usr/bin/cloudstack-setup-agent -s - /sbin/service libvirtd restart - /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - /sbin/service iptables save +# Enable TLS enabled VM migration for libvirtd +if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then + iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + iptables-save > /etc/iptables/rules.v4 fi %preun usage diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index 07ef3c44b407..b499866f1363 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -437,12 +437,10 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi -# Enable libvirt TLS if host is secured -if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then - /usr/bin/cloudstack-setup-agent -s - /sbin/service libvirtd restart - /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - /sbin/service iptables save +# Enable TLS enabled VM migration for libvirtd +if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then + iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + iptables-save > /etc/iptables/rules.v4 fi %pre usage diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index d7c1b497b4d4..a32b60d692c1 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2601,7 +2601,7 @@ public StartupCommand[] initialize() { fillNetworkInformation(cmd); _privateIp = cmd.getPrivateIpAddress(); cmd.getHostDetails().putAll(getVersionStrings()); - cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(KeyStoreUtils.isHostSecured()).toLowerCase()); + cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(isHostSecured()).toLowerCase()); cmd.setPool(_pool); cmd.setCluster(_clusterId); cmd.setGatewayIpAddress(_localGateway); @@ -3788,4 +3788,18 @@ public long getTotalMemory() { public String getHostDistro() { return _hostDistro; } + + public boolean isHostSecured() { + // Test for host certificates + final File confFile = PropertiesUtil.findConfigFile(KeyStoreUtils.AGENT_PROPSFILE); + boolean certExists = confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists(); + // Test for libvirt TLS configuration + boolean libvirtTlsEnabled = true; + try { + new Connect(String.format("qemu+tls://%s/system", _privateIp)); + } catch (final LibvirtException ignored) { + libvirtTlsEnabled = false; + } + return certExists && libvirtTlsEnabled; + } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 79db62368f28..005c937c9e8f 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -80,17 +80,17 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper Date: Fri, 23 Mar 2018 16:09:20 +0530 Subject: [PATCH 07/14] Fix build issues Signed-off-by: Rohit Yadav --- .../kvm/resource/LibvirtComputingResource.java | 10 ++++++---- .../wrapper/LibvirtMigrateCommandWrapperTest.java | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index a32b60d692c1..fc5e5395b87f 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3792,14 +3792,16 @@ public String getHostDistro() { public boolean isHostSecured() { // Test for host certificates final File confFile = PropertiesUtil.findConfigFile(KeyStoreUtils.AGENT_PROPSFILE); - boolean certExists = confFile != null && confFile.exists() && new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists(); + if (confFile == null || !confFile.exists() || !new File(confFile.getParent() + "/" + KeyStoreUtils.CERT_FILENAME).exists()) { + return false; + } + // Test for libvirt TLS configuration - boolean libvirtTlsEnabled = true; try { new Connect(String.format("qemu+tls://%s/system", _privateIp)); } catch (final LibvirtException ignored) { - libvirtTlsEnabled = false; + return false; } - return certExists && libvirtTlsEnabled; + return true; } } diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index 7cfa0eb74eb7..da71e40c30f0 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -312,16 +312,17 @@ public void testReplaceFqdnForVNCInDesc() { public void testMigrationUri() { final String ip = "10.1.1.1"; LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); - if (new LibvirtComputingResource().isHostSecured()) { - assertEquals(lw.createMigrationURI(ip), String.format("qemu+tls://%s/system", ip)); + LibvirtComputingResource lcr = new LibvirtComputingResource(); + if (lcr.isHostSecured()) { + assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tls://%s/system", ip)); } else { - assertEquals(lw.createMigrationURI(ip), String.format("qemu+tcp://%s/system", ip)); + assertEquals(lw.createMigrationURI(ip, lcr), String.format("qemu+tcp://%s/system", ip)); } } @Test(expected = CloudRuntimeException.class) public void testMigrationUriException() { LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper(); - lw.createMigrationURI(null); + lw.createMigrationURI(null, new LibvirtComputingResource()); } } From 439a20a0c7714e279b1b49c6af33c7e20fafdb4c Mon Sep 17 00:00:00 2001 From: Boris Date: Mon, 26 Mar 2018 13:44:18 +0300 Subject: [PATCH 08/14] Add marvin tests to cover secured and non-secured hosts vm migration --- test/integration/smoke/test_vm_life_cycle.py | 321 ++++++++++++++++++- 1 file changed, 317 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 3504849786ba..3ad02ed7b461 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -21,9 +21,11 @@ from marvin.cloudstackAPI import (recoverVirtualMachine, destroyVirtualMachine, attachIso, - detachIso) -from marvin.lib.utils import (cleanup_resources, - validateList) + detachIso, + provisionCertificate, + updateConfiguration) +from marvin.lib.utils import * + from marvin.lib.base import (Account, ServiceOffering, VirtualMachine, @@ -33,11 +35,13 @@ Configurations) from marvin.lib.common import (get_domain, get_zone, - get_template) + get_template, + list_hosts) from marvin.codes import FAILED, PASS from nose.plugins.attrib import attr #Import System modules import time +import re _multiprocess_shared_ = True class TestDeployVM(cloudstackTestCase): @@ -781,3 +785,312 @@ def test_10_attachAndDetach_iso(self): "Check if ISO is detached from virtual machine" ) return + +class TestSecuredVmMigration(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestSecuredVmMigration, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + + # Get Zone, Domain and templates + domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ + cls.management_ip = cls.config.__dict__["mgtSvr"][0].__dict__["mgtSvrIp"] + + template = get_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"] + ) + if template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + cls.services["iso1"]["zoneid"] = cls.zone.id + + # Create VMs, NAT Rules etc + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) + + cls.small_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["small"] + ) + + cls._cleanup = [ + cls.small_offering, + cls.account + ] + + @classmethod + def tearDownClass(cls): + + cls.apiclient = super(TestSecuredVmMigration, cls).getClsTestClient().getApiClient() + try: + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + self.updateConfiguration("ca.plugin.root.auth.strictness", "false") + self.make_all_hosts_secure() + + + if self.hypervisor.lower() not in ["kvm"]: + self.skipTest("Secured migration is not supported on other than KVM") + + def tearDown(self): + self.make_all_hosts_secure() + + try: + #Clean up, terminate the created ISOs + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_01_secured_vm_migration(self): + """Test secured VM migration""" + + # Validate the following + # 1. Environment has enough hosts for migration + # 2. DeployVM on suitable host (with another host in the cluster) + # 3. Migrate the VM and assert migration successful + + hosts = self.get_hosts() + + secured_hosts = [] + + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + if len(secured_hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") + + origin_host = secured_hosts[0] + + self.vm_to_migrate = self.deploy_vm(origin_host) + + target_host = self.get_target_host(secured='true', virtualmachineid=self.vm_to_migrate.id) + + self.migrate_and_check(origin_host, target_host, proto='tls') + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_02_not_secured_vm_migration(self): + """Test Non-secured VM Migration + """ + #self.skipTest() + # Validate the following + # 1. Prepare 2 hosts to run in non-secured more + # 2. DeployVM on suitable host (with another host in the cluster) + # 3. Migrate the VM and assert migration successful + hosts = self.get_hosts() + for host in hosts: + self.make_unsecure_connection(host) + + non_secured_hosts = [] + + hosts = self.get_hosts() + + for host in hosts: + if host.details.secured == 'false': + non_secured_hosts.append(host) + + if len(non_secured_hosts) < 2: + self.skipTest("At least two hosts should be present in the zone for migration") + origin_host = non_secured_hosts[0] + + self.vm_to_migrate = self.deploy_vm(origin_host) + + target_host = self.get_target_host(secured='false', virtualmachineid=self.vm_to_migrate.id) + + self.migrate_and_check(origin_host, target_host, proto='tcp') + + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_03_secured_to_nonsecured_vm_migration(self): + """Test destroy Virtual Machine + """ + + # Validate the following + # 1. Makes one of the hosts non-secured + # 2. Deploys a VM to a Secured host + # 3. Migrates the VM to the non-secured host and assers the migration is via TCP. + + hosts = self.get_hosts() + + non_secured_host = self.make_unsecure_connection(hosts[0]) + + secured_hosts = [] + hosts = self.get_hosts() + + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + self.vm_to_migrate = self.deploy_vm(secured_hosts[0]) + try: + self.migrate_and_check(origin_host=secured_hosts[0], destination_host=non_secured_host, proto='tcp') + except Exception: + pass + else: self.fail("Migration succeed, instead it should fail") + + + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_04_nonsecured_to_secured_vm_migration(self): + """Test Non-secured VM Migration + """ + + # Validate the following + # 1. Makes one of the hosts non-secured + # 2. Deploys a VM to the non-secured host + # 3. Migrates the VM to the secured host and assers the migration is via TCP. + hosts = self.get_hosts() + + non_secured_host = self.make_unsecure_connection(hosts[0]) + + secured_hosts = [] + + hosts = self.get_hosts() + for host in hosts: + if host.details.secured == 'true': + secured_hosts.append(host) + + self.vm_to_migrate = self.deploy_vm(non_secured_host) + + try: + self.migrate_and_check(origin_host=non_secured_host, destination_host=secured_hosts[0], proto='tcp') + except Exception: + pass + else: self.fail("Migration succeed, instead it should fail") + + return + + def get_target_host(self, secured, virtualmachineid): + target_hosts = Host.listForMigration(self.apiclient, + virtualmachineid=virtualmachineid) + for host in target_hosts: + h = list_hosts(self.apiclient,type='Routing', id=host.id)[0] + if h.details.secured == secured: + return h + + cloudstackTestCase.skipTest(self, "No target hosts available, skipping test.") + + def check_migration_protocol(self, protocol, host): + resp = SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("cat /var/log/cloudstack/agent/agent.log | grep Live | tail -1") + + if protocol not in resp[0]: + print 'It failed' + cloudstackTestCase.fail(self, "Migration protocol was not as expected: '" + protocol + "\n" + "Instead we got: " + resp[0]) + pass + + def make_unsecure_connection(self, host): + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("rm -f /etc/cloudstack/agent/cloud*") + + guid = re.sub('\-LibvirtComputingResource$', '', self.dbclient.execute("select guid from host where host.uuid = '"+host.id+"';")[0][0]) + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute(" cloudstack-setup-agent -m "+self.management_ip+" -z 1 -p 1 -c 1 -g "+guid+" -a --pubNic=cloudbr1 --prvNic=cloudbr0 --guestNic=cloudbr1 --hypervisor=kvm") + + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("service cloudstack-agent restart") + + self.check_connection(host=host, secured='false') + return host + + def make_all_hosts_secure(self): + hosts = Host.list( + self.apiclient, + zoneid=self.zone.id, + type='Routing' + ) + for host in hosts: + cmd = provisionCertificate.provisionCertificateCmd() + cmd.hostid = host.id + self.apiclient.updateConfiguration(cmd) + + for host in hosts: + self.check_connection(secured='true', host=host) + + pass + + def get_hosts(self): + + hosts = Host.list( + self.apiclient, + zoneid=self.zone.id, + type='Routing' + ) + self.assertEqual(validateList(hosts)[0], PASS, "hosts list validation failed") + return hosts + + def deploy_vm(self, origin_host): + return VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.small_offering.id, + mode=self.services["mode"], + hostid=origin_host.id + ) + + def check_connection(self, secured, host, retries=5, interval=5): + + while retries > -1: + time.sleep(interval) + host = Host.list( + self.apiclient, + zoneid=self.zone.id, + hostid=host.id, + type='Routing' + )[0] + if host.details.secured != secured: + if retries >= 0: + retries = retries - 1 + continue + else: + return + + raise Exception("Host communication is not as expected: " + secured + + ". Instead it's: " + host.details.secured) + + def migrate_and_check(self, origin_host, destination_host, proto): + + self.vm_to_migrate.migrate(self.apiclient, hostid=destination_host.id) + + self.check_migration_protocol(protocol=proto, host=origin_host) + + vm_response = VirtualMachine.list(self.apiclient, id=self.vm_to_migrate.id)[0] + + self.assertEqual(vm_response.hostid, destination_host.id, "Check destination hostID of migrated VM") + pass + + def updateConfiguration(self, name, value): + cmd = updateConfiguration.updateConfigurationCmd() + cmd.name = name + cmd.value = value + self.apiclient.updateConfiguration(cmd) \ No newline at end of file From 78a15fed7058717e3bfbd2d77044424f5b5def9b Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 26 Mar 2018 16:27:54 +0530 Subject: [PATCH 09/14] Remove iptables addition during agent upgrade for ubuntu/debian Signed-off-by: Rohit Yadav --- debian/cloudstack-agent.postinst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index 673e7f9d6007..a9b8b687fab1 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -50,13 +50,6 @@ case "$1" in mkdir /etc/libvirt/hooks fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu - - # Enable TLS enabled VM migration for libvirtd - if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then - iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - iptables-save > /etc/iptables/rules.v4 - fi - ;; esac From 1729d7793546a108cb733f8fab14d05e914c2137 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 27 Mar 2018 14:20:18 +0530 Subject: [PATCH 10/14] Remove iptables from post-install, remove unused import Signed-off-by: Rohit Yadav --- packaging/centos63/cloud.spec | 6 ------ packaging/centos7/cloud.spec | 6 ------ .../kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java | 1 - 3 files changed, 13 deletions(-) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index d25c28a6d693..898118fa96df 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -493,12 +493,6 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi -# Enable TLS enabled VM migration for libvirtd -if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then - iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - iptables-save > /etc/iptables/rules.v4 -fi - %preun usage /sbin/service cloudstack-usage stop || true if [ "$1" == "0" ] ; then diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index b499866f1363..f16858a4a8f1 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -437,12 +437,6 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi -# Enable TLS enabled VM migration for libvirtd -if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then - iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - iptables-save > /etc/iptables/rules.v4 -fi - %pre usage id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -c "CloudStack unprivileged user" \ -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 005c937c9e8f..bfbf6d0d1f65 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -43,7 +43,6 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; From aaa0d04e2b193c9706231f66fbfbc844cec96fee Mon Sep 17 00:00:00 2001 From: Boris Date: Thu, 29 Mar 2018 16:40:18 +0300 Subject: [PATCH 11/14] Refaktoring the way we 'unsecure' hosts to making a faulty libvirt conf --- test/integration/smoke/test_vm_life_cycle.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 3ad02ed7b461..5954f7a527e0 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -1011,14 +1011,19 @@ def make_unsecure_connection(self, host): SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ .execute("rm -f /etc/cloudstack/agent/cloud*") - guid = re.sub('\-LibvirtComputingResource$', '', self.dbclient.execute("select guid from host where host.uuid = '"+host.id+"';")[0][0]) SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ - .execute(" cloudstack-setup-agent -m "+self.management_ip+" -z 1 -p 1 -c 1 -g "+guid+" -a --pubNic=cloudbr1 --prvNic=cloudbr0 --guestNic=cloudbr1 --hypervisor=kvm") - + .execute("sed -i 's/listen_tls.*/listen_tls=0/g' /etc/libvirt/libvirtd.conf") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("sed -i 's/listen_tcp.*/listen_tcp=1/g' /etc/libvirt/libvirtd.conf ") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("sed -i '/.*_file.*/d' /etc/libvirt/libvirtd.conf") + SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ + .execute("service libvirtd restart") SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ .execute("service cloudstack-agent restart") self.check_connection(host=host, secured='false') + time.sleep(10) return host def make_all_hosts_secure(self): From 3718209287b18069a675897b6aea51e71849978a Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Sun, 1 Apr 2018 03:26:25 +0530 Subject: [PATCH 12/14] Fix pkg issues Signed-off-by: Rohit Yadav --- .../kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java | 1 - scripts/util/keystore-cert-import | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 005c937c9e8f..bfbf6d0d1f65 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -43,7 +43,6 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; diff --git a/scripts/util/keystore-cert-import b/scripts/util/keystore-cert-import index eeaae7c83550..d0f84cdfa8e5 100755 --- a/scripts/util/keystore-cert-import +++ b/scripts/util/keystore-cert-import @@ -81,6 +81,7 @@ mv -f "$NEW_KS_FILE" "$KS_FILE" # Secure libvirtd on cert import if [ -f "$LIBVIRTD_FILE" ]; then + mkdir -p /etc/pki/CA/ mkdir -p /etc/pki/libvirt/private ln -sf /etc/cloudstack/agent/cloud.ca.crt /etc/pki/CA/cacert.pem ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/clientcert.pem From d5a6294a532145a399cc6d6cbeb65c5b92038a3d Mon Sep 17 00:00:00 2001 From: Boris Date: Fri, 13 Apr 2018 12:08:55 +0300 Subject: [PATCH 13/14] Fixing a case where the log starts with a non-text data and 'cat' command returns a non-usable output --- test/integration/smoke/test_vm_life_cycle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 5954f7a527e0..5c5e4f8f81f7 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -999,7 +999,7 @@ def get_target_host(self, secured, virtualmachineid): def check_migration_protocol(self, protocol, host): resp = SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ - .execute("cat /var/log/cloudstack/agent/agent.log | grep Live | tail -1") + .execute("grep -a Live /var/log/cloudstack/agent/agent.log | tail -1") if protocol not in resp[0]: print 'It failed' From b42199fad764a2676f82e2db75e661eb912b06b8 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 20 Apr 2018 00:08:20 +0530 Subject: [PATCH 14/14] address code review comments Signed-off-by: Rohit Yadav --- agent/src/com/cloud/agent/Agent.java | 33 +++++++++++-------- debian/cloudstack-agent.postinst | 6 ---- packaging/centos63/cloud.spec | 6 ---- packaging/centos7/cloud.spec | 6 ---- .../wrapper/LibvirtMigrateCommandWrapper.java | 2 +- python/lib/cloudutils/serviceConfig.py | 16 ++------- scripts/util/keystore-cert-import | 2 +- test/integration/smoke/test_vm_life_cycle.py | 20 ++--------- 8 files changed, 26 insertions(+), 65 deletions(-) diff --git a/agent/src/com/cloud/agent/Agent.java b/agent/src/com/cloud/agent/Agent.java index 74b5205df08c..90e379094346 100644 --- a/agent/src/com/cloud/agent/Agent.java +++ b/agent/src/com/cloud/agent/Agent.java @@ -244,6 +244,25 @@ public String getResourceName() { return _resource.getClass().getSimpleName(); } + /** + * In case of a software based agent restart, this method + * can help to perform explicit garbage collection of any old + * agent instances and its inner objects. + */ + private void scavengeOldAgentObjects() { + _executor.submit(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(2000L); + } catch (final InterruptedException ignored) { + } finally { + System.gc(); + } + } + }); + } + public void start() { if (!_resource.start()) { s_logger.error("Unable to start the resource: " + _resource.getName()); @@ -279,19 +298,7 @@ public void start() { } } _shell.updateConnectedHost(); - - // In case of software based restart, GC to remove old instances - _executor.submit(new Runnable() { - @Override - public void run() { - try { - Thread.sleep(2000L); - } catch (final InterruptedException ignored) { - } finally { - System.gc(); - } - } - }); + scavengeOldAgentObjects(); } public void stop(final String reason, final String detail) { diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index 673e7f9d6007..c358c3ca680a 100755 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -51,12 +51,6 @@ case "$1" in fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu - # Enable TLS enabled VM migration for libvirtd - if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then - iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - iptables-save > /etc/iptables/rules.v4 - fi - ;; esac diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 844db9f79603..87d2c1ec5780 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -497,12 +497,6 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi -# Enable TLS enabled VM migration for libvirtd -if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then - iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - iptables-save > /etc/iptables/rules.v4 -fi - %preun usage /sbin/service cloudstack-usage stop || true if [ "$1" == "0" ] ; then diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index fc3f4a9316c4..525421c0108d 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -439,12 +439,6 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi -# Enable TLS enabled VM migration for libvirtd -if ! iptables-save | grep -- "-A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT" > /dev/null; then - iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT - iptables-save > /etc/iptables/rules.v4 -fi - %pre usage id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -c "CloudStack unprivileged user" \ -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index bfbf6d0d1f65..67ec1b731af9 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -211,7 +211,7 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0. s_logger.debug("Can't migrate domain: " + e.getMessage()); result = e.getMessage(); if (result.startsWith("unable to connect to server") && result.endsWith("refused")) { - result = String.format("Migration was refused connection to destination: %s. Please check libvirt configuration compatibility on the source and destination hosts.", destinationUri); + result = String.format("Migration was refused connection to destination: %s. Please check libvirt configuration compatibility and firewall rules on the source and destination hosts.", destinationUri); } } catch (final InterruptedException e) { s_logger.debug("Interrupted while migrating domain: " + e.getMessage()); diff --git a/python/lib/cloudutils/serviceConfig.py b/python/lib/cloudutils/serviceConfig.py index 3d1a2951cd81..2b27868db05f 100755 --- a/python/lib/cloudutils/serviceConfig.py +++ b/python/lib/cloudutils/serviceConfig.py @@ -528,12 +528,7 @@ def __init__(self, syscfg): self.serviceName = "Libvirt" def setupLiveMigration(self): - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\""); - cfo.addEntry("auth_tcp", "\"none\""); - cfo.addEntry("listen_tls", "0") - cfo.save() + configureLibvirtConfig(self.syscfg.env.secure, self) if os.path.exists("/etc/init/libvirt-bin.conf"): cfo = configFileOps("/etc/init/libvirt-bin.conf", self) @@ -547,14 +542,7 @@ def setupLiveMigration(self): def config(self): try: - configureLibvirtConfig(self.syscfg.env.secure, self) - if os.path.exists("/etc/init/libvirt-bin.conf"): - cfo = configFileOps("/etc/init/libvirt-bin.conf", self) - cfo.replace_line("exec /usr/sbin/libvirtd","exec /usr/sbin/libvirtd -d -l") - else: - cfo = configFileOps("/etc/default/libvirt-bin", self) - cfo.replace_or_add_line("libvirtd_opts=","libvirtd_opts='-l -d'") - + self.setupLiveMigration() filename = "/etc/libvirt/qemu.conf" diff --git a/scripts/util/keystore-cert-import b/scripts/util/keystore-cert-import index d0f84cdfa8e5..96196d939021 100755 --- a/scripts/util/keystore-cert-import +++ b/scripts/util/keystore-cert-import @@ -81,7 +81,7 @@ mv -f "$NEW_KS_FILE" "$KS_FILE" # Secure libvirtd on cert import if [ -f "$LIBVIRTD_FILE" ]; then - mkdir -p /etc/pki/CA/ + mkdir -p /etc/pki/CA mkdir -p /etc/pki/libvirt/private ln -sf /etc/cloudstack/agent/cloud.ca.crt /etc/pki/CA/cacert.pem ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/clientcert.pem diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 5c5e4f8f81f7..d882c1fc9aea 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -841,7 +841,6 @@ def tearDownClass(cls): cleanup_resources(cls.apiclient, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) - return def setUp(self): self.apiclient = self.testClient.getApiClient() @@ -850,7 +849,6 @@ def setUp(self): self.updateConfiguration("ca.plugin.root.auth.strictness", "false") self.make_all_hosts_secure() - if self.hypervisor.lower() not in ["kvm"]: self.skipTest("Secured migration is not supported on other than KVM") @@ -858,11 +856,9 @@ def tearDown(self): self.make_all_hosts_secure() try: - #Clean up, terminate the created ISOs cleanup_resources(self.apiclient, self.cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) - return @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_01_secured_vm_migration(self): @@ -891,7 +887,6 @@ def test_01_secured_vm_migration(self): target_host = self.get_target_host(secured='true', virtualmachineid=self.vm_to_migrate.id) self.migrate_and_check(origin_host, target_host, proto='tls') - return @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_02_not_secured_vm_migration(self): @@ -924,8 +919,6 @@ def test_02_not_secured_vm_migration(self): self.migrate_and_check(origin_host, target_host, proto='tcp') - return - @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_03_secured_to_nonsecured_vm_migration(self): """Test destroy Virtual Machine @@ -954,9 +947,6 @@ def test_03_secured_to_nonsecured_vm_migration(self): pass else: self.fail("Migration succeed, instead it should fail") - - return - @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") def test_04_nonsecured_to_secured_vm_migration(self): """Test Non-secured VM Migration @@ -983,9 +973,8 @@ def test_04_nonsecured_to_secured_vm_migration(self): self.migrate_and_check(origin_host=non_secured_host, destination_host=secured_hosts[0], proto='tcp') except Exception: pass - else: self.fail("Migration succeed, instead it should fail") - - return + else: + self.fail("Migration succeed, instead it should fail") def get_target_host(self, secured, virtualmachineid): target_hosts = Host.listForMigration(self.apiclient, @@ -1002,10 +991,8 @@ def check_migration_protocol(self, protocol, host): .execute("grep -a Live /var/log/cloudstack/agent/agent.log | tail -1") if protocol not in resp[0]: - print 'It failed' cloudstackTestCase.fail(self, "Migration protocol was not as expected: '" + protocol + "\n" "Instead we got: " + resp[0]) - pass def make_unsecure_connection(self, host): SshClient(host.ipaddress, port=22, user=self.hostConfig["username"],passwd=self.hostConfig["password"])\ @@ -1040,8 +1027,6 @@ def make_all_hosts_secure(self): for host in hosts: self.check_connection(secured='true', host=host) - pass - def get_hosts(self): hosts = Host.list( @@ -1092,7 +1077,6 @@ def migrate_and_check(self, origin_host, destination_host, proto): vm_response = VirtualMachine.list(self.apiclient, id=self.vm_to_migrate.id)[0] self.assertEqual(vm_response.hostid, destination_host.id, "Check destination hostID of migrated VM") - pass def updateConfiguration(self, name, value): cmd = updateConfiguration.updateConfigurationCmd()