CLOUDSTACK-8628: Send an alert when fencing a KVM host failed Also change the logging a bit so that you get useful logs when not running on DEBUG level
Signed-off-by: Wido den Hollander <w...@widodh.nl> This closes #580 Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/fbe3b04a Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/fbe3b04a Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/fbe3b04a Branch: refs/heads/reporter Commit: fbe3b04a43ff7380a6874a2c331f372aa59ccf3d Parents: d1f76a2 Author: Wido den Hollander <w...@widodh.nl> Authored: Thu Jul 16 16:15:20 2015 +0200 Committer: Wido den Hollander <w...@widodh.nl> Committed: Fri Jul 17 12:33:11 2015 +0200 ---------------------------------------------------------------------- server/src/com/cloud/ha/KVMFencer.java | 26 +++++++++++++++--------- server/test/com/cloud/ha/KVMFencerTest.java | 22 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/fbe3b04a/server/src/com/cloud/ha/KVMFencer.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/ha/KVMFencer.java b/server/src/com/cloud/ha/KVMFencer.java index 7392dff..b5834ef 100644 --- a/server/src/com/cloud/ha/KVMFencer.java +++ b/server/src/com/cloud/ha/KVMFencer.java @@ -26,6 +26,7 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; +import com.cloud.alert.AlertManager; import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; @@ -48,6 +49,8 @@ public class KVMFencer extends AdapterBase implements FenceBuilder { @Inject AgentManager _agentMgr; @Inject + AlertManager _alertMgr; + @Inject ResourceManager _resourceMgr; @Override @@ -75,18 +78,22 @@ public class KVMFencer extends AdapterBase implements FenceBuilder { @Override public Boolean fenceOff(VirtualMachine vm, Host host) { if (host.getHypervisorType() != HypervisorType.KVM && host.getHypervisorType() != HypervisorType.LXC) { - s_logger.debug("Don't know how to fence non kvm hosts " + host.getHypervisorType()); + s_logger.warn("Don't know how to fence non kvm hosts " + host.getHypervisorType()); return null; } List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId()); FenceCommand fence = new FenceCommand(vm, host); + int i = 0; for (HostVO h : hosts) { if (h.getHypervisorType() == HypervisorType.KVM || h.getHypervisorType() == HypervisorType.LXC) { if (h.getStatus() != Status.Up) { continue; } + + i++; + if (h.getId() == host.getId()) { continue; } @@ -94,14 +101,10 @@ public class KVMFencer extends AdapterBase implements FenceBuilder { try { answer = (FenceAnswer)_agentMgr.send(h.getId(), fence); } catch (AgentUnavailableException e) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable"); - } + s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable"); continue; } catch (OperationTimedoutException e) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable"); - } + s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable"); continue; } if (answer != null && answer.getResult()) { @@ -110,9 +113,12 @@ public class KVMFencer extends AdapterBase implements FenceBuilder { } } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Unable to fence off " + vm.toString() + " on " + host.toString()); - } + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), + "Unable to fence off host: " + host.getId(), + "Fencing off host " + host.getId() + " did not succeed after asking " + i + " hosts. " + + "Check Agent logs for more information."); + + s_logger.error("Unable to fence off " + vm.toString() + " on " + host.toString()); return false; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/fbe3b04a/server/test/com/cloud/ha/KVMFencerTest.java ---------------------------------------------------------------------- diff --git a/server/test/com/cloud/ha/KVMFencerTest.java b/server/test/com/cloud/ha/KVMFencerTest.java index cdd13b6..da120af 100644 --- a/server/test/com/cloud/ha/KVMFencerTest.java +++ b/server/test/com/cloud/ha/KVMFencerTest.java @@ -31,6 +31,7 @@ import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; import com.cloud.agent.AgentManager; +import com.cloud.alert.AlertManager; import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.exception.AgentUnavailableException; @@ -50,6 +51,8 @@ public class KVMFencerTest { @Mock AgentManager agentManager; @Mock + AlertManager alertMgr; + @Mock ResourceManager resourceManager; KVMFencer fencer; @@ -58,6 +61,7 @@ public class KVMFencerTest { public void setup() { fencer = new KVMFencer(); fencer._agentMgr = agentManager; + fencer._alertMgr = alertMgr; fencer._hostDao = hostDao; fencer._resourceMgr = resourceManager; } @@ -67,6 +71,8 @@ public class KVMFencerTest { HostVO host = Mockito.mock(HostVO.class); Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(host.getDataCenterId()).thenReturn(1l); + Mockito.when(host.getPodId()).thenReturn(1l); Mockito.when(host.getStatus()).thenReturn(Status.Up); Mockito.when(host.getId()).thenReturn(1l); VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class); @@ -80,6 +86,8 @@ public class KVMFencerTest { HostVO host = Mockito.mock(HostVO.class); Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(host.getDataCenterId()).thenReturn(1l); + Mockito.when(host.getPodId()).thenReturn(1l); Mockito.when(host.getStatus()).thenReturn(Status.Down); Mockito.when(host.getId()).thenReturn(1l); VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class); @@ -94,12 +102,16 @@ public class KVMFencerTest { Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getDataCenterId()).thenReturn(1l); + Mockito.when(host.getPodId()).thenReturn(1l); Mockito.when(host.getId()).thenReturn(1l); HostVO secondHost = Mockito.mock(HostVO.class); Mockito.when(secondHost.getClusterId()).thenReturn(1l); Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM); Mockito.when(secondHost.getStatus()).thenReturn(Status.Up); + Mockito.when(secondHost.getDataCenterId()).thenReturn(1l); + Mockito.when(secondHost.getPodId()).thenReturn(1l); Mockito.when(host.getId()).thenReturn(2l); VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class); @@ -118,12 +130,16 @@ public class KVMFencerTest { Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getDataCenterId()).thenReturn(1l); + Mockito.when(host.getPodId()).thenReturn(1l); Mockito.when(host.getId()).thenReturn(1l); HostVO secondHost = Mockito.mock(HostVO.class); Mockito.when(secondHost.getClusterId()).thenReturn(1l); Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM); Mockito.when(secondHost.getStatus()).thenReturn(Status.Up); + Mockito.when(secondHost.getDataCenterId()).thenReturn(1l); + Mockito.when(secondHost.getPodId()).thenReturn(1l); Mockito.when(host.getId()).thenReturn(2l); VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class); @@ -141,12 +157,16 @@ public class KVMFencerTest { Mockito.when(host.getClusterId()).thenReturn(1l); Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getDataCenterId()).thenReturn(1l); + Mockito.when(host.getPodId()).thenReturn(1l); Mockito.when(host.getId()).thenReturn(1l); HostVO secondHost = Mockito.mock(HostVO.class); Mockito.when(secondHost.getClusterId()).thenReturn(1l); Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM); Mockito.when(secondHost.getStatus()).thenReturn(Status.Up); + Mockito.when(secondHost.getDataCenterId()).thenReturn(1l); + Mockito.when(secondHost.getPodId()).thenReturn(1l); Mockito.when(host.getId()).thenReturn(2l); VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class); @@ -165,6 +185,8 @@ public class KVMFencerTest { Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.Any); Mockito.when(host.getStatus()).thenReturn(Status.Down); Mockito.when(host.getId()).thenReturn(1l); + Mockito.when(host.getDataCenterId()).thenReturn(1l); + Mockito.when(host.getPodId()).thenReturn(1l); VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class); Mockito.when(resourceManager.listAllHostsInCluster(1l)).thenReturn(Collections.singletonList(host));
