diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 054dfd572a0..7bab18f0fce 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -399,15 +399,16 @@ public void run(FlowTrigger trigger, Map data) { @Override public void run(MessageReply reply) { if (!reply.isSuccess()) { - logger.warn(String.format("failed to prepare vhost target env on host[%s]: %s", - h.getUuid(), reply.getError().getDetails())); - } else { - AgentResponse rsp = reply.castReply().toResponse(AgentResponse.class); - if (!rsp.isSuccess()) { - logger.warn(String.format("failed to prepare vhost target env on host[%s]: %s", - h.getUuid(), rsp.getError())); - } + trigger.fail(reply.getError()); + return; } + + AgentResponse rsp = reply.castReply().toResponse(AgentResponse.class); + if (!rsp.isSuccess()) { + trigger.fail(operr(ORG_ZSTACK_STORAGE_ZBS_10042, rsp.getError())); + return; + } + trigger.next(); } }); @@ -437,9 +438,7 @@ public void success(AgentResponse rsp) { @Override public void fail(ErrorCode errorCode) { - logger.warn(String.format("failed to deploy vhost target on host[%s]: %s", - h.getUuid(), errorCode.getDetails())); - trigger.next(); + trigger.fail(errorCode); } }); } diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java index a73d593ecc3..6e222dd4959 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorage.java @@ -62,6 +62,8 @@ import org.zstack.utils.logging.CLogger; import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -136,6 +138,11 @@ public void attachHook(String clusterUuid, Completion completion) { .select(PrimaryStorageOutputProtocolRefVO_.outputProtocol) .listValues(); + int totalHosts = hostInventories.size(); + double threshold = ExternalPrimaryStorageGlobalConfig.ATTACH_HOST_DEPLOY_FAILURE_RATIO_THRESHOLD.value(Double.class); + AtomicInteger failedCount = new AtomicInteger(); + Queue deployedHosts = new ConcurrentLinkedQueue<>(); + // the next attach or reconnect to overwrite. new While<>(hostInventories).each((host, compl) -> { node.deployClient(host, protocols, new Completion(compl) { @@ -146,6 +153,7 @@ public void success() { updateHostProtocolStatus(host.getUuid(), statuses, null, new NoErrorCompletion(compl) { @Override public void done() { + deployedHosts.add(host); compl.done(); } }); @@ -153,13 +161,19 @@ public void done() { @Override public void fail(ErrorCode errorCode) { + logger.warn(String.format("host[uuid:%s] failed to deploy client for primary storage[uuid:%s]", + host.getUuid(), self.getUuid())); Map statuses = new HashMap<>(); protocols.forEach(p -> statuses.put(p, PrimaryStorageHostStatus.Disconnected)); updateHostProtocolStatus(host.getUuid(), statuses, errorCode, new NoErrorCompletion(compl) { @Override public void done() { - compl.addError(errorCode); - compl.allDone(); + if ((double) failedCount.incrementAndGet() / totalHosts > threshold) { + compl.addError(errorCode); + compl.allDone(); + return; + } + compl.done(); } }); } @@ -172,7 +186,7 @@ public void done(ErrorCodeList errorCodeList) { return; } - activateHeartbeatVolumeForAttach(hostInventories.get(0), completion); + activateHeartbeatVolumeForAttach(deployedHosts.peek(), completion); } }); } diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java index 3a471d74589..33321dfbed6 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java @@ -18,6 +18,7 @@ import org.zstack.header.core.*; import org.zstack.header.core.workflow.*; import org.zstack.header.errorcode.ErrorCode; +import org.zstack.core.config.GlobalConfigException; import org.zstack.header.errorcode.ErrorCodeList; import org.zstack.header.errorcode.OperationFailureException; import org.zstack.header.exception.CloudRuntimeException; @@ -174,6 +175,20 @@ protected void run(Map tokens, ExternalPrimaryStorageCanonicalEvent.AddonInfoCha } }); + ExternalPrimaryStorageGlobalConfig.ATTACH_HOST_DEPLOY_FAILURE_RATIO_THRESHOLD.installValidateExtension((category, name, oldValue, newValue) -> { + double v; + try { + v = Double.parseDouble(newValue); + } catch (NumberFormatException e) { + throw new GlobalConfigException(String.format( + "the value[%s] of %s.%s is not a valid number", newValue, category, name), e); + } + if (v <= 0 || v > 1) { + throw new GlobalConfigException(String.format( + "the value[%s] of %s.%s must be greater than 0 and not greater than 1", newValue, category, name)); + } + }); + return true; } diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageGlobalConfig.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageGlobalConfig.java index d0225c19797..56b3e7f9515 100644 --- a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageGlobalConfig.java +++ b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageGlobalConfig.java @@ -15,4 +15,10 @@ public class ExternalPrimaryStorageGlobalConfig { @GlobalConfigDef(defaultValue = "iSCSI", description = "image export protocol of external primary storage") @BindResourceConfig({PrimaryStorageVO.class}) public static GlobalConfig IMAGE_EXPORT_PROTOCOL = new GlobalConfig(CATEGORY, "image.export.protocol"); + + @GlobalConfigValidation() + @GlobalConfigDef(defaultValue = "0.3", type = Double.class, description = "when attaching external primary storage to a cluster, " + + "if the ratio of hosts that fail to deploy the storage client reaches this threshold, the attach fails; " + + "otherwise it succeeds and the failed hosts recover through periodic ping self-heal") + public static GlobalConfig ATTACH_HOST_DEPLOY_FAILURE_RATIO_THRESHOLD = new GlobalConfig(CATEGORY, "attach.hostDeployFailureRatioThreshold"); } \ No newline at end of file diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostAttachDeployFailureCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostAttachDeployFailureCase.groovy new file mode 100644 index 00000000000..a275d3b868e --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostAttachDeployFailureCase.groovy @@ -0,0 +1,228 @@ +package org.zstack.test.integration.storage.primary.addon.zbs + +import org.springframework.http.HttpEntity +import org.zstack.core.db.Q +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefVO +import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageHostProtocolRefVO_ +import org.zstack.header.storage.backup.UploadImageToRemoteTargetMsg +import org.zstack.header.storage.backup.UploadImageToRemoteTargetReply +import org.zstack.header.storage.primary.PrimaryStorageHostStatus +import org.zstack.core.cloudbus.CloudBus +import org.zstack.header.volume.VolumeProtocol +import org.zstack.sdk.ClusterInventory +import org.zstack.sdk.HostInventory +import org.zstack.sdk.PrimaryStorageInventory +import org.zstack.storage.zbs.ZbsConstants +import org.zstack.storage.zbs.ZbsStorageController +import org.zstack.test.integration.storage.StorageTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import org.zstack.utils.data.SizeUnit +import org.zstack.utils.gson.JSONObjectUtil + +import java.util.concurrent.atomic.AtomicReference + +class ZbsVhostAttachDeployFailureCase extends SubCase { + EnvSpec env + CloudBus bus + PrimaryStorageInventory ps + ClusterInventory cluster + HostInventory kvm1, kvm2 + + @Override + void clean() { + env.delete() + } + + @Override + void setup() { + useSpring(StorageTest.springSpec) + } + + @Override + void environment() { + env = makeEnv { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(8) + cpu = 4 + } + + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "127.0.0.2" + + image { + name = "image" + url = "http://zstack.org/download/test.qcow2" + size = SizeUnit.GIGABYTE.toByte(1) + virtio = true + } + } + + zone { + name = "zone" + + cluster { + name = "cluster" + hypervisorType = "KVM" + + kvm { + name = "kvm1" + managementIp = "127.0.0.1" + username = "root" + password = "password" + } + + kvm { + name = "kvm2" + managementIp = "127.0.0.2" + username = "root" + password = "password" + } + + attachL2Network("l2") + } + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "l3" + ip { + startIp = "192.168.100.10" + endIp = "192.168.100.100" + netmask = "255.255.255.0" + gateway = "192.168.100.1" + } + } + } + + externalPrimaryStorage { + name = "zbs-vhost" + identity = "zbs" + defaultOutputProtocol = "Vhost" + config = "{\"mdsUrls\":[\"root:password@127.0.1.1\",\"root:password@127.0.1.2\",\"root:password@127.0.1.3\"],\"logicalPoolName\":\"lpool1\"}" + url = "zbs" + } + + attachBackupStorage("sftp") + } + } + } + + @Override + void test() { + env.create { + ps = env.inventoryByName("zbs-vhost") as PrimaryStorageInventory + cluster = env.inventoryByName("cluster") as ClusterInventory + kvm1 = env.inventoryByName("kvm1") as HostInventory + kvm2 = env.inventoryByName("kvm2") as HostInventory + bus = bean(CloudBus.class) + + testAttachFailsWhenDeployFailureRatioReachesThreshold() + testAttachSucceedsBelowThresholdThenSelfHeals() + } + } + + // one of two hosts fails vhost deploy -> 50% > default 0.3 threshold -> attach fails + void testAttachFailsWhenDeployFailureRatioReachesThreshold() { + AtomicReference failHostIp = new AtomicReference<>("127.0.0.2") + registerBaseStubs(failHostIp) + + expect(AssertionError.class) { + attachPrimaryStorageToCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + } + } + + // raise threshold to 0.6 so 50% < 0.6 -> attach succeeds; failed host stays Disconnected then self-heals + void testAttachSucceedsBelowThresholdThenSelfHeals() { + updateGlobalConfig { + category = "externalPrimaryStorage" + name = "attach.hostDeployFailureRatioThreshold" + value = 0.6 + } + + AtomicReference failHostIp = new AtomicReference<>("127.0.0.2") + registerBaseStubs(failHostIp) + + attachPrimaryStorageToCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, kvm2.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.Vhost.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Disconnected) + .isExists() : "failed host must be recorded Disconnected" + + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, kvm1.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.Vhost.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected) + .isExists() : "healthy host must be Connected" + + failHostIp.set(null) + updateGlobalConfig { + category = "host" + name = "ping.interval" + value = 1 + } + + retryInSecs(30) { + assert Q.New(ExternalPrimaryStorageHostProtocolRefVO.class) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.primaryStorageUuid, ps.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.hostUuid, kvm2.uuid) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.protocol, VolumeProtocol.Vhost.toString()) + .eq(ExternalPrimaryStorageHostProtocolRefVO_.status, PrimaryStorageHostStatus.Connected) + .isExists() : "periodic ping did not self-heal the previously failed host" + } + + detachPrimaryStorageFromCluster { + primaryStorageUuid = ps.uuid + clusterUuid = cluster.uuid + } + } + + void registerBaseStubs(AtomicReference failHostIp) { + env.message(UploadImageToRemoteTargetMsg.class) { UploadImageToRemoteTargetMsg msg, CloudBus b -> + b.reply(msg, new UploadImageToRemoteTargetReply()) + } + + env.simulator(ZbsStorageController.PREPARE_VHOST_TARGET_ENV_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.PrepareVhostTargetEnvCmd.class) + return new ZbsStorageController.AgentResponse() + } + env.simulator(ZbsStorageController.DEPLOY_VHOST_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.DeployVhostCmd.class) + if (cmd.hostIp == failHostIp.get()) { + throw new RuntimeException("vhost deploy fails on purpose for host ${cmd.hostIp}") + } + return new ZbsStorageController.AgentResponse() + } + env.simulator(ZbsStorageController.VHOST_TARGET_HEALTH_PATH) { HttpEntity e, EnvSpec spec -> + def rsp = new ZbsStorageController.VhostTargetHealthRsp() + rsp.targetRunning = true + return rsp + } + env.afterSimulator(ZbsStorageController.CREATE_VOLUME_PATH) { rsp, HttpEntity e -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.CreateVolumeCmd) + if (cmd.volume == ZbsConstants.ZBS_HEARTBEAT_VOLUME_NAME) { + def vrsp = new ZbsStorageController.CreateVolumeRsp() + vrsp.installPath = "zbs://${cmd.logicalPool}/${cmd.volume}".toString() + return vrsp + } + return rsp + } + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy index 941152f237b..af9b34730f0 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsVhostVolumeCase.groovy @@ -624,6 +624,12 @@ class ZbsVhostVolumeCase extends SubCase { value = 1 } + updateGlobalConfig { + category = "externalPrimaryStorage" + name = "attach.hostDeployFailureRatioThreshold" + value = 1 + } + attachPrimaryStorageToCluster { primaryStorageUuid = ps.uuid clusterUuid = cluster.uuid diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 54cdb8d4cf6..db8d5cdbe9a 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -6632,6 +6632,8 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10040 = "ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10040"; + public static final String ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10041 = "ORG_ZSTACK_STORAGE_ADDON_PRIMARY_10041"; + public static final String ORG_ZSTACK_NETWORK_HOSTNETWORKINTERFACE_LLDP_10000 = "ORG_ZSTACK_NETWORK_HOSTNETWORKINTERFACE_LLDP_10000"; public static final String ORG_ZSTACK_NETWORK_HOSTNETWORKINTERFACE_LLDP_10001 = "ORG_ZSTACK_NETWORK_HOSTNETWORKINTERFACE_LLDP_10001"; @@ -14777,6 +14779,8 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_STORAGE_ZBS_10041 = "ORG_ZSTACK_STORAGE_ZBS_10041"; + public static final String ORG_ZSTACK_STORAGE_ZBS_10042 = "ORG_ZSTACK_STORAGE_ZBS_10042"; + public static final String ORG_ZSTACK_TEST_10000 = "ORG_ZSTACK_TEST_10000"; public static final String ORG_ZSTACK_TEST_10001 = "ORG_ZSTACK_TEST_10001";