From 6a5b8ca2e017df5e80985cf1e2b98580ef2e894e Mon Sep 17 00:00:00 2001 From: "tao.gan" Date: Fri, 9 Jan 2026 17:20:13 +0800 Subject: [PATCH] [compute]: 100644 header/src/main/java/org/zstack/header/vm/APIReimageVmInstanceEvent.java mode change 100755 => 100644 header/src/main/java/org/zstack/header/vm/APIReimageVmInstanceEventDoc_zh_cn.groovy create mode 100644 header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataMsg.java create mode 100644 header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnHypervisorMsg.java create mode 100644 header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnHypervisorReply.java create mode 100644 header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnPrimaryStorageMsg.java create mode 100644 header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnPrimaryStorageReply.java create mode 100644 header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataReply.java create mode 100644 header/src/main/java/org/zstack/header/vm/VmInstanceMetadataDTO.java create mode 100644 header/src/main/java/org/zstack/header/vm/VmMetadata.java create mode 100644 sdk/src/main/java/org/zstack/sdk/RegisterVmInstanceAction.java create mode 100644 sdk/src/main/java/org/zstack/sdk/RegisterVmInstanceResult.java diff --git a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java index bd79900c13c..fe60c278eba 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmGlobalConfig.java @@ -133,4 +133,10 @@ public class VmGlobalConfig { @GlobalConfigValidation(validValues = {"None", "AuthenticAMD"}) @BindResourceConfig(value = {VmInstanceVO.class}) public static GlobalConfig VM_CPUID_VENDOR = new GlobalConfig(CATEGORY, "vm.cpuid.vendor"); + + @GlobalConfigValidation(numberGreaterThan = 1) + public static GlobalConfig GC_INTERVAL = new GlobalConfig(CATEGORY, "deletion.gcInterval"); + + @GlobalConfigValidation(validValues = {"true", "false"}) + public static GlobalConfig VM_METADATA = new GlobalConfig(CATEGORY, "vm.metadata"); } diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java index c17cf5d5179..0ce04419dcb 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceApiInterceptor.java @@ -21,6 +21,7 @@ import org.zstack.header.message.APIMessage; import org.zstack.header.network.l2.*; import org.zstack.header.network.l3.*; +import org.zstack.header.storage.primary.APIRegisterVmInstanceMsg; import org.zstack.header.storage.primary.PrimaryStorageClusterRefVO; import org.zstack.header.storage.primary.PrimaryStorageClusterRefVO_; import org.zstack.header.storage.snapshot.VolumeSnapshotVO; diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java index e31bc001218..391323081ae 100755 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceBase.java @@ -45,6 +45,15 @@ import org.zstack.header.message.*; import org.zstack.header.network.l3.*; import org.zstack.header.storage.primary.*; +import org.zstack.header.storage.snapshot.*; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO_; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO_; +import org.zstack.header.tag.SystemTagVO; +import org.zstack.header.tag.SystemTagVO_; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import org.zstack.header.vm.*; import org.zstack.header.vm.ChangeVmMetaDataMsg.AtomicHostUuid; import org.zstack.header.vm.ChangeVmMetaDataMsg.AtomicVmState; @@ -66,23 +75,19 @@ import org.zstack.network.l3.L3NetworkManager; import org.zstack.network.service.DnsUtils; import org.zstack.network.service.NetworkServiceManager; -import org.zstack.resourceconfig.ResourceConfig; -import org.zstack.resourceconfig.ResourceConfigFacade; +import org.zstack.resourceconfig.*; import org.zstack.tag.SystemTagCreator; import org.zstack.tag.SystemTagUtils; import org.zstack.tag.TagManager; -import org.zstack.utils.CollectionUtils; -import org.zstack.utils.ExceptionDSL; -import org.zstack.utils.ObjectUtils; -import org.zstack.utils.Utils; +import org.zstack.utils.*; import org.zstack.utils.function.ForEachFunction; import org.zstack.utils.function.Function; import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.logging.CLogger; -import org.zstack.utils.network.NicIpAddressInfo; import org.zstack.utils.network.IPv6Constants; import org.zstack.utils.network.IPv6NetworkUtils; import org.zstack.utils.network.NetworkUtils; +import org.zstack.utils.network.NicIpAddressInfo; import javax.persistence.PersistenceException; import javax.persistence.Tuple; @@ -90,6 +95,7 @@ import java.sql.Timestamp; import java.time.LocalDateTime; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static java.util.Arrays.asList; @@ -140,6 +146,8 @@ public class VmInstanceBase extends AbstractVmInstance { private VmInstanceResourceMetadataManager vidm; @Autowired private NetworkServiceManager nwServiceMgr; + @Autowired + private ResourceDestinationMaker destMaker; protected VmInstanceVO self; protected VmInstanceVO originalCopy; @@ -533,6 +541,8 @@ protected void handleLocalMessage(Message msg) { handle((CancelFlattenVmInstanceMsg) msg); } else if (msg instanceof KvmReportVmShutdownEventMsg) { handle((KvmReportVmShutdownEventMsg) msg); + } else if (msg instanceof UpdateVmInstanceMetadataMsg) { + handle((UpdateVmInstanceMetadataMsg) msg); } else { VmInstanceBaseExtensionFactory ext = vmMgr.getVmInstanceBaseExtensionFactory(msg); if (ext != null) { @@ -9369,5 +9379,139 @@ public void run(MessageReply reply) { } }); } + + /** + * 处理元数据更新消息。 + * + *

Layer 2 串行化保证：通过 ChainTask 确保同一 VM 的元数据更新串行执行。 + * 该消息由 hash 环路由，同一 VM 必定到达同一 MN，因此此处的 ChainTask + * 是跨 MN 串行的全局唯一汇聚点。

+ * + *

失败路径不创建新 GC，直接返回错误 reply，由 GC 端的 onUpdateFail() 统一处理重试。

+ */ + private void handle(UpdateVmInstanceMetadataMsg msg) { + thdf.chainSubmit(new ChainTask(msg) { + @Override + public String getSyncSignature() { + return String.format("handle-update-vm-%s-metadata", msg.getUuid()); + } + + @Override + public void run(SyncTaskChain chain) { + doHandleUpdateVmInstanceMetadata(msg); + chain.next(); + } + + @Override + public String getName() { + return String.format("handle-update-vm-%s-metadata-task", msg.getUuid()); + } + }); + } + + private void doHandleUpdateVmInstanceMetadata(UpdateVmInstanceMetadataMsg msg) { + Tuple tuple = Q.New(VolumeVO.class).select(VolumeVO_.primaryStorageUuid, VolumeVO_.uuid) + .eq(VolumeVO_.vmInstanceUuid, msg.getUuid()).eq(VolumeVO_.type, VolumeType.Root).findTuple(); + String primaryStorageUuid = tuple.get(0, String.class); + String rootVolumeUuid = tuple.get(1, String.class); + + UpdateVmInstanceMetadataOnPrimaryStorageMsg umsg = new UpdateVmInstanceMetadataOnPrimaryStorageMsg(); + umsg.setMetadata(buildVmInstanceMetadata(msg.getUuid())); + umsg.setPrimaryStorageUuid(primaryStorageUuid); + umsg.setRootVolumeUuid(rootVolumeUuid); + bus.makeLocalServiceId(umsg, PrimaryStorageConstant.SERVICE_ID); + bus.send(umsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + UpdateVmInstanceMetadataOnPrimaryStorageReply reply = new UpdateVmInstanceMetadataOnPrimaryStorageReply(); + + if (!r.isSuccess()) { + // 失败：直接返回错误 reply，不创建新 GC（避免滚雪球膨胀） + // GC 端收到失败 reply 后由 onUpdateFail() 统一走指数退避重试 + reply.setError(Platform.operr("failed to update vm[uuid=%s] metadata on primary storage", + msg.getUuid()).withCause(r.getError())); + } + bus.reply(msg, reply); + } + }); + } + + private String buildVmInstanceMetadata(String vmInstanceUuid) { + VmInstanceMetadataDTO dto = new VmInstanceMetadataDTO(); + + // ── VM 本体 ── + VmInstanceVO vm = Q.New(VmInstanceVO.class).eq(VmInstanceVO_.uuid, vmInstanceUuid).find(); + dto.vm = buildResourceMetadata(vm.getUuid(), vm); + + // ── 云盘（挂载的 + 已卸载但 lastVmInstanceUuid 指向本 VM 的） ── + List volumes = new ArrayList<>(); + volumes.addAll(Q.New(VolumeVO.class).eq(VolumeVO_.vmInstanceUuid, vmInstanceUuid).list()); + volumes.addAll(Q.New(VolumeVO.class).isNull(VolumeVO_.vmInstanceUuid) + .eq(VolumeVO_.lastVmInstanceUuid, vmInstanceUuid).list()); + volumes.forEach(v -> dto.volumes.add(buildResourceMetadata(v.getUuid(), v))); + + // ── 网卡 ── + List nics = Q.New(VmNicVO.class).eq(VmNicVO_.vmInstanceUuid, vmInstanceUuid).list(); + nics.forEach(n -> dto.nics.add(buildResourceMetadata(n.getUuid(), n))); + + // ── 快照 ── + List volumeUuids = volumes.stream().map(VolumeVO::getUuid).collect(Collectors.toList()); + if (!volumeUuids.isEmpty()) { + Q.New(VolumeSnapshotVO.class).in(VolumeSnapshotVO_.volumeUuid, volumeUuids).list() + .forEach(s -> dto.snapshots + .computeIfAbsent(s.getVolumeUuid(), k -> new ArrayList<>()) + .add(JSONObjectUtil.toJsonString(s))); + } + + // ── 快照组 ── + List groups = Q.New(VolumeSnapshotGroupVO.class) + .eq(VolumeSnapshotGroupVO_.vmInstanceUuid, vmInstanceUuid).list(); + dto.snapshotGroups = groups.stream() + .map(JSONObjectUtil::toJsonString).collect(Collectors.toList()); + + List groupUuids = groups.stream() + .map(VolumeSnapshotGroupVO::getUuid).collect(Collectors.toList()); + if (!groupUuids.isEmpty()) { + dto.snapshotGroupRefs = Q.New(VolumeSnapshotGroupRefVO.class) + .in(VolumeSnapshotGroupRefVO_.volumeSnapshotGroupUuid, groupUuids).list() + .stream().map(JSONObjectUtil::toJsonString).collect(Collectors.toList()); + } + + return JSONObjectUtil.toJsonString(dto); + } + + /** + * 构建单个资源的 {@link VmInstanceMetadataDTO.ResourceMetadata}。 + * + *

VO 全量 JSON 明文存储；SystemTagVO 和 ResourceConfigVO 整体列表序列化为 JSON 数组后 + * 一次性 Base64 编码，以保护可能包含的密码、密钥等敏感信息。

+ * + * @param resourceUuid 资源 UUID + * @param vo 资源 VO 对象（VmInstanceVO / VolumeVO / VmNicVO） + * @return 填充完毕的 ResourceMetadata + */ + private VmInstanceMetadataDTO.ResourceMetadata buildResourceMetadata(String resourceUuid, Object vo) { + VmInstanceMetadataDTO.ResourceMetadata meta = new VmInstanceMetadataDTO.ResourceMetadata(); + meta.resourceUuid = resourceUuid; + meta.vo = JSONObjectUtil.toJsonString(vo); + + // SystemTagVO: 全部 → JSON 数组 → Base64 + List tagVOs = Q.New(SystemTagVO.class) + .eq(SystemTagVO_.resourceUuid, resourceUuid).list(); + List tagJsons = tagVOs.stream() + .map(JSONObjectUtil::toJsonString).collect(Collectors.toList()); + meta.systemTags = Base64.getEncoder().encodeToString( + JSONObjectUtil.toJsonString(tagJsons).getBytes(StandardCharsets.UTF_8)); + + // ResourceConfigVO: 全部 → JSON 数组 → Base64 + List cfgVOs = Q.New(ResourceConfigVO.class) + .eq(ResourceConfigVO_.resourceUuid, resourceUuid).list(); + List cfgJsons = cfgVOs.stream() + .map(JSONObjectUtil::toJsonString).collect(Collectors.toList()); + meta.resourceConfigs = Base64.getEncoder().encodeToString( + JSONObjectUtil.toJsonString(cfgJsons).getBytes(StandardCharsets.UTF_8)); + + return meta; + } } diff --git a/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java b/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java index 33afa043278..1f916560f58 100644 --- a/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java +++ b/compute/src/main/java/org/zstack/compute/vm/VmInstanceUtils.java @@ -2,22 +2,29 @@ import org.apache.commons.collections.CollectionUtils; import org.zstack.core.Platform; +import org.zstack.core.db.Q; import org.zstack.header.configuration.InstanceOfferingInventory; import org.zstack.header.errorcode.OperationFailureException; -import org.zstack.header.vm.APIChangeInstanceOfferingMsg; -import org.zstack.header.vm.APICreateVmInstanceMsg; -import org.zstack.header.vm.CreateVmInstanceMsg; -import org.zstack.header.vm.DiskAO; -import org.zstack.header.vm.UpdateVmInstanceMsg; -import org.zstack.header.vm.UpdateVmInstanceSpec; -import org.zstack.header.vm.VmInstanceVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO; +import org.zstack.header.storage.snapshot.VolumeSnapshotVO_; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO_; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO_; +import org.zstack.header.tag.SystemTagVO; +import org.zstack.header.tag.SystemTagVO_; +import org.zstack.header.vm.*; +import org.zstack.header.volume.VolumeVO; +import org.zstack.header.volume.VolumeVO_; +import org.zstack.resourceconfig.ResourceConfigVO; +import org.zstack.resourceconfig.ResourceConfigVO_; import org.zstack.tag.SystemTagUtils; +import org.zstack.utils.function.ForEachFunction; +import org.zstack.utils.gson.JSONObjectUtil; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; +import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; import static org.zstack.compute.vm.VmSystemTags.PRIMARY_STORAGE_UUID_FOR_DATA_VOLUME; diff --git a/compute/src/main/java/org/zstack/compute/vm/metadata/MetadataImpact.java b/compute/src/main/java/org/zstack/compute/vm/metadata/MetadataImpact.java new file mode 100644 index 00000000000..627f82fb123 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/metadata/MetadataImpact.java @@ -0,0 +1,204 @@ +package org.zstack.compute.vm.metadata; + +import org.codehaus.plexus.component.annotations.Component; +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.db.DatabaseFacade; +import org.zstack.header.message.APIMessage; +import org.zstack.header.vm.VmInstanceMessage; +import org.zstack.header.volume.APICreateVolumeSnapshotGroupMsg; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.Collections; +import java.util.List; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface MetadataImpact { + /** + * 影响级别 + */ + MetadataImpactLevel value() default MetadataImpactLevel.CONFIG; + + /** + * 失败时是否也更新 + */ + boolean updateOnFailure() default false; + + /** + * 自定义 VmUuid 解析器（可选） + * 不指定时使用注册表中的默认解析器 + */ + Class resolver() default VmUuidResolver.class; + + /** + * VM UUID 解析器接口 + */ + public interface VmUuidResolver { + /** + * 从 API 消息解析出需要更新元数据的 vmUuid 列表 + * + * @param msg API 消息 + * @return vmUuid 列表（可能为空，可能多个） + */ + List resolve(APIMessage msg); + } + + /** + * 默认解析器：从 VmInstanceMessage 接口获取 + */ + public class DefaultVmUuidResolver implements VmUuidResolver { + @Override + public List resolve(APIMessage msg) { + if (msg instanceof VmInstanceMessage) { + String vmUuid = ((VmInstanceMessage) msg).getVmInstanceUuid(); + if (vmUuid != null) { + return Collections.singletonList(vmUuid); + } + } + return Collections.emptyList(); + } + } + + /** + * Volume 相关 API 解析器 + */ + public class VolumeBasedVmUuidResolver implements VmUuidResolver { + @Autowired + private DatabaseFacade dbf; + + @Override + public List resolve(APIMessage msg) { + List volumeUuids = extractVolumeUuids(msg); + if (volumeUuids.isEmpty()) { + return Collections.emptyList(); + } + + // 查询这些 Volume 关联的 VM + List vmUuids = SQL.New( + "SELECT DISTINCT v.vmInstanceUuid FROM VolumeVO v " + + "WHERE v.uuid IN (:uuids) AND v.vmInstanceUuid IS NOT NULL", String.class) + .param("uuids", volumeUuids) + .list(); + + return vmUuids; + } + + private List extractVolumeUuids(APIMessage msg) { + if (msg instanceof APICreateVolumesSnapshotMsg) { + return ((APICreateVolumesSnapshotMsg) msg).getVolumeUuids(); + } else if (msg instanceof APICreateVolumeSnapshotGroupMsg) { + return Collections.singletonList( + ((APICreateVolumeSnapshotGroupMsg) msg).getRootVolumeUuid()); + } else if (msg instanceof VolumeMessage) { + return Collections.singletonList(((VolumeMessage) msg).getVolumeUuid()); + } + return Collections.emptyList(); + } + } + + /** + * SystemTag/ResourceConfig 动态解析器 + */ + public class ResourceBasedVmUuidResolver implements VmUuidResolver { + @Autowired + private DatabaseFacade dbf; + + @Override + public List resolve(APIMessage msg) { + String resourceType = null; + String resourceUuid = null; + + if (msg instanceof APIAbstractCreateTagMsg) { + resourceType = ((APIAbstractCreateTagMsg) msg).getResourceType(); + resourceUuid = ((APIAbstractCreateTagMsg) msg).getResourceUuid(); + } else if (msg instanceof APIDeleteTagMsg) { + // 需要先查询 Tag 获取 resourceType 和 resourceUuid + TagVO tag = dbf.findByUuid(((APIDeleteTagMsg) msg).getUuid(), TagVO.class); + if (tag != null) { + resourceType = tag.getResourceType(); + resourceUuid = tag.getResourceUuid(); + } + } + + if (resourceType == null || resourceUuid == null) { + return Collections.emptyList(); + } + + return resolveByResourceType(resourceType, resourceUuid); + } + + private List resolveByResourceType(String resourceType, String resourceUuid) { + // VmInstanceVO 直接返回 + if ("VmInstanceVO".equals(resourceType)) { + return Collections.singletonList(resourceUuid); + } + + // VolumeVO 查询关联的 VM + if ("VolumeVO".equals(resourceType)) { + VolumeVO vol = dbf.findByUuid(resourceUuid, VolumeVO.class); + if (vol != null && vol.getVmInstanceUuid() != null) { + return Collections.singletonList(vol.getVmInstanceUuid()); + } + return Collections.emptyList(); + } + + // VmNicVO 查询关联的 VM + if ("VmNicVO".equals(resourceType)) { + VmNicVO nic = dbf.findByUuid(resourceUuid, VmNicVO.class); + if (nic != null && nic.getVmInstanceUuid() != null) { + return Collections.singletonList(nic.getVmInstanceUuid()); + } + return Collections.emptyList(); + } + + // VolumeSnapshotVO 需要查询 Volume 再查询 VM + if ("VolumeSnapshotVO".equals(resourceType)) { + VolumeSnapshotVO snap = dbf.findByUuid(resourceUuid, VolumeSnapshotVO.class); + if (snap != null && snap.getVolumeUuid() != null) { + VolumeVO vol = dbf.findByUuid(snap.getVolumeUuid(), VolumeVO.class); + if (vol != null && vol.getVmInstanceUuid() != null) { + return Collections.singletonList(vol.getVmInstanceUuid()); + } + } + return Collections.emptyList(); + } + + // 其他类型不影响 VM 元数据 + return Collections.emptyList(); + } + } + + /** + * 解析器注册表 + */ + @Component + public class VmUuidResolverRegistry { + private Map, VmUuidResolver> resolvers = new HashMap<>(); + private VmUuidResolver defaultResolver = new DefaultVmUuidResolver(); + + @PostConstruct + public void init() { + // 注册特定 API 的解析器 + VolumeBasedVmUuidResolver volumeResolver = new VolumeBasedVmUuidResolver(); + resolvers.put(APICreateVolumesSnapshotMsg.class, volumeResolver); + resolvers.put(APICreateVolumeSnapshotGroupMsg.class, volumeResolver); + resolvers.put(APIAttachDataVolumeToVmMsg.class, volumeResolver); + resolvers.put(APIDetachDataVolumeFromVmMsg.class, volumeResolver); + + ResourceBasedVmUuidResolver resourceResolver = new ResourceBasedVmUuidResolver(); + resolvers.put(APICreateSystemTagMsg.class, resourceResolver); + resolvers.put(APIDeleteTagMsg.class, resourceResolver); + resolvers.put(APIUpdateSystemTagMsg.class, resourceResolver); + resolvers.put(APIUpdateResourceConfigMsg.class, resourceResolver); + resolvers.put(APIDeleteResourceConfigMsg.class, resourceResolver); + } + + public List resolve(APIMessage msg) { + VmUuidResolver resolver = resolvers.getOrDefault(msg.getClass(), defaultResolver); + return resolver.resolve(msg); + } + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/metadata/MetadataImpactLevel.java b/compute/src/main/java/org/zstack/compute/vm/metadata/MetadataImpactLevel.java new file mode 100644 index 00000000000..53ea57b0837 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/metadata/MetadataImpactLevel.java @@ -0,0 +1,15 @@ +package org.zstack.compute.vm.metadata; + +public enum MetadataImpactLevel { + /** + * 配置变更：仅更新数据库中的元数据，VM 运行时不感知，适用于大多数场景 + */ + CONFIG, + + /** + * 运行时变更：除了更新数据库，还需要通过消息通知 VM 实时更新元数据，适用于需要即时生效的场景 + */ + STORAGE, + + NONE +} diff --git a/compute/src/main/java/org/zstack/compute/vm/metadata/UpdateVmInstanceMetadataGC.java b/compute/src/main/java/org/zstack/compute/vm/metadata/UpdateVmInstanceMetadataGC.java new file mode 100644 index 00000000000..c62c90f70cf --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/metadata/UpdateVmInstanceMetadataGC.java @@ -0,0 +1,453 @@ +package org.zstack.compute.vm.metadata; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.cloudbus.ResourceDestinationMaker; +import org.zstack.core.gc.GC; +import org.zstack.core.gc.GCCompletion; +import org.zstack.core.gc.GCConstants; +import org.zstack.core.gc.SubmitTimeBasedGarbageCollectorMsg; +import org.zstack.core.gc.TimeBasedGarbageCollector; +import org.zstack.core.thread.ChainTask; +import org.zstack.core.thread.SyncTaskChain; +import org.zstack.core.thread.ThreadFacade; +import org.zstack.header.errorcode.ErrorCode; +import org.zstack.header.message.MessageReply; +import org.zstack.header.vm.UpdateVmInstanceMetadataMsg; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceVO; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.util.concurrent.TimeUnit; + +/** + * 虚拟机元数据更新 GC 任务。 + * + *

设计要点

+ *
    + *
  1. 持久化重试计数：{@code retryCount} 标记了 {@link GC}，序列化到 + * {@code GarbageCollectorVO.context}，MN 重启后孤儿加载时不会丢失。
    2. + *
  2. 指数退避：每次失败后 {@code NEXT_TIME = BASE_DELAY_SECONDS * 2^retryCount}， + * 由父类 {@link TimeBasedGarbageCollector#fail} 内部调用 {@code setupTimer()} + * 使用更新后的值。退避序列：10s → 20s → 40s → 80s → 160s。
    3. + *
  3. ChainTask 内建去重：通过 {@code getDeduplicateString()} + {@code getMaxPendingTasks(1)} + * 在 {@code DispatchQueueImpl.doChainSyncSubmit} 的 {@code synchronized(chainTasks)} 块内 + * 原子地完成检查+入队，避免手动 {@code getChainTaskInfo + chainSubmit} 两步操作的竞态条件。
    4. + *
  4. 去掉 GC 嵌套创建：{@code triggerNow} 直接发送 {@code UpdateVmInstanceMetadataMsg}， + * 重试统一由本 GC 实例的 {@code fail → setupTimer} 驱动， + * 不再由 handler 失败路径创建新 GC 导致滚雪球式膨胀。
    5. + *
+ * + *

串行化保证（四层）

+ * 
+ *   Layer 0 — Owner 归集（方案 C）：triggerNow() 检查 isManagedByUs，非 owner 通过 delegate 转移到 owner MN
+ *   Layer 1 — GC 框架：同一 GC 实例的 runTrigger() 通过 lockJob CAS 防止并发
+ *   Layer 2 — ChainTask 队列 "update-vm-{vmUuid}-metadata"：同一 VM 最多 1 个正在执行 + 1 个排队
+ *   Layer 3 — 主存储级队列 "update-metadata-on-ps-{psUuid}"：同一存储最多 N 个并发写入
+ *
+ * + *

调用方注意事项

+ *

外部创建本 GC 实例时，必须使用 {@link #getGCName(String)} 设置 NAME， + * 并通过 {@code SubmitTimeBasedGarbageCollectorMsg} 路由到 hash 环对应 MN， + * reply 失败时回退本地 submit（方案 B）：

+ * 
+ *   UpdateVmInstanceMetadataGC gc = new UpdateVmInstanceMetadataGC();
+ *   gc.vmInstanceUuid = vmUuid;
+ *   gc.NAME = UpdateVmInstanceMetadataGC.getGCName(vmUuid);
+ *
+ *   SubmitTimeBasedGarbageCollectorMsg gcmsg = new SubmitTimeBasedGarbageCollectorMsg();
+ *   gcmsg.setGc(gc);
+ *   gcmsg.setGcInterval(BASE_DELAY_SECONDS);
+ *   gcmsg.setUnit(TimeUnit.SECONDS);
+ *   bus.makeTargetServiceIdByResourceUuid(gcmsg, GCConstants.SERVICE_ID, vmUuid);
+ *   bus.send(gcmsg, new CloudBusCallBack(null) {
+ *       public void run(MessageReply reply) {
+ *           if (!reply.isSuccess()) gc.submit(BASE_DELAY_SECONDS, TimeUnit.SECONDS);
+ *       }
+ *   });
+ *
+ * + *

为什么不用 deduplicateSubmit：deduplicateSubmit 通过 existedAndNotCompleted() + * 检查 DB 中是否存在同 NAME 且未完成的 GC。但 GC 从 timer 触发到实际执行完毕期间 + * status 始终为 Idle，新请求会被误判为"已有 GC 在处理"，导致元数据更新丢失。 + * 使用 submit() + ChainTask maxPendingTasks=1 可正确控制并发。

+ */ +public class UpdateVmInstanceMetadataGC extends TimeBasedGarbageCollector { + private static final CLogger logger = Utils.getLogger(UpdateVmInstanceMetadataGC.class); + + // ===================================================================== + // 持久化字段（标记 @GC，序列化到 GarbageCollectorVO.context） + // ===================================================================== + + /** 目标虚拟机 UUID */ + @GC + public String vmInstanceUuid; + + /** + * 已重试次数。 + *

标记 {@link GC} 后会被序列化到 {@code GarbageCollectorVO.context} JSON 中， + * MN 重启后通过孤儿加载 {@code loadFromVO()} 恢复，不会重置为 0。

+ */ + @GC + public int retryCount = 0; + + // ===================================================================== + // 常量 + // ===================================================================== + + /** 指数退避基准延迟（秒）。退避公式：baseDelay * 2^retryCount */ + private static final long BASE_DELAY_SECONDS = 10; + + /** + * 最大重试次数。超过后 GC 标记 Done 不再执行，触发运维告警。 + *

如需运行时可配置，可改为读取 GlobalConfig，此处为编译期默认值。

+ */ + private static final int DEFAULT_MAX_RETRY = 5; + + /** + * 指数退避的指数上限，防止左移溢出。 + *

2^10 = 1024，最大延迟 = 10 * 1024 = 10240s ≈ 2.8 小时。

+ */ + private static final int MAX_EXPONENT = 5; + + // ===================================================================== + // 注入 + // ===================================================================== + + @Autowired + protected ThreadFacade thdf; + + @Autowired + protected ResourceDestinationMaker destMaker; + + // ===================================================================== + // 静态工具方法 + // ===================================================================== + + /** + * 生成 ChainTask 的 syncSignature / deduplicateString。 + *

同一 VM 的所有元数据更新任务共享同一个队列名，确保串行执行。

+ * + * @param vmInstanceUuid 虚拟机 UUID + * @return 队列签名，如 "update-vm-abc123-metadata" + */ + public static String getUpdateVmInstanceMetadataSyncSignature(String vmInstanceUuid) { + return String.format("update-vm-%s-metadata", vmInstanceUuid); + } + + /** + * 生成 GC 任务的全局唯一 NAME。 + * + *

重要：所有创建本 GC 实例的调用方必须统一使用此方法设置 NAME， + * 保证 {@link TimeBasedGarbageCollector#deduplicateSubmit} 和 + * {@link org.zstack.core.gc.GarbageCollector#existedAndNotCompleted()} + * 能基于 NAME 正确去重。

+ * + *

错误示范（会导致去重失效）：

+ * 
+     *   gc.NAME = String.format("gc-update-vm-%s-metadata", vmUuid);  // ❌ 格式不一致
+     *
+ * + * @param vmInstanceUuid 虚拟机 UUID + * @return GC NAME，如 "update-vm-abc123-metadata-gc" + */ + public static String getGCName(String vmInstanceUuid) { + return String.format("update-vm-%s-metadata-gc", vmInstanceUuid); + } + + // ===================================================================== + // 核心逻辑 + // ===================================================================== + + /** + * GC 触发入口，由父类 {@code GarbageCollector.runTrigger()} 调用。 + * + *

执行流程：

+ * 
+     *  1. 前置检查
+     *     ├─ VM 不存在 → cancel()（GC 标记 Done）
+     *     ├─ 不是 hash 环 owner → delegateToOwner()（方案 C）
+     *     │   ├─ delegate 成功 → success()（职责已转移）
+     *     │   └─ delegate 失败 → onUpdateFail()（退避后重试）
+     *     └─ 超过最大重试次数 → success()（GC 标记 Done + 触发告警）
+     *
+     *  2. 提交 ChainTask 到 "update-vm-{vmUuid}-metadata" 队列
+     *     ├─ 队列已满（maxPending=1）→ exceedMaxPendingCallback → success()
+     *     └─ 入队成功 → run()
+     *
+     *  3. ChainTask.run()
+     *     ├─ 发送 UpdateVmInstanceMetadataMsg
+     *     ├─ 成功 → completion.success() → GC Done
+     *     └─ 失败 → onUpdateFail() → retryCount++, 指数退避, completion.fail()
+     *                                → 父类 setupTimer() 在新延迟后重新触发
+     *
+ * + * @param completion GC 完成回调，调用 success/fail/cancel 驱动 GC 状态机 + */ + @Override + protected void triggerNow(GCCompletion completion) { + // ── 前置检查 1：VM 已删除则取消 GC ── + // VM 删除后元数据文件由 DeleteVmInstanceMetadataGC 负责清理 + if (!dbf.isExist(vmInstanceUuid, VmInstanceVO.class)) { + logger.debug(String.format("[MetadataGC] VM[uuid:%s] has been deleted, " + + "cancel GC[name:%s, id:%s]", vmInstanceUuid, NAME, getUuid())); + completion.cancel(); + return; + } + + // ── 前置检查 2：owner 归集（方案 C）── + // 如果当前 MN 不是 hash 环上该 VM 的 owner，将 GC 委托给 owner MN 执行。 + // 场景：MN 拓扑变更（加入/离开）后，回退到本地的 GC 可能不再属于当前 MN。 + if (!destMaker.isManagedByUs(vmInstanceUuid)) { + delegateToOwner(completion); + return; + } + + // ── 前置检查 3：超过最大重试次数 ── + if (retryCount >= DEFAULT_MAX_RETRY) { + logger.warn(String.format("[MetadataGC] VM[uuid:%s] exceeded max retry count %d, " + + "giving up GC[name:%s, id:%s]. " + + "Please check primary storage status and use manual API to retry.", + vmInstanceUuid, DEFAULT_MAX_RETRY, NAME, getUuid())); + + // TODO: 触发运维告警（ZWatch 报警器），推送主存储 UUID 到监控系统 + // 参考文档 3.4 节：新增报警器 "更新虚拟机元数据失败" + + // 标记 Done：不再自动重试。运维可通过手动生成元数据 API 重新触发。 + completion.success(); + return; + } + + // ── 提交到 ChainTask 队列 ── + // + // 队列设计要点： + // - syncSignature = "update-vm-{vmUuid}-metadata"，每 VM 独立队列 + // - syncLevel = 1（默认），同时只有 1 个任务在执行 + // - maxPendingTasks = 1，最多 1 个排队任务 + // - deduplicateString = syncSignature，同队列内按此 key 去重 + // + // 效果：队列中最多 1 running + 1 pending，满足文档要求 "pendingTask <= 1" + // + final String queueName = getUpdateVmInstanceMetadataSyncSignature(vmInstanceUuid); + + thdf.chainSubmit(new ChainTask(completion) { + @Override + public String getSyncSignature() { + // 队列名：每个 VM 独立队列，保证同一 VM 的元数据更新串行执行 + return queueName; + } + + /** + * 去重 key：与 syncSignature 相同。 + * + *

当多个 GC 几乎同时 trigger 时， + * {@code DispatchQueueImpl.ChainTaskQueueWrapper.addTask()} 在 + * {@code synchronized(chainTasks)} 块内检查 {@code subPendingMap}， + * 发现同一 deduplicateString 的 pending 数量已达 maxPendingTasks 时， + * 拒绝入队并调用 {@code exceedMaxPendingCallback()}。

+ * + *

为什么不用 getChainTaskInfo + chainSubmit 两步检查： + * 两步操作之间释放了 chainTasks 锁，存在竞态窗口—— + * 两个 GC 可能同时通过检查，导致 pendingQueue 中出现两个任务。 + * 框架内建去重在同一个 synchronized 块内原子完成，没有竞态。

+ */ + @Override + protected String getDeduplicateString() { + return queueName; + } + + /** + * 最大排队任务数 = 1。 + * + *

效果：队列中最多有 1 个正在执行的任务 + 1 个等待的任务。

+ * + *

为什么 1 就够：元数据是全量更新（从 DB 读取最新状态 → 覆盖写入存储）， + * 排队的任务执行时会拿到最新数据，中间丢弃的 GC 触发不会导致数据丢失。

+ */ + @Override + protected int getMaxPendingTasks() { + return 1; + } + + /** + * 队列已满时的回调（在 synchronized(chainTasks) 块内被调用）。 + * + *

此时队列中已有 1 running + 1 pending，pending 任务执行时 + * 会从 DB 全量读取最新数据，因此当前 GC 的更新内容必然被覆盖。

+ * + *

为什么调用 success() 而不是 cancel()：

+ *
    + *
  • 两者对 GarbageCollectorVO 的最终效果相同（状态 → Done）， + * 但 success() 在日志中语义更清晰："职责已被接管"。
    • + *
  • 如果队列中的排队任务后续也失败了，那个任务有自己的 GC 实例 + * 会走 fail → setupTimer 重试，不依赖当前 GC 实例。
    • + *
+ */ + @Override + protected void exceedMaxPendingCallback() { + logger.debug(String.format( + "[MetadataGC] VM[uuid:%s] queue[%s] already has pending task, " + + "current GC[name:%s, id:%s] is superseded, mark success", + vmInstanceUuid, queueName, NAME, getUuid())); + completion.success(); + } + + /** + * 实际执行元数据更新。 + * + *

发送 {@code UpdateVmInstanceMetadataMsg} 到 VM 在 hash 环上对应的 MN， + * 由 {@code VmInstanceBase.handle(UpdateVmInstanceMetadataMsg)} 从 DB 全量构建 + * 元数据后，通过 {@code UpdateVmInstanceMetadataOnPrimaryStorageMsg} 转发到 + * 主存储所属 MN，最终由 hypervisor agent 写入存储。

+ * + *

注意：{@code VmInstanceBase.handle(UpdateVmInstanceMetadataMsg)} 的失败路径 + * 不应再创建新的 GC（避免 GC 滚雪球膨胀），应直接返回失败 reply， + * 由此处的 {@link #onUpdateFail} 统一处理重试。

+ */ + @Override + public void run(final SyncTaskChain chain) { + UpdateVmInstanceMetadataMsg msg = new UpdateVmInstanceMetadataMsg(); + msg.setUuid(vmInstanceUuid); + bus.makeLocalServiceId(msg, VmInstanceConstant.SERVICE_ID); + bus.send(msg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + // ── 更新成功 ── + // 重置 retryCount 并持久化，确保下次 GC 触发时从 0 开始计数。 + // 场景：GC 之前失败过 2 次（retryCount=2），第 3 次成功了， + // 但 GC 还未被清理就被其他操作再次触发—— + // 如果不重置，retryCount 会从 2 开始计，剩余重试次数变少。 + retryCount = 0; + updateContext(); + completion.success(); + } else { + // ── 更新失败：走指数退避重试路径 ── + onUpdateFail(completion, reply.getError()); + } + + // 无论成功失败，必须调用 chain.next() 释放 ChainTask 队列槽位， + // 让排队的下一个任务（如果有）可以执行。 + chain.next(); + } + }); + } + + @Override + public String getName() { + return String.format("update-vm-%s-metadata-chain-task", vmInstanceUuid); + } + }); + } + + // ===================================================================== + // Owner 归集（方案 C） + // ===================================================================== + + /** + * 将本 GC 委托给 hash 环 owner MN 执行。 + * + *

发送 {@code SubmitTimeBasedGarbageCollectorMsg} 到 owner MN：

+ *
    + *
  • 成功 → owner MN 上创建了新 GC，本地 GC 标记 Done（职责已转移）
    • + *
  • 失败 → owner 不可达，走 fail 路径，指数退避后重试。 + * 下次 triggerNow() 时 hash 环可能已修正，当前 MN 成为 owner 则直接执行。
    • + *
+ * + * @param completion GC 完成回调 + */ + private void delegateToOwner(GCCompletion completion) { + String owner = destMaker.makeDestination(vmInstanceUuid); + logger.debug(String.format("[MetadataGC] vm[uuid:%s] not managed by us, " + + "delegating GC[name:%s, id:%s] to owner MN[%s]", + vmInstanceUuid, NAME, getUuid(), owner)); + + UpdateVmInstanceMetadataGC delegateGc = new UpdateVmInstanceMetadataGC(); + delegateGc.vmInstanceUuid = vmInstanceUuid; + delegateGc.NAME = getGCName(vmInstanceUuid); + + SubmitTimeBasedGarbageCollectorMsg gcmsg = new SubmitTimeBasedGarbageCollectorMsg(); + gcmsg.setGc(delegateGc); + gcmsg.setGcInterval(BASE_DELAY_SECONDS); + gcmsg.setUnit(TimeUnit.SECONDS); + bus.makeServiceIdByManagementNodeId(gcmsg, GCConstants.SERVICE_ID, owner); + + bus.send(gcmsg, new CloudBusCallBack(completion) { + @Override + public void run(MessageReply reply) { + if (reply.isSuccess()) { + // owner 接管成功 → 本地 GC 使命完成 + completion.success(); + } else { + // owner 不可达 → 走 fail 路径，退避后重试 + // 下次 triggerNow() 时 hash 环可能已变化 + logger.warn(String.format("[MetadataGC] delegate to owner MN[%s] failed " + + "for vm[uuid:%s], will retry with backoff: %s", + owner, vmInstanceUuid, reply.getError())); + onUpdateFail(completion, reply.getError()); + } + } + }); + } + + // ===================================================================== + // 失败处理 + // ===================================================================== + + /** + * 元数据更新失败时的处理：递增重试计数 + 指数退避 + 持久化 + 触发 GC 重新调度。 + * + *

执行步骤：

+ *
    + *
  1. 递增 {@code retryCount}
    2. + *
  2. 计算指数退避延迟：{@code BASE_DELAY_SECONDS * 2^min(retryCount, MAX_EXPONENT)} + *
    退避序列：10s → 20s → 40s → 80s → 160s → 320s（若 MAX_RETRY 更大时）
    3. + *
  3. 更新父类 {@code NEXT_TIME} / {@code NEXT_TIME_UNIT} 字段
    4. + *
  4. 调用 {@code updateContext()} 将 retryCount + NEXT_TIME 持久化到 DB
    5. + *
  5. 调用 {@code completion.fail(err)} 触发父类失败流程
    6. + *
+ * + *

为什么在 completion.fail() 之前更新 NEXT_TIME：

+ *

父类 {@link TimeBasedGarbageCollector#fail} 的执行顺序是：

+ * 
+     *   super.fail(err)     → GarbageCollectorVO.status = Idle
+     *   setupTimer()        → new Timer().schedule(task, NEXT_TIME_UNIT.toMillis(NEXT_TIME))
+     *
+ *

{@code setupTimer()} 直接读取 {@code NEXT_TIME}。 + * 如果不提前设置，它会使用上一次的旧间隔（初始固定间隔），无法实现指数退避。

+ * + *

为什么要 updateContext()：

+ *

如果 {@code fail()} 之后 MN 崩溃，孤儿 GC 被其他 MN 的 {@code loadOrphanJobs()} 加载时， + * {@code retryCount} 和 {@code NEXT_TIME} 需要从 {@code GarbageCollectorVO.context} 恢复， + * 否则会重置为 0 导致无限重试。

+ * + * @param completion GC 完成回调 + * @param err 失败错误码 + */ + private void onUpdateFail(GCCompletion completion, ErrorCode err) { + retryCount++; + + // 指数退避：baseDelay * 2^retryCount，上限 2^MAX_EXPONENT 防止左移溢出 + int exponent = Math.min(retryCount, MAX_EXPONENT); + NEXT_TIME = BASE_DELAY_SECONDS * (1L << exponent); + NEXT_TIME_UNIT = TimeUnit.SECONDS; + + logger.warn(String.format("[MetadataGC] VM[uuid:%s] metadata update failed " + + "(retry %d/%d), next attempt in %ds. GC[name:%s, id:%s]. Error: %s", + vmInstanceUuid, retryCount, DEFAULT_MAX_RETRY, + NEXT_TIME, NAME, getUuid(), err)); + + // 持久化 retryCount + NEXT_TIME 到 GarbageCollectorVO.context， + // 确保 MN 重启后孤儿加载能恢复正确的重试状态和退避间隔 + updateContext(); + + // 触发父类 fail 流程： + // GarbageCollector.fail() → 设置 VO.status = Idle, unlock() + // TimeBasedGarbageCollector.fail() → 调用 setupTimer() + // setupTimer() → 使用已更新的 NEXT_TIME 创建新的 TimerTask + // + // 下次 TimerTask 到期时会调用 runTrigger() → triggerNow()，重新走一遍完整流程 + completion.fail(err); + } +} diff --git a/compute/src/main/java/org/zstack/compute/vm/metadata/VmMetadataUpdateInterceptor.java b/compute/src/main/java/org/zstack/compute/vm/metadata/VmMetadataUpdateInterceptor.java new file mode 100644 index 00000000000..e1753290d44 --- /dev/null +++ b/compute/src/main/java/org/zstack/compute/vm/metadata/VmMetadataUpdateInterceptor.java @@ -0,0 +1,219 @@ +package org.zstack.compute.vm.metadata; + + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.Platform; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.cloudbus.CloudBusCallBack; +import org.zstack.core.gc.GCConstants; +import org.zstack.core.gc.SubmitTimeBasedGarbageCollectorMsg; +import org.zstack.header.Component; +import org.zstack.header.message.*; +import org.zstack.utils.BeanUtils; +import org.zstack.utils.Utils; +import org.zstack.utils.logging.CLogger; + +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * 拦截标注了 {@link MetadataImpact} 的 API 消息，在 API 成功后提交元数据更新 GC 任务。 + * + *

工作流程

+ * 
+ * API Message 投递                          Event 发布
+ *       │                                      │
+ *       ▼                                      ▼
+ * BeforeDeliveryMessageInterceptor    BeforePublishEventInterceptor
+ *       │                                      │
+ *       │  检测 @MetadataImpact               │  通过 apiId 匹配
+ *       │  提取 vmInstanceUuid                 │  检查 API 是否成功
+ *       │  缓存到 pendingApis                  │  提交 GC 任务
+ *       │  (key = apiId)                       │  清理 pendingApis
+ *       ▼                                      ▼
+ *
+ * + *

GC 提交策略

+ *

使用 {@code submit()}（非 {@code deduplicateSubmit}），每次 API 成功都创建新 GC 行。 + * 通过 ChainTask {@code maxPendingTasks=1} 控制同一 VM 的执行扩散。 + * 详见 {@link UpdateVmInstanceMetadataGC} Javadoc 中的设计说明。

+ */ +public class VmMetadataUpdateInterceptor implements Component { + private static final CLogger logger = Utils.getLogger(VmMetadataUpdateInterceptor.class); + + /** + * 初始 GC 延迟（秒）。GC timer 到期后触发第一次元数据写入尝试。 + *

使用较短的初始延迟，让 API 成功后尽快写入存储。 + * 失败后由 UpdateVmInstanceMetadataGC 内部的指数退避接管。

+ */ + private static final long INITIAL_GC_DELAY_SECONDS = 10; + + @Autowired + private CloudBus bus; + + // apiId -> MetadataImpactInfo 映射，在 API 投递时写入，在 Event 发布时消费 + private final Map pendingApis = new ConcurrentHashMap<>(); + + // 标注了 @MetadataImpact 的 API 消息类集合 + private final Set> impactApiClasses = ConcurrentHashMap.newKeySet(); + + @Override + public boolean start() { + // 1. 扫描所有标注了 @MetadataImpact 的 API 消息类 + scanMetadataImpactApis(); + + // 2. 注册 BeforeDeliveryMessageInterceptor，在 API 消息被投递处理前， + // 提取 vmInstanceUuid 并缓存 + bus.installBeforeDeliveryMessageInterceptor(new AbstractBeforeDeliveryMessageInterceptor() { + @Override + public void beforeDeliveryMessage(Message msg) { + if (!(msg instanceof APIMessage)) { + return; + } + if (!impactApiClasses.contains(msg.getClass())) { + return; + } + + APIMessage apiMsg = (APIMessage) msg; + String vmUuid = extractVmInstanceUuid(apiMsg); + if (vmUuid == null) { + return; + } + + MetadataImpact impact = msg.getClass().getAnnotation(MetadataImpact.class); + pendingApis.put(apiMsg.getId(), new MetadataImpactInfo(vmUuid, impact.value())); + } + }, impactApiClasses.toArray(new Class[0])); + + // 3. 注册 BeforePublishEventInterceptor，在 Event 发布前检查并提交 GC + bus.installBeforePublishEventInterceptor(new AbstractBeforePublishEventInterceptor() { + @Override + public void beforePublishEvent(Event evt) { + if (!(evt instanceof APIEvent)) { + return; + } + + APIEvent apiEvent = (APIEvent) evt; + MetadataImpactInfo info = pendingApis.remove(apiEvent.getApiId()); + if (info == null) { + return; + } + + // API 失败则跳过（除非 @MetadataImpact(updateOnFailure=true)） + if (apiEvent.getError() != null) { + return; + } + + submitUpdateVmInstanceMetadataGC(info.vmInstanceUuid); + } + }); + + return true; + } + + private void scanMetadataImpactApis() { + // 利用 ZStack 的反射工具扫描所有带 @MetadataImpact 注解的 APIMessage 子类 + BeanUtils.reflections.getTypesAnnotatedWith(MetadataImpact.class).forEach(clz -> { + if (APIMessage.class.isAssignableFrom(clz)) { + impactApiClasses.add((Class) clz); + logger.debug(String.format("detected @MetadataImpact API: %s", clz.getName())); + } + }); + } + + /** + * 从 API 消息中提取 vmInstanceUuid。 + *

约定：标注 @MetadataImpact 的 API 消息必须有 getVmInstanceUuid() 方法， + * fallback 到 getResourceUuid()。

+ */ + private String extractVmInstanceUuid(APIMessage msg) { + try { + Method method = msg.getClass().getMethod("getVmInstanceUuid"); + return (String) method.invoke(msg); + } catch (NoSuchMethodException e) { + // 尝试 getResourceUuid 作为 fallback + try { + Method method = msg.getClass().getMethod("getResourceUuid"); + return (String) method.invoke(msg); + } catch (Exception ex) { + logger.warn(String.format("cannot extract vmInstanceUuid from %s", msg.getClass().getName())); + return null; + } + } catch (Exception e) { + logger.warn(String.format("failed to extract vmInstanceUuid from %s: %s", + msg.getClass().getName(), e.getMessage())); + return null; + } + } + + /** + * 通过 SubmitTimeBasedGarbageCollectorMsg 将 GC 提交到 hash 环拥有 vmInstanceUuid 的 MN。 + * + *

方案 B：远程提交 + reply 回退

+ *

正常情况下，GC 通过 hash 环路由到 owner MN 完成提交，保证同一 VM 的 GC 归集到一个 MN。

+ *

如果 owner MN 不可达（宕机/网络故障），reply 超时后回退到本地 submit，确保 GC 不丢失。

+ * + *

使用 submit()（非 deduplicateSubmit），每次创建新 GC 行。 + * GarbageCollectorManagerImpl.handle(SubmitTimeBasedGarbageCollectorMsg) 调用 gc.submit() + * 完成持久化 + timer 注册。

+ * + *

同一 VM 的并发控制由 UpdateVmInstanceMetadataGC.triggerNow() 中的 + * ChainTask maxPendingTasks=1 保证，超出的 GC 立即标记 Done。

+ * + *

极小概率场景（MN 加入时 hash 环瞬时不一致）可能导致两个 MN 各有 GC， + * 由 triggerNow() 中的 owner 检查 + delegate 机制（方案 C）兜底归集。

+ * + * @param vmInstanceUuid 目标虚拟机 UUID + */ + private void submitUpdateVmInstanceMetadataGC(String vmInstanceUuid) { + logger.debug(String.format("[MetadataGC] API succeeded, submitting metadata update GC " + + "for vm[uuid:%s]", vmInstanceUuid)); + + UpdateVmInstanceMetadataGC gc = new UpdateVmInstanceMetadataGC(); + gc.vmInstanceUuid = vmInstanceUuid; + gc.NAME = UpdateVmInstanceMetadataGC.getGCName(vmInstanceUuid); + + SubmitTimeBasedGarbageCollectorMsg gcmsg = new SubmitTimeBasedGarbageCollectorMsg(); + gcmsg.setGc(gc); + gcmsg.setGcInterval(INITIAL_GC_DELAY_SECONDS); + gcmsg.setUnit(TimeUnit.SECONDS); + + // 路由到 hash 环拥有 vmInstanceUuid 的 MN + bus.makeTargetServiceIdByResourceUuid(gcmsg, GCConstants.SERVICE_ID, vmInstanceUuid); + bus.send(gcmsg, new CloudBusCallBack(null) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + // owner MN 不可达（宕机/网络故障）→ 回退到本地 submit，确保 GC 不丢失。 + // 本地 GC 在 triggerNow() 时如果发现自己不是 owner，会通过 delegate 机制转移。 + logger.warn(String.format("[MetadataGC] failed to submit GC to owner MN " + + "for vm[uuid:%s], fallback to local submit: %s", + vmInstanceUuid, reply.getError())); + gc.submit(INITIAL_GC_DELAY_SECONDS, TimeUnit.SECONDS); + } + } + }); + } + + @Override + public boolean stop() { + pendingApis.clear(); + return true; + } + + /** + * 缓存 API 投递时提取的元数据影响信息，供 Event 发布时匹配使用。 + */ + private static class MetadataImpactInfo { + final String vmInstanceUuid; + final MetadataImpactLevel level; + + MetadataImpactInfo(String vmInstanceUuid, MetadataImpactLevel level) { + this.vmInstanceUuid = vmInstanceUuid; + this.level = level; + } + } +} \ No newline at end of file diff --git a/conf/globalConfig/vm.xml b/conf/globalConfig/vm.xml index 8563169b335..518369f1d93 100755 --- a/conf/globalConfig/vm.xml +++ b/conf/globalConfig/vm.xml @@ -317,4 +317,20 @@ java.lang.Boolean false + + + vm + deletion.gcInterval + update vm metadata interval + java.lang.Long + 30 + + + + vm + vm.metadata + save vm metadata + java.lang.Boolean + false + diff --git a/conf/serviceConfig/primaryStorage.xml b/conf/serviceConfig/primaryStorage.xml index 337ce4eaac3..06f4d94bc07 100755 --- a/conf/serviceConfig/primaryStorage.xml +++ b/conf/serviceConfig/primaryStorage.xml @@ -84,4 +84,7 @@ org.zstack.header.storage.primary.APIAddStorageProtocolMsg + + org.zstack.header.storage.primary.APIRegisterVmInstanceMsg + diff --git a/core/src/main/java/org/zstack/core/gc/EventBasedGarbageCollector.java b/core/src/main/java/org/zstack/core/gc/EventBasedGarbageCollector.java index 35a3aa72182..fa47b156c83 100755 --- a/core/src/main/java/org/zstack/core/gc/EventBasedGarbageCollector.java +++ b/core/src/main/java/org/zstack/core/gc/EventBasedGarbageCollector.java @@ -12,7 +12,10 @@ */ public abstract class EventBasedGarbageCollector extends GarbageCollector { public void load(GarbageCollectorVO vo) { - loadFromVO(vo); + if (!loadFromVO(vo)) { + // 乐观锁竞争失败：该 GC 已被其他线程/MN 认领，跳过 + return; + } setup(); installTriggers(); logger.debug(String.format("[GC] loaded a job[name:%s, id:%s]", NAME, uuid)); diff --git a/core/src/main/java/org/zstack/core/gc/GarbageCollector.java b/core/src/main/java/org/zstack/core/gc/GarbageCollector.java index 734818f1288..62f4b85b71d 100755 --- a/core/src/main/java/org/zstack/core/gc/GarbageCollector.java +++ b/core/src/main/java/org/zstack/core/gc/GarbageCollector.java @@ -173,7 +173,16 @@ public static String updateContext(String uuid, Str return context; } - void loadFromVO(GarbageCollectorVO vo) { + /** + * 从 GarbageCollectorVO 恢复 GC 实例（孤儿加载 / 手动触发场景）。 + * + *

使用乐观锁（条件更新）认领 GC，防止多个线程/MN 并发加载同一个孤儿 GC。 + * 只有 managementNodeUuid 仍为 NULL 的 VO 才会被认领成功。

+ * + * @param vo GC 数据库记录 + * @return true=认领成功，false=已被其他线程/MN 认领 + */ + boolean loadFromVO(GarbageCollectorVO vo) { Object dataObj = JSONObjectUtil.toObject(vo.getContext(), getClass()); for (Field f : FieldUtils.getAllFields(getClass())) { @@ -190,11 +199,23 @@ void loadFromVO(GarbageCollectorVO vo) { } uuid = vo.getUuid(); - vo.setStatus(GCStatus.Idle); - vo.setManagementNodeUuid(Platform.getManagementServerId()); - dbf.update(vo); + + // 乐观锁：只认领 managementNodeUuid 为 NULL 的 GC（防止并发加载同一孤儿） + int updated = SQL.New(GarbageCollectorVO.class) + .eq(GarbageCollectorVO_.uuid, vo.getUuid()) + .isNull(GarbageCollectorVO_.managementNodeUuid) + .set(GarbageCollectorVO_.status, GCStatus.Idle) + .set(GarbageCollectorVO_.managementNodeUuid, Platform.getManagementServerId()) + .update(); + + if (updated == 0) { + logger.debug(String.format("[GC] job[name:%s, id:%s] already claimed by another node, skip", + vo.getName(), vo.getUuid())); + return false; + } gcMgr.registerGC(this); + return true; } @SyncThread(level = 50) diff --git a/core/src/main/java/org/zstack/core/gc/GarbageCollectorManagerImpl.java b/core/src/main/java/org/zstack/core/gc/GarbageCollectorManagerImpl.java index 9cbc35b3217..d012531bc53 100755 --- a/core/src/main/java/org/zstack/core/gc/GarbageCollectorManagerImpl.java +++ b/core/src/main/java/org/zstack/core/gc/GarbageCollectorManagerImpl.java @@ -255,6 +255,8 @@ public void handleMessage(Message msg) { private void handleLocalMessage(Message msg) { if (msg instanceof TriggerGcJobMsg) { handle((TriggerGcJobMsg) msg); + } else if (msg instanceof SubmitTimeBasedGarbageCollectorMsg) { + handle((SubmitTimeBasedGarbageCollectorMsg) msg); } else { bus.dealWithUnknownMessage(msg); } @@ -282,6 +284,12 @@ public String getName() { }); } + private void handle(final SubmitTimeBasedGarbageCollectorMsg msg) { + MessageReply reply = new MessageReply(); + msg.getGc().submit(msg.getGcInterval(), msg.getUnit()); + bus.reply(msg, reply); + } + private void handleApiMessage(APIMessage msg) { if (msg instanceof APITriggerGCJobMsg) { handle((APITriggerGCJobMsg) msg); diff --git a/core/src/main/java/org/zstack/core/gc/SubmitTimeBasedGarbageCollectorMsg.java b/core/src/main/java/org/zstack/core/gc/SubmitTimeBasedGarbageCollectorMsg.java new file mode 100644 index 00000000000..67aebc6accf --- /dev/null +++ b/core/src/main/java/org/zstack/core/gc/SubmitTimeBasedGarbageCollectorMsg.java @@ -0,0 +1,35 @@ +package org.zstack.core.gc; + +import org.zstack.header.message.NeedReplyMessage; + +import java.util.concurrent.TimeUnit; + +public class SubmitTimeBasedGarbageCollectorMsg extends NeedReplyMessage { + private TimeBasedGarbageCollector gc; + private Long gcInterval; + private TimeUnit unit; + + public TimeBasedGarbageCollector getGc() { + return gc; + } + + public void setGc(TimeBasedGarbageCollector gc) { + this.gc = gc; + } + + public Long getGcInterval() { + return gcInterval; + } + + public void setGcInterval(Long gcInterval) { + this.gcInterval = gcInterval; + } + + public TimeUnit getUnit() { + return unit; + } + + public void setUnit(TimeUnit unit) { + this.unit = unit; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/zstack/core/gc/TimeBasedGarbageCollector.java b/core/src/main/java/org/zstack/core/gc/TimeBasedGarbageCollector.java index e7b3075b554..2ab108b2b51 100755 --- a/core/src/main/java/org/zstack/core/gc/TimeBasedGarbageCollector.java +++ b/core/src/main/java/org/zstack/core/gc/TimeBasedGarbageCollector.java @@ -51,7 +51,10 @@ protected void fail(ErrorCode err) { } public void load(GarbageCollectorVO vo) { - loadFromVO(vo); + if (!loadFromVO(vo)) { + // 乐观锁竞争失败：该 GC 已被其他线程/MN 认领，跳过 + return; + } prepare(); setupTimer(); } diff --git a/docs/modules/developer/pages/vm-metadata-gc-01-core-design.md b/docs/modules/developer/pages/vm-metadata-gc-01-core-design.md new file mode 100644 index 00000000000..94ae9180746 --- /dev/null +++ b/docs/modules/developer/pages/vm-metadata-gc-01-core-design.md @@ -0,0 +1,112 @@ +# 虚拟机元数据 GC — 核心设计决策 + +## 1. 概述 + +虚拟机元数据持久化到主存储后，任何影响元数据的 API 操作成功后，都需要将最新元数据同步更新到主存储。 +由于主存储写入可能失败（存储不可用、agent 故障等），引入 GC（GarbageCollector）机制进行可靠重试。 + +--- + +## 2. 每次 submit()，不用 deduplicateSubmit() + +`deduplicateSubmit()` 内部调用 `existedAndNotCompleted()` 检查 DB 中是否存在同 NAME 且未完成的 GC。 +但 GC 从 timer 触发到实际执行完毕期间，status 始终为 `Idle`，新请求会被误判为"已有 GC 在处理"： + +``` +T0: API-1 成功 → DB metadata=v1 → GC-1 创建 (status=Idle) +T1: GC-1 timer 到期 → triggerNow() → handler 读 DB → 得到 v1 → 开始写 PS... + 此时 GC-1 status 仍为 Idle（只在 success/fail/cancel 时才变） +T2: API-2 成功 → DB metadata=v2 → deduplicateSubmit: + existedAndNotCompleted() → 发现 GC-1 (status=Idle ≠ Done) → 跳过！ +T3: GC-1 写入 v1 → success() → Done +T4: PS 上是 v1，DB 上是 v2 → 元数据丢失！ +``` + +**方案**：每次都 `submit()` 创建新 GC，通过 ChainTask `maxPendingTasks=1` 控制同一 VM 的执行扩散。 + +- 队列中最多 1 running + 1 pending +- pending 的 GC 执行时从 DB 全量读取最新数据，覆盖写入 +- 多余的 GC 在 `exceedMaxPendingCallback` 中标记 `success()`（Done），表示"职责已被接管" + +**GC 行数影响**：每个 metadata-impacting API 创建一行。Done 状态的 GC 由 GCGlobalConfig cleanUp 周期清理（默认 24h）。 + +--- + +## 3. Handler 端不创建新 GC + +**问题**：`VmInstanceBase.handle(UpdateVmInstanceMetadataMsg)` 原实现在写主存储失败时创建新 GC： + +```java +if (!r.isSuccess()) { + submitUpdateVmInstanceMetadataGC(); // ← 创建新 GC → 滚雪球 +} +``` + +GC-1 失败 → 创建 GC-2 → GC-1 的 fail() 又触发 setupTimer → 两个 GC 并行重试 → 指数膨胀。 + +**方案**：handler 失败时直接 `bus.reply(msg, errorReply)`，由 GC 端的 `onUpdateFail()` 统一走指数退避重试。 + +--- + +## 4. 统一 GC NAME 格式 + +**问题**：不同代码路径使用不同格式的 NAME 导致去重失效： + +```java +gc.NAME = String.format("gc-update-vm-%s-metadata", vmUuid); // ❌ 格式 A +return String.format("update-vm-%s-metadata-gc", vmInstanceUuid); // ✓ 格式 B +``` + +**方案**：所有调用方统一使用 `UpdateVmInstanceMetadataGC.getGCName(vmUuid)` 设置 NAME。 + +--- + +## 5. 指数退避重试 + +``` +retryCount | delay | 公式 +0 | 10s | 10 × 2^0 +1 | 20s | 10 × 2^1 +2 | 40s | 10 × 2^2 +3 | 80s | 10 × 2^3 +4 | 160s | 10 × 2^4 +≥5 | 放弃 | 标记 Done，触发运维告警 +``` + +- `retryCount` 标记 `@GC`，序列化到 `GarbageCollectorVO.context` JSON +- 每次失败通过 `updateContext()` 持久化 retryCount + NEXT_TIME +- MN 重启后孤儿加载通过 `loadFromVO()` 恢复，不会重置为 0 +- 超过 `DEFAULT_MAX_RETRY`(5) 后标记 Done，打印 WARN 日志，TODO 触发 ZWatch 运维告警 + +--- + +## 6. 孤儿 GC 并发加载保护 + +**问题**：`loadOrphanJobs()` 不是 synchronized 的，周期扫描和 nodeLeft 触发可能并发加载同一个 GC。 + +**方案**：`loadFromVO()` 中使用条件更新（乐观锁）： + +```java +int updated = SQL.New(GarbageCollectorVO.class) + .eq(GarbageCollectorVO_.uuid, vo.getUuid()) + .isNull(GarbageCollectorVO_.managementNodeUuid) // 关键条件 + .set(GarbageCollectorVO_.managementNodeUuid, Platform.getManagementServerId()) + .set(GarbageCollectorVO_.status, GCStatus.Idle) + .update(); + +if (updated == 0) { + // 已被其他线程/MN 认领，放弃 + return false; +} +``` + +--- + +## 7. 代码修改清单 + +| 文件 | 修改内容 | +|------|----------| +| `UpdateVmInstanceMetadataGC.java` | ChainTask 去重、指数退避、retryCount 持久化、owner 归集 | +| `VmMetadataUpdateInterceptor.java` | 使用 gc.submit()，统一 NAME 格式，方案 B 远程提交+回退 | +| `VmInstanceBase.java` | handler 加 ChainTask 串行化；失败路径不创建新 GC；消息改 makeLocalServiceId | +| `GarbageCollector.java` | loadFromVO 加乐观锁（条件更新） | diff --git a/docs/modules/developer/pages/vm-metadata-gc-02-dual-mn.md b/docs/modules/developer/pages/vm-metadata-gc-02-dual-mn.md new file mode 100644 index 00000000000..b4c2fdfb21d --- /dev/null +++ b/docs/modules/developer/pages/vm-metadata-gc-02-dual-mn.md @@ -0,0 +1,213 @@ +# 虚拟机元数据 GC — 双 MN 哈希环分析与方案 + +## 1. 哈希环路由机制 + +双 MN 使用 `ConsistentHash`（500 虚拟节点）做资源路由： + +- `ResourceDestinationMakerImpl` 维护每个 MN 本地的 JVM 哈希环 `nodeHash` +- `makeDestination(resourceUuid)` 读取**本地** `nodeHash.get(resourceUuid)` 返回 owner MN +- `CloudBusImpl3.makeTargetServiceIdByResourceUuid()` 内部调用 `destMaker.makeDestination()` 做路由 + +``` +ResourceDestinationMakerImpl: + ConsistentHash nodeHash = new ConsistentHash<>(new ApacheHash(), 500, ...) + + nodeJoin(inv) → nodeHash.add(inv.getUuid()) + nodeLeft(inv) → nodeHash.remove(inv.getUuid()) + + makeDestination(resourceUuid) → nodeHash.get(resourceUuid) +``` + +--- + +## 2. 节点生命周期事件传播 + +MN 拓扑变更事件通过以下链路传播到其他 MN： + +``` +MN-B 启动完成 → "say-I-join" flow: + evtf.fire(NODE_LIFECYCLE_PATH, data) + ├─ fireLocal(evt) → 本地 MN-B 收到（isFromThisManagementNode → 忽略） + ├─ callWebhooks(evt) + └─ bus.publish(evt) → 通过 RabbitMQ 广播 CanonicalEvent 到所有 MN + +其他 MN（如 MN-A）接收: + RabbitMQ consumer → EventFacadeImpl.handleEvent() + → 匹配 glob → CallbackWrapper.call(cevt) + → @AsyncThread → 提交到线程池异步执行 ← 关键：异步！ + → nodeLifeCycleCallback.run() + → nodeLifeCycle.nodeJoin(inv) + → destinationMaker.nodeJoin(inv) ← JVM 哈希环更新 + → lifeCycleExtension forEach nodeJoin(inv) +``` + +**关键点**：`CallbackWrapper.call()` 标注了 `@AsyncThread`，从收到 RabbitMQ 消息到**实际更新 JVM 哈希环**，中间存在线程调度延迟。 + +--- + +## 3. 异常情况一：MN-B 宕机（窗口 ~30 秒，严重） + +参数：`heartbeatInterval = 5s`，`MAX_HEARTBEAT_FAILURE = 5`，检测超时 = `(5+1) × 5 = 30s` + +**初始状态**：MN-A + MN-B 正常运行，两者哈希环都是 `{A, B}` + +``` +时间线: +──────────────────────────────────────────────────────────────────► +T0 T1 T2 +MN-B宕机 MN-A心跳检测 MN-A确认MN-B死亡 + 发现MN-B可疑 nodeLeft() + (加入suspects) 哈希环→{A} + +│←─────── 不一致窗口：最长约 30 秒 ────────────────►│ +``` + +在 T0 ~ T2 窗口内，MN-A 的哈希环仍然是 `{A, B}`。如果 API 在 MN-A 上成功触发 GC 提交： + +```java +bus.makeTargetServiceIdByResourceUuid(gcmsg, GCConstants.SERVICE_ID, "vm-123"); +// MN-A 哈希环 = {A, B}，假设 hash("vm-123") → MN-B +bus.send(gcmsg); // fire-and-forget，发往已死的 MN-B +``` + +**后果（无防护时）**：消息丢失。`bus.send()` 是 fire-and-forget，GC 没有被持久化到 DB，这条 metadata 更新**彻底丢失**。 + +**影响范围**：在 ~30 秒窗口内，所有 `hash(vmUuid) → MN-B` 的 VM 的 metadata 更新 GC 都会丢失。 + +--- + +## 4. 异常情况二：MN-B 加入（窗口毫秒级，较轻） + +**初始状态**：MN-A 独自运行，哈希环 `{A}` + +MN-B 启动流程顺序（`ManagementNodeManagerImpl.start()`）： + +``` +Step 1: persist ManagementNodeVO (DB 有记录) +Step 2: start-heartbeat +Step 3: start-api-mediator +Step 4: set-node-to-running → DB: state=RUNNING +Step 5: I-join → MN-B 本地哈希环加载所有节点 → MN-B 哈希环 = {A, B} +Step 6: node-is-ready → managementNodeReady() +Step 7: listen-node-life-cycle-events → 注册远程事件回调 +Step 8: say-I-join → evtf.fire() → 通知 MN-A ← 事件发出 + → MN-A 收到 → @AsyncThread → 线程调度延迟 ← MN-A 哈希环尚未更新 + → nodeJoin(B) → MN-A 哈希环 = {A, B} ← MN-A 哈希环更新 +``` + +**不一致窗口**： + +``` +────────────────────────────────────────────────────► +T0 T1 T2 +MN-B完成iJoin MN-B发送 MN-A @AsyncThread +MN-B哈希环={A,B} NodeJoin事件 nodeJoin(B) + MN-A哈希环→{A,B} + +│←── 不一致窗口：毫秒级 ──►│ + +MN-A 哈希环 = {A} MN-B 哈希环 = {A, B} +``` + +若此窗口内 MN-A 和 MN-B 上同时有同一 VM 的 API 完成： + +``` +MN-A: hash("vm-123") 在 {A} 下 → 必然 = A → GC 提交到 MN-A +MN-B: hash("vm-123") 在 {A,B} 下 → 可能 = B → GC 提交到 MN-B +``` + +**后果（无防护时）**：同一 VM 的两个 GC 落在不同 MN 上，ChainTask 无法跨 MN 串行化。 + +--- + +## 5. 异常情况对比 + +| | MN-B 宕机 | MN-B 加入 | +|---|---|---| +| **窗口长度** | ~30 秒 | 毫秒级 | +| **后果（无防护时）** | **GC 消息丢失** | 同一 VM 可能被两个 MN 同时更新 | +| **严重程度** | **高** | 低 | +| **本方案防护** | 方案 B：reply 失败 → 本地 submit | 方案 C：triggerNow delegate 归集 | + +--- + +## 6. 方案 B：远程提交 + reply 回退（提交层） + +**入口**：`VmMetadataUpdateInterceptor.submitUpdateVmInstanceMetadataGC()` + +```java +private void submitUpdateVmInstanceMetadataGC(String vmInstanceUuid) { + UpdateVmInstanceMetadataGC gc = new UpdateVmInstanceMetadataGC(); + gc.vmInstanceUuid = vmInstanceUuid; + gc.NAME = UpdateVmInstanceMetadataGC.getGCName(vmInstanceUuid); + + SubmitTimeBasedGarbageCollectorMsg gcmsg = new SubmitTimeBasedGarbageCollectorMsg(); + gcmsg.setGc(gc); + gcmsg.setGcInterval(INITIAL_GC_DELAY_SECONDS); + gcmsg.setUnit(TimeUnit.SECONDS); + + // 路由到 hash 环拥有 vmInstanceUuid 的 MN + bus.makeTargetServiceIdByResourceUuid(gcmsg, GCConstants.SERVICE_ID, vmInstanceUuid); + + bus.send(gcmsg, new CloudBusCallBack(null) { + @Override + public void run(MessageReply reply) { + if (!reply.isSuccess()) { + // owner MN 不可达 → 回退到本地 submit，确保 GC 不丢失 + gc.submit(INITIAL_GC_DELAY_SECONDS, TimeUnit.SECONDS); + } + } + }); +} +``` + +**行为**： +- 正常态：GC 通过 hash 环路由到 owner MN，归集到同一 MN +- owner 不可达：reply 超时/失败 → 回退 `gc.submit()` 本地持久化，**GC 不丢失** + +--- + +## 7. 方案 C：triggerNow 路由归集（执行层） + +**入口**：`UpdateVmInstanceMetadataGC.triggerNow()` + +```java +@Override +protected void triggerNow(GCCompletion completion) { + if (!destMaker.isManagedByUs(vmInstanceUuid)) { + delegateToOwner(completion); // 非 owner → 委托给 owner + return; + } + // owner → 正常 ChainTask 执行... +} +``` + +**delegateToOwner**：发送 `SubmitTimeBasedGarbageCollectorMsg` 到 owner MN，成功则本地 Done，失败则指数退避重试。 + +--- + +## 8. MN 宕机 → 接管 → 恢复 时序 + +``` +T0 MN-A 运行 GC-X for vmA + DB: {uuid:GC-X, managementNodeUuid:MN-A, status:Idle} + +T1 MN-A 进程死亡 + +T2 MN-B 心跳检测 → 删除 ManagementNodeVO(MN-A) + FK ON DELETE SET NULL → GC-X.managementNodeUuid = NULL + +T3 MN-B 孤儿扫描 (每60s) → loadFromVO(GC-X): + 乐观锁: UPDATE SET managementNodeUuid=MN-B WHERE managementNodeUuid IS NULL + updated=1 → 认领成功 → setupTimer() + +T4 MN-A 恢复加入 hash 环 + MN-A 孤儿扫描 → GC-X.managementNodeUuid=MN-B (非NULL) → 跳过 + +T5 GC-X 在 MN-B triggerNow() + → isManagedByUs=true 或 delegateToOwner → 最终执行 ✓ +``` + +**接管延迟**：最长 = 心跳超时判定(~30s) + 孤儿扫描间隔(60s) ≈ 90s + +**优化**：`GarbageCollectorManagerImpl` 实现 `ManagementNodeChangeListener`，在 `nodeLeft()` 时立即触发 `loadOrphanJobs()`，将延迟降到 ~31s。 diff --git a/docs/modules/developer/pages/vm-metadata-gc-03-message-flow.md b/docs/modules/developer/pages/vm-metadata-gc-03-message-flow.md new file mode 100644 index 00000000000..26d009b4db8 --- /dev/null +++ b/docs/modules/developer/pages/vm-metadata-gc-03-message-flow.md @@ -0,0 +1,109 @@ +# 虚拟机元数据 GC — 消息流与串行化 + +## 1. 完整消息流 + +``` +API (e.g. StartVmInstanceMsg) 成功 + ↓ +VmMetadataUpdateInterceptor.beforePublishEvent() + ↓ +submitUpdateVmInstanceMetadataGC(vmUuid) + ↓ SubmitTimeBasedGarbageCollectorMsg → hash 环路由到 owner MN + ↓ （reply 失败 → 本地 submit 回退，方案 B） + ↓ +GarbageCollectorManagerImpl.handle(SubmitTimeBasedGarbageCollectorMsg) + → gc.submit() → saveToDb() + setupTimer() + ↓ timer 到期 + ↓ +UpdateVmInstanceMetadataGC.triggerNow() + ├─ 非 owner → delegateToOwner() → 方案 C + └─ 是 owner ↓ + ↓ Layer 1 ChainTask "update-vm-{vmUuid}-metadata" (maxPending=1) + ↓ +bus.send(UpdateVmInstanceMetadataMsg) → makeLocalServiceId（本地处理） + ↓ +VmInstanceBase.handle(UpdateVmInstanceMetadataMsg) + ↓ Layer 2 ChainTask "handle-update-vm-{vmUuid}-metadata" + ↓ buildVmInstanceMetadata(vmUuid) — 从 DB 全量读取 + ↓ +bus.send(UpdateVmInstanceMetadataOnPrimaryStorageMsg) → makeLocalServiceId（本地处理） + ↓ +NFS/LocalStorage.handle() + ↓ 查询 hostUuid → 转发 UpdateVmInstanceMetadataOnHypervisorMsg + ↓ makeTargetServiceIdByResourceUuid(hostUuid) ← 保留 hash 环路由 + ↓ +HostBase.handle() → HTTP call to KVM agent + ↓ +成功 → bus.reply → GC.success() → Done +失败 → bus.reply → GC.onUpdateFail() → 指数退避 → setupTimer() +``` + +--- + +## 2. 消息路由策略 + +| 消息 | 路由方式 | 原因 | +|------|----------|------| +| `SubmitTimeBasedGarbageCollectorMsg` | `makeTargetServiceIdByResourceUuid(vmUuid)` | hash 环归集，GC 提交到 owner MN | +| `UpdateVmInstanceMetadataMsg` | **`makeLocalServiceId`** | GC 已在 owner MN 上，本地处理即可 | +| `UpdateVmInstanceMetadataOnPrimaryStorageMsg` | **`makeLocalServiceId`** | PS handler 无本地状态依赖，避免跨 MN 开销 | +| `UpdateVmInstanceMetadataOnHypervisorMsg` | `makeTargetServiceIdByResourceUuid(hostUuid)` | 需路由到 host-owner MN 发 HTTP 请求 | + +前两跳消息从"可能跨 MN"变成"确定性本地"，减少最多 2 次跨 MN 消息传递。第三跳保留 hash 环路由到正确的 host-owner MN。 + +--- + +## 3. 四层串行化保证 + +``` +Layer 0 — Owner 归集（方案 C） + triggerNow() 检查 isManagedByUs，非 owner 通过 delegate 转移到 owner MN + ⇒ 同一 VM 的 GC 执行归集到一个 MN + +Layer 1 — GC 框架 + 同一 GC 实例的 runTrigger() 通过 lockJob CAS 防止并发 + ⇒ 单个 GC 不会被并行触发 + +Layer 2 — ChainTask 队列 "update-vm-{vmUuid}-metadata" + syncLevel=1, maxPendingTasks=1 + ⇒ 同一 VM 最多 1 个正在执行 + 1 个排队 + +Layer 3 — 主存储级队列 "update-metadata-on-ps-{psUuid}" + ⇒ 同一存储最多 N 个并发写入 +``` + +--- + +## 4. 正常态时序验证 + +### 4.1 hash → MN-B + +``` +MN-A: API 成功 → send(SubmitGCMsg) → MN-B → gc.submit() → GC 在 MN-B +MN-B: API 成功 → send(SubmitGCMsg) → MN-B → gc.submit() → GC 在 MN-B +→ 全在 MN-B ✓ +``` + +### 4.2 MN-B 宕机（~30 秒窗口） + +``` +T0: MN-B 宕机 +T5: MN-A API 成功 → send(SubmitGCMsg → MN-B) → reply 超时 + → 回退: gc.submit() 本地 → GC 在 MN-A（DB 已持久化） +T15: GC triggerNow() → hash 环仍 {A,B} → 非 owner → delegateToOwner → 超时 + → onUpdateFail() → retryCount=1, 退避 20s +T35: GC triggerNow() → hash 环已变 {A} → isManagedByUs=true + → ChainTask → UpdateVmInstanceMetadataMsg → 成功 ✓ +``` + +### 4.3 MN-B 加入（毫秒级窗口） + +``` +T0: MN-B iJoin，MN-A hash 环尚未更新 +T0: MN-A GC → hash {A} → A → 本地；MN-B GC → hash {A,B} → B → 本地 +T0+ms: MN-A 收到 NodeJoin → hash 环变 {A, B} +T10: GC-A triggerNow() on MN-A → hash={A,B} → 非 owner → delegateToOwner → MN-B + → GC-A Done ✓ +T10: GC-B triggerNow() on MN-B → owner → 执行 → 成功 ✓ +→ 自动归集到 MN-B ✓ +``` diff --git a/docs/modules/developer/pages/vm-metadata-gc-04-extreme-cases.md b/docs/modules/developer/pages/vm-metadata-gc-04-extreme-cases.md new file mode 100644 index 00000000000..d8c302e20b2 --- /dev/null +++ b/docs/modules/developer/pages/vm-metadata-gc-04-extreme-cases.md @@ -0,0 +1,122 @@ +# 虚拟机元数据 GC — 极端情况分析 + +## 1. delegate 消耗 retryCount + +**场景**：`delegateToOwner()` 失败时调用 `onUpdateFail()`，递增 `retryCount`。 + +``` +T10: triggerNow() → delegate MN-B → 超时 → retryCount=1 +T30: triggerNow() → delegate MN-B → 超时 → retryCount=2 +T70: triggerNow() → hash 环已变 {A} → isManagedByUs=true → 正常执行 ✓ +``` + +**结论：不是问题。** 心跳检测超时约 30 秒，指数退避序列 10s→20s→40s，在第 2~3 次重试时 hash 环已修正。消耗 1~2 次 retryCount，剩余 3~4 次用于实际更新失败，足够。 + +--- + +## 2. delegate 成功后 owner MN 立即宕机 + +**场景**：GC 委托到 MN-B 成功（GC-3 持久化在 MN-B），随后 MN-B 宕机。 + +``` +T10: GC-1 on MN-A → delegateToOwner → MN-B gc.submit() → GC-1 Done + GC-3 在 MN-B 上（DB 已持久化, managementNodeUuid=MN-B） +T11: MN-B 宕机 +T~40: MN-A 检测到 MN-B 死亡 → 删除 ManagementNodeVO(B) + → FK ON DELETE SET NULL → GC-3.managementNodeUuid=NULL +T~40+scan: MN-A loadOrphanJobs() → 发现 GC-3 → loadFromVO → 认领 +T~50: GC-3 triggerNow() on MN-A → isManagedByUs=true → 执行 ✓ +``` + +**结论：不是问题。** 已持久化的 GC 通过 FK 级联 SET NULL + 孤儿扫描被 MN-A 接管。 + +相关 DB 约束： +```sql +ALTER TABLE GarbageCollectorVO + ADD CONSTRAINT fkGarbageCollectorVOManagementNodeVO + FOREIGN KEY (managementNodeUuid) REFERENCES ManagementNodeVO (uuid) + ON DELETE SET NULL; +``` + +--- + +## 3. delegation 循环（hash 环反复变化） + +**场景**：GC 在 MN-A 和 MN-B 之间反复 delegate。 + +``` +T0: hash=B → GC-1 on MN-A delegate → GC-3 on MN-B, GC-1 Done +T10: hash=A → GC-3 on MN-B delegate → GC-4 on MN-A, GC-3 Done +T20: hash=B → GC-4 on MN-A delegate → GC-5 on MN-B, GC-4 Done +``` + +分析： +- 每次 delegation 创建新 GC（retryCount=0），旧 GC 标记 Done +- 任意时刻只有 **1 个活跃 GC**（前一个已 Done） +- hash 环变化需要 MN 拓扑变更（运维操作），不会自发反复变化 + +**结论：理论存在，实际不可能。** MN 拓扑变更是运维操作，不会在 10 秒内反复发生。 + +--- + +## 4. 本地 MN 在 bus.send 和 callback 之间崩溃 + +**场景**：`Interceptor.submitUpdateVmInstanceMetadataGC()` 中 `bus.send()` 发出消息后、callback 执行前，本地 MN 崩溃。 + +``` +Interceptor: + bus.send(gcmsg, callback) ← 消息已发出 + // MN-A 崩溃，callback 永远不会执行 +``` + +两种可能： +- **远程 MN-B 收到并 submit 成功** → GC 在 MN-B 上，不丢 ✓ +- **远程 MN-B 也不可达** → 消息丢失，GC 未创建 ✗ + +**结论：唯一真正的丢失窗口。** 但与改动前行为一致（原方案 fire-and-forget 也有同样窗口），窗口极短（毫秒级），无法避免。 + +--- + +## 5. 大量 GC 同时 delegate + +**场景**：短时间内 100 个 API 触发 100 个 GC，部分在非 owner MN 上 delegate。 + +``` +MN-A 上 50 个 GC → delegate 到 MN-B +MN-B 上累计 100 个 GC → triggerNow() +→ ChainTask maxPendingTasks=1 → 最多 1 running + 1 pending +→ 98 个 GC 立即标记 Done（exceedMaxPendingCallback） +``` + +**结论：不是问题。** ChainTask 天然限流。delegate 只增加 GC 行数，不增加执行并发。 + +--- + +## 6. delegateToOwner 中 makeDestination 返回自己 + +**场景**：`isManagedByUs()` 和 `makeDestination()` 两次调用之间 hash 环变化。 + +``` +triggerNow(): + → isManagedByUs("vm-123") = false ← 第一次读 + → delegateToOwner() + → owner = makeDestination("vm-123") = 自己 ← hash 环已变 + → send SubmitGCMsg to 自己 → gc.submit() → GC-3 在本地 + → GC-1 Done + → GC-3 triggerNow() → isManagedByUs=true → 执行 ✓ +``` + +**结论：不是问题。** 多了一轮间接，但功能正确。 + +--- + +## 7. 汇总表 + +| # | 极端情况 | 是否丢 GC | 归集效果 | 需要额外处理 | +|---|----------|-----------|----------|--------------| +| 1 | delegate 消耗 retryCount | 否 | ✓ | 否 | +| 2 | delegate 成功后 owner 宕机 | 否（FK SET NULL + 孤儿扫描） | ✓ | 否 | +| 3 | delegation 循环 | 否 | ✓ | 否（实际不会发生） | +| 4 | 本地 MN 在 send 和 callback 之间崩溃 | 极小概率丢 | — | 否（与改动前一致） | +| 5 | 大量 GC 同时 delegate | 否 | ✓ | 否（ChainTask 限流） | +| 6 | delegate 目标是自己 | 否 | ✓ | 否（多一轮间接但正确） | diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIGetVmInstanceMetadataFromPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/APIGetVmInstanceMetadataFromPrimaryStorageMsg.java new file mode 100644 index 00000000000..aab3976a182 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/APIGetVmInstanceMetadataFromPrimaryStorageMsg.java @@ -0,0 +1,36 @@ +package org.zstack.header.storage.primary; + +import org.springframework.http.HttpMethod; +import org.zstack.header.message.APIParam; +import org.zstack.header.message.APISyncCallMessage; +import org.zstack.header.rest.RestRequest; + + +@RestRequest( + path = "/primary-storage/vm-instances/metadata", + method = HttpMethod.GET, + responseClass = APIGetVmInstanceMetadataFromPrimaryStorageReply.class +) +public class APIGetVmInstanceMetadataFromPrimaryStorageMsg extends APISyncCallMessage implements PrimaryStorageMessage { + @APIParam(resourceType = PrimaryStorageVO.class) + private String uuid; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public String getPrimaryStorageUuid() { + return uuid; + } + + public static APIGetVmInstanceMetadataFromPrimaryStorageMsg __example__() { + APIGetVmInstanceMetadataFromPrimaryStorageMsg msg = new APIGetVmInstanceMetadataFromPrimaryStorageMsg(); + msg.setUuid(uuid()); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIGetVmInstanceMetadataFromPrimaryStorageReply.java b/header/src/main/java/org/zstack/header/storage/primary/APIGetVmInstanceMetadataFromPrimaryStorageReply.java new file mode 100644 index 00000000000..5e2dfa123fe --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/APIGetVmInstanceMetadataFromPrimaryStorageReply.java @@ -0,0 +1,26 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.APIReply; +import org.zstack.header.rest.RestResponse; + +import java.util.ArrayList; +import java.util.List; + + +@RestResponse(allTo = "all") +public class APIGetVmInstanceMetadataFromPrimaryStorageReply extends APIReply { + private List vmInstanceMetadata = new ArrayList<>(); + + public List getVmInstanceMetadata() { + return vmInstanceMetadata; + } + + public void setVmInstanceMetadata(List vmInstanceMetadata) { + this.vmInstanceMetadata = vmInstanceMetadata; + } + + public static APIGetVmInstanceMetadataFromPrimaryStorageReply __example__() { + APIGetVmInstanceMetadataFromPrimaryStorageReply reply = new APIGetVmInstanceMetadataFromPrimaryStorageReply(); + return reply; + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceEventDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceEventDoc_zh_cn.groovy new file mode 100644 index 00000000000..972cbf23154 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceEventDoc_zh_cn.groovy @@ -0,0 +1,32 @@ +package org.zstack.header.storage.primary + +import org.zstack.header.vm.VmInstanceInventory +import org.zstack.header.errorcode.ErrorCode + +doc { + + title "注册虚拟机返回" + + ref { + name "inventory" + path "org.zstack.header.storage.primary.APIRegisterVmInstanceEvent.inventory" + desc "null" + type "VmInstanceInventory" + since "4.10.0" + clz VmInstanceInventory.class + } + field { + name "success" + desc "" + type "boolean" + since "4.10.0" + } + ref { + name "error" + path "org.zstack.header.storage.primary.APIRegisterVmInstanceEvent.error" + desc "错误码，若不为null，则表示操作失败, 操作成功时该字段为null" + type "ErrorCode" + since "4.10.0" + clz ErrorCode.class + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceMsg.java b/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceMsg.java new file mode 100644 index 00000000000..4b2c4e80778 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceMsg.java @@ -0,0 +1,63 @@ +package org.zstack.header.storage.primary; + +import org.springframework.http.HttpMethod; +import org.zstack.header.cluster.ClusterVO; +import org.zstack.header.host.HostVO; +import org.zstack.header.message.APIMessage; +import org.zstack.header.message.APIParam; +import org.zstack.header.rest.RestRequest; + +@RestRequest( + path = "/vm-instances/register", + method = HttpMethod.POST, + responseClass = APIRegisterVmInstanceReply.class, + parameterName = "params" +) +public class APIRegisterVmInstanceMsg extends APIMessage implements PrimaryStorageMessage { + @APIParam() + private String metadataPath; + @APIParam(resourceType = PrimaryStorageVO.class) + private String primaryStorageUuid; + @APIParam(resourceType = ClusterVO.class) + private String clusterUuid; + @APIParam(required = false, resourceType = HostVO.class) + private String hostUuid; + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } + + public String getClusterUuid() { + return clusterUuid; + } + + public void setClusterUuid(String clusterUuid) { + this.clusterUuid = clusterUuid; + } + + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getMetadataPath() { + return metadataPath; + } + + public void setMetadataPath(String metadataPath) { + this.metadataPath = metadataPath; + } + + public static APIRegisterVmInstanceMsg __example__() { + APIRegisterVmInstanceMsg msg = new APIRegisterVmInstanceMsg(); + return msg; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceMsgDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceMsgDoc_zh_cn.groovy new file mode 100644 index 00000000000..9772948e4dd --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceMsgDoc_zh_cn.groovy @@ -0,0 +1,83 @@ +package org.zstack.header.storage.primary + +doc { + title "RegisterVmInstance" + + category "storage.primary" + + desc """注册虚拟机""" + + rest { + request { + url "POST /v1/vm-instances/register" + + header (Authorization: 'OAuth the-session-uuid') + + clz APIRegisterVmInstanceMsg.class + + desc """""" + + params { + + column { + name "primaryStorageUuid" + enclosedIn "params" + desc "主存储UUID" + location "body" + type "String" + optional false + since "4.10.0" + } + column { + name "clusterUuid" + enclosedIn "params" + desc "集群UUID" + location "body" + type "String" + optional false + since "4.10.0" + } + column { + name "hostUuid" + enclosedIn "params" + desc "物理机UUID" + location "body" + type "String" + optional true + since "4.10.0" + } + column { + name "metadataPath" + enclosedIn "params" + desc "" + location "body" + type "String" + optional false + since "4.10.0" + } + column { + name "systemTags" + enclosedIn "" + desc "系统标签" + location "body" + type "List" + optional true + since "4.10.0" + } + column { + name "userTags" + enclosedIn "" + desc "用户标签" + location "body" + type "List" + optional true + since "4.10.0" + } + } + } + + response { + clz APIRegisterVmInstanceReply.class + } + } +} \ No newline at end of file diff --git a/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceReply.java b/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceReply.java new file mode 100644 index 00000000000..eb5c1fdda69 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/APIRegisterVmInstanceReply.java @@ -0,0 +1,103 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.allocator.HostAllocatorConstant; +import org.zstack.header.message.APIEvent; +import org.zstack.header.rest.RestResponse; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmInstanceInventory; +import org.zstack.header.vm.VmInstanceState; +import org.zstack.header.vm.VmNicInventory; +import org.zstack.header.volume.VolumeInventory; +import org.zstack.header.volume.VolumeState; +import org.zstack.header.volume.VolumeStatus; +import org.zstack.header.volume.VolumeType; +import org.zstack.utils.data.SizeUnit; + +import java.sql.Timestamp; + +import static java.util.Arrays.asList; + +@RestResponse(allTo = "inventory") +public class APIRegisterVmInstanceReply extends APIEvent { + private VmInstanceInventory inventory; + + public APIRegisterVmInstanceReply() { + } + + public APIRegisterVmInstanceReply(String apiId) { + super(apiId); + } + + public VmInstanceInventory getInventory() { + return inventory; + } + + public void setInventory(VmInstanceInventory inventory) { + this.inventory = inventory; + } + + public static APIRegisterVmInstanceReply __example__() { + APIRegisterVmInstanceReply event = new APIRegisterVmInstanceReply(); + + + String defaultL3Uuid = uuid(); + String rootVolumeUuid = uuid(); + + VmInstanceInventory vm = new VmInstanceInventory(); + vm.setName("Test-VM"); + vm.setUuid(uuid()); + vm.setAllocatorStrategy(HostAllocatorConstant.LAST_HOST_PREFERRED_ALLOCATOR_STRATEGY_TYPE); + vm.setClusterUuid(uuid()); + vm.setCpuNum(1); + vm.setCreateDate(new Timestamp(org.zstack.header.message.DocUtils.date)); + vm.setDefaultL3NetworkUuid(defaultL3Uuid); + vm.setDescription("web server VM"); + vm.setHostUuid(uuid()); + vm.setHypervisorType("KVM"); + vm.setImageUuid(uuid()); + vm.setInstanceOfferingUuid(uuid()); + vm.setLastHostUuid(uuid()); + vm.setMemorySize(SizeUnit.GIGABYTE.toByte(8)); + vm.setPlatform("Linux"); + vm.setRootVolumeUuid(rootVolumeUuid); + vm.setState(VmInstanceState.Stopped.toString()); + vm.setType(VmInstanceConstant.USER_VM_TYPE); + vm.setLastOpDate(new Timestamp(org.zstack.header.message.DocUtils.date)); + vm.setZoneUuid(uuid()); + + VolumeInventory vol = new VolumeInventory(); + vol.setName(String.format("Root-Volume-For-VM-%s", vm.getUuid())); + vol.setCreateDate(new Timestamp(org.zstack.header.message.DocUtils.date)); + vol.setLastOpDate(new Timestamp(org.zstack.header.message.DocUtils.date)); + vol.setType(VolumeType.Root.toString()); + vol.setUuid(rootVolumeUuid); + vol.setSize(SizeUnit.GIGABYTE.toByte(100)); + vol.setActualSize(SizeUnit.GIGABYTE.toByte(20)); + vol.setDeviceId(0); + vol.setState(VolumeState.Enabled.toString()); + vol.setFormat("qcow2"); + vol.setDiskOfferingUuid(uuid()); + vol.setInstallPath(String.format("/zstack_ps/rootVolumes/acct-36c27e8ff05c4780bf6d2fa65700f22e/vol-%s/%s.qcow2", rootVolumeUuid, rootVolumeUuid)); + vol.setStatus(VolumeStatus.Ready.toString()); + vol.setPrimaryStorageUuid(uuid()); + vol.setVmInstanceUuid(vm.getUuid()); + vol.setRootImageUuid(vm.getImageUuid()); + vm.setAllVolumes(asList(vol)); + + VmNicInventory nic = new VmNicInventory(); + nic.setVmInstanceUuid(vm.getUuid()); + nic.setCreateDate(vm.getCreateDate()); + nic.setLastOpDate(vm.getLastOpDate()); + nic.setDeviceId(0); + nic.setL3NetworkUuid(defaultL3Uuid); + nic.setMac("00:0c:29:bd:99:fc"); + nic.setHypervisorType("KVM"); + nic.setUuid(uuid()); + vm.setVmNics(asList(nic)); + + event.setInventory(vm); + + return event; + } + +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/GetVmInstanceMetadataFromPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/storage/primary/GetVmInstanceMetadataFromPrimaryStorageMsg.java new file mode 100644 index 00000000000..d08098380d9 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/GetVmInstanceMetadataFromPrimaryStorageMsg.java @@ -0,0 +1,17 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.NeedReplyMessage; + + +public class GetVmInstanceMetadataFromPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { + private String primaryStorageUuid; + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/GetVmInstanceMetadataFromPrimaryStorageReply.java b/header/src/main/java/org/zstack/header/storage/primary/GetVmInstanceMetadataFromPrimaryStorageReply.java new file mode 100644 index 00000000000..cfa378a28e9 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/GetVmInstanceMetadataFromPrimaryStorageReply.java @@ -0,0 +1,20 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.MessageReply; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GetVmInstanceMetadataFromPrimaryStorageReply extends MessageReply { + private List vmInstanceMetadata = new ArrayList<>(); + + public List getVmInstanceMetadata() { + return vmInstanceMetadata; + } + + public void setVmInstanceMetadata(List vmInstanceMetadata) { + this.vmInstanceMetadata = vmInstanceMetadata; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataMsg.java b/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataMsg.java new file mode 100644 index 00000000000..ed09f15eb4d --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataMsg.java @@ -0,0 +1,21 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.vm.VmInstanceMessage; + +public class ReadVmInstanceMetadataMsg extends NeedReplyMessage implements VmInstanceMessage { + private String uuid; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public String getVmInstanceUuid() { + return uuid; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataOnHypervisorMsg.java b/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataOnHypervisorMsg.java new file mode 100644 index 00000000000..d5d43cb36ea --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataOnHypervisorMsg.java @@ -0,0 +1,26 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.message.NeedReplyMessage; + +public class ReadVmInstanceMetadataOnHypervisorMsg extends NeedReplyMessage implements HostMessage { + private String hostUuid; + private String metadataPath; + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } + + public String getMetadataPath() { + return metadataPath; + } + + public void setMetadataPath(String metadataPath) { + this.metadataPath = metadataPath; + } + + @Override + public String getHostUuid() { + return hostUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataOnHypervisorReply.java b/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataOnHypervisorReply.java new file mode 100644 index 00000000000..25044b944a6 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataOnHypervisorReply.java @@ -0,0 +1,15 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.MessageReply; + +public class ReadVmInstanceMetadataOnHypervisorReply extends MessageReply { + private String metadata; + + public String getMetadata() { + return metadata; + } + + public void setMetadata(String metadata) { + this.metadata = metadata; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataReply.java b/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataReply.java new file mode 100644 index 00000000000..04462f849ad --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/ReadVmInstanceMetadataReply.java @@ -0,0 +1,15 @@ +package org.zstack.header.storage.primary; + +import org.zstack.header.message.MessageReply; + +public class ReadVmInstanceMetadataReply extends MessageReply { + private String vmMetadata; + + public String getVmMetadata() { + return vmMetadata; + } + + public void setVmMetadata(String vmMetadata) { + this.vmMetadata = vmMetadata; + } +} diff --git a/header/src/main/java/org/zstack/header/storage/primary/RegisterVmInstanceException.java b/header/src/main/java/org/zstack/header/storage/primary/RegisterVmInstanceException.java new file mode 100644 index 00000000000..cef54c1c4e5 --- /dev/null +++ b/header/src/main/java/org/zstack/header/storage/primary/RegisterVmInstanceException.java @@ -0,0 +1,9 @@ +package org.zstack.header.storage.primary; + +public interface RegisterVmInstanceException { + String updateVolumeInstallPath(String installPath); + + String updateVolumeSnapshotInstallPath(String installPath); + + PrimaryStorageType getPrimaryStorageType(); +} diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java index 57b4fab4099..cd6a33b4cd3 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotTree.java @@ -442,4 +442,24 @@ public SnapshotLeaf findSnapshot(String snapshotUuid) { return findSnapshot(arg -> arg.getUuid().equals(snapshotUuid)); } + + public List levelOrderTraversal() { + List result = new ArrayList<>(); + if (this.root == null) { + return result; + } + + Queue queue = new LinkedList<>(); + queue.offer(this.root); + + while (!queue.isEmpty()) { + SnapshotLeaf currentLeaf = queue.poll(); + result.add(currentLeaf.getInventory()); + for (SnapshotLeaf child : currentLeaf.getChildren()) { + queue.offer(child); + } + } + + return result; + } } diff --git a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotVO_.java b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotVO_.java index 2d5abaf4f7f..8aeb5873f6b 100755 --- a/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotVO_.java +++ b/header/src/main/java/org/zstack/header/storage/snapshot/VolumeSnapshotVO_.java @@ -1,9 +1,7 @@ package org.zstack.header.storage.snapshot; -/** - */ - import javax.persistence.metamodel.StaticMetamodel; +import java.sql.Timestamp; @StaticMetamodel(VolumeSnapshotVO.class) public class VolumeSnapshotVO_ extends VolumeSnapshotAO_ { diff --git a/header/src/main/java/org/zstack/header/vm/APIReimageVmInstanceEvent.java b/header/src/main/java/org/zstack/header/vm/APIReimageVmInstanceEvent.java old mode 100755 new mode 100644 diff --git a/header/src/main/java/org/zstack/header/vm/APIReimageVmInstanceEventDoc_zh_cn.groovy b/header/src/main/java/org/zstack/header/vm/APIReimageVmInstanceEventDoc_zh_cn.groovy old mode 100755 new mode 100644 diff --git a/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataMsg.java b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataMsg.java new file mode 100644 index 00000000000..bba392b322a --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataMsg.java @@ -0,0 +1,20 @@ +package org.zstack.header.vm; + +import org.zstack.header.message.NeedReplyMessage; + +public class UpdateVmInstanceMetadataMsg extends NeedReplyMessage implements VmInstanceMessage { + private String uuid; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public String getVmInstanceUuid() { + return uuid; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnHypervisorMsg.java b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnHypervisorMsg.java new file mode 100644 index 00000000000..3c4a3b2700f --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnHypervisorMsg.java @@ -0,0 +1,35 @@ +package org.zstack.header.vm; + +import org.zstack.header.host.HostMessage; +import org.zstack.header.message.NeedReplyMessage; + +public class UpdateVmInstanceMetadataOnHypervisorMsg extends NeedReplyMessage implements HostMessage { + private String metadata; + private String metadataPath; + private String hostUuid; + + public String getMetadata() { + return metadata; + } + + public void setMetadata(String metadata) { + this.metadata = metadata; + } + + public String getMetadataPath() { + return metadataPath; + } + + public void setMetadataPath(String metadataPath) { + this.metadataPath = metadataPath; + } + + @Override + public String getHostUuid() { + return hostUuid; + } + + public void setHostUuid(String hostUuid) { + this.hostUuid = hostUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnHypervisorReply.java b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnHypervisorReply.java new file mode 100644 index 00000000000..755db394fcf --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnHypervisorReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm; + +import org.zstack.header.message.MessageReply; + +public class UpdateVmInstanceMetadataOnHypervisorReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnPrimaryStorageMsg.java b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnPrimaryStorageMsg.java new file mode 100644 index 00000000000..36d7722d8ce --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnPrimaryStorageMsg.java @@ -0,0 +1,35 @@ +package org.zstack.header.vm; + +import org.zstack.header.message.NeedReplyMessage; +import org.zstack.header.storage.primary.PrimaryStorageMessage; + +public class UpdateVmInstanceMetadataOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage { + private String metadata; + private String rootVolumeUuid; + private String primaryStorageUuid; + + public String getMetadata() { + return metadata; + } + + public void setMetadata(String metadata) { + this.metadata = metadata; + } + + public String getRootVolumeUuid() { + return rootVolumeUuid; + } + + public void setRootVolumeUuid(String rootVolumeUuid) { + this.rootVolumeUuid = rootVolumeUuid; + } + + @Override + public String getPrimaryStorageUuid() { + return primaryStorageUuid; + } + + public void setPrimaryStorageUuid(String primaryStorageUuid) { + this.primaryStorageUuid = primaryStorageUuid; + } +} diff --git a/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnPrimaryStorageReply.java b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnPrimaryStorageReply.java new file mode 100644 index 00000000000..91d67fc607f --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataOnPrimaryStorageReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm; + +import org.zstack.header.message.MessageReply; + +public class UpdateVmInstanceMetadataOnPrimaryStorageReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataReply.java b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataReply.java new file mode 100644 index 00000000000..244ca2bcc9a --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/UpdateVmInstanceMetadataReply.java @@ -0,0 +1,6 @@ +package org.zstack.header.vm; + +import org.zstack.header.message.MessageReply; + +public class UpdateVmInstanceMetadataReply extends MessageReply { +} diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceConstant.java b/header/src/main/java/org/zstack/header/vm/VmInstanceConstant.java index 9d0efdd77f1..a2716386957 100755 --- a/header/src/main/java/org/zstack/header/vm/VmInstanceConstant.java +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceConstant.java @@ -96,4 +96,6 @@ enum Capability { String VM_CDROM_OCCUPANT_ISO = "ISO"; String VM_CDROM_OCCUPANT_GUEST_TOOLS = "GuestTools"; + + String VM_META_SUFFIX = "_meta"; } diff --git a/header/src/main/java/org/zstack/header/vm/VmInstanceMetadataDTO.java b/header/src/main/java/org/zstack/header/vm/VmInstanceMetadataDTO.java new file mode 100644 index 00000000000..fa85e4fd455 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/VmInstanceMetadataDTO.java @@ -0,0 +1,108 @@ +package org.zstack.header.vm; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 虚拟机元数据 DTO，用于将 VM 及其关联资源的配置信息序列化到主存储。 + * + *

设计原则

+ *
    + *
  • 保存全量 VO：VO 通过 {@code JSONObjectUtil.toJsonString()} 全字段序列化， + * 恢复端按需选择性还原字段（如跳过 zoneUuid、clusterUuid、hostUuid 等调度字段）。
    • + *
  • ResourceMetadata 内聚：每个资源的 VO JSON + SystemTagVO + ResourceConfigVO + * 打包在一个 {@link ResourceMetadata} 对象中，消除平行 Map。
    • + *
  • 敏感信息编码：SystemTagVO 和 ResourceConfigVO 的 JSON 使用 Base64 编码存储， + * 防止 grep/日志泄露密码、密钥等敏感信息。VO 本身明文存储方便调试查看。
    • + *
+ * + *

结构

+ * 
+ * VmInstanceMetadataDTO
+ *   ├── vm: ResourceMetadata              // VmInstanceVO + tags + configs
+ *   ├── volumes: List<ResourceMetadata>   // VolumeVO + tags + configs
+ *   ├── nics: List<ResourceMetadata>      // VmNicVO + tags + configs
+ *   ├── snapshots: Map<volumeUuid, List<snapshotJson>>
+ *   ├── snapshotGroups: List<groupJson>
+ *   ├── snapshotGroupRefs: List<refJson>
+ *   ├── snapshotReferences: Map<volumeUuid, refJson>
+ *   └── snapshotReferenceTrees: Map<volumeUuid, treeJson>
+ *
+ */ +public class VmInstanceMetadataDTO { + + /** + * 单个资源的元数据：VO 全量 JSON + Base64 编码的 SystemTag/ResourceConfig。 + * + *

示例（VM 资源）：

+ * 
+     * {
+     *   "resourceUuid": "vm-abc123",
+     *   "vo": "{\"uuid\":\"vm-abc123\",\"name\":\"web-server\",\"zoneUuid\":\"zone-1\",...}",
+     *   "systemTags": "WyJ7XCJ1dWlkXCI6Li4ufSIsICJ7XCJ0YWdcIjouLi59Il0=",
+     *   "resourceConfigs": "WyJ7XCJjYXRlZ29yeVwiOi4uLn0iXQ=="
+     * }
+     *
+ */ + public static class ResourceMetadata { + /** 资源 UUID，冗余存储方便索引 */ + public String resourceUuid; + + /** VO 全量 JSON（明文），恢复时按需选择性还原字段 */ + public String vo; + + /** + * 该资源关联的所有 SystemTagVO。 + *

存储格式：List<SystemTagVO JSON> → 整体序列化为 JSON 数组字符串 → Base64 编码。

+ *

解码：Base64 decode → JSON 数组字符串 → 逐条反序列化为 SystemTagVO。

+ */ + public String systemTags; + + /** + * 该资源关联的所有 ResourceConfigVO。 + *

存储格式：List<ResourceConfigVO JSON> → 整体序列化为 JSON 数组字符串 → Base64 编码。

+ */ + public String resourceConfigs; + } + + // ── VM 本体 ── + + /** VM 的 VO + SystemTag + ResourceConfig */ + public ResourceMetadata vm; + + // ── 云盘 ── + + /** 所有云盘（挂载的 + 已卸载但 lastVmInstanceUuid 指向本 VM 的） */ + public List volumes = new ArrayList<>(); + + // ── 网卡 ── + + /** VM 的所有网卡 */ + public List nics = new ArrayList<>(); + + // ── 快照 ── + + /** + * 快照数据。key = volumeUuid，value = 该云盘下所有 VolumeSnapshotVO 的 JSON 列表。 + *

快照 VO 明文存储（不含敏感信息）。

+ */ + public Map> snapshots = new HashMap<>(); + + // ── 快照组 ── + + /** VolumeSnapshotGroupVO JSON 列表 */ + public List snapshotGroups = new ArrayList<>(); + + /** VolumeSnapshotGroupRefVO JSON 列表 */ + public List snapshotGroupRefs = new ArrayList<>(); + + // ── 快照引用 ── + + /** key = volumeUuid, value = VolumeSnapshotReferenceVO JSON */ + public Map snapshotReferences = new HashMap<>(); + + /** key = volumeUuid, value = VolumeSnapshotReferenceTreeVO JSON */ + public Map snapshotReferenceTrees = new HashMap<>(); +} diff --git a/header/src/main/java/org/zstack/header/vm/VmMetadata.java b/header/src/main/java/org/zstack/header/vm/VmMetadata.java new file mode 100644 index 00000000000..92452753804 --- /dev/null +++ b/header/src/main/java/org/zstack/header/vm/VmMetadata.java @@ -0,0 +1,44 @@ +package org.zstack.header.vm; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class VmMetadata { + public String vmInstanceVO; + public List vmSystemTags = new ArrayList<>(); + public List vmResourceConfigs = new ArrayList<>(); + + public List volumeVOs = new ArrayList<>(); + // key = volumeUuid + // value = SystemTag + public Map> volumeSystemTags = new HashMap<>(); + // key = volumeUuid + // value = ResourceConfig + public Map> volumeResourceConfigs = new HashMap<>(); + + public List vmNicVOs = new ArrayList<>(); + // key = nicUuid + // value = SystemTag + public Map> vmNicSystemTags = new HashMap<>(); + // key = nicUuid + // value = ResourceConfig + public Map> vmNicResourceConfigs = new HashMap<>(); + + // key = volumeUuid + // value = List + public Map> volumeSnapshots = new HashMap<>(); + + // VolumeSnapshotGroupVO.toString + public List volumeSnapshotGroupVO = new ArrayList<>(); + // VolumeSnapshotGroupRefVO.toString + public List volumeSnapshotGroupRefVO = new ArrayList<>(); + + // key = volumeUuid + // value = VolumeSnapshotReferenceVO.toString + public Map volumeSnapshotReferenceVO = new HashMap<>(); + // key = volumeUuid + // value = VolumeSnapshotReferenceTreeVO.toString + public Map volumeSnapshotReferenceTreeVO = new HashMap<>(); +} diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index 92e76ede2c5..51e3c4f1dee 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -4716,4 +4716,20 @@ public void setMemoryUsage(long memoryUsage) { this.memoryUsage = memoryUsage; } } + + public static class WriteVmInstanceMetadataCmd extends AgentCommand { + public String metadata; + public String metadataPath; + } + + public static class WriteVmInstanceMetadataRsp extends AgentResponse { + } + + public static class ReadVmInstanceMetadataCmd extends AgentCommand { + public String metadataPath; + } + + public static class ReadVmInstanceMetadataRsp extends AgentResponse { + public String metadata; + } } diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java index 7cd78c36c93..6c845676ca8 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMConstant.java @@ -86,6 +86,9 @@ public interface KVMConstant { String CLEAN_FIRMWARE_FLASH = "/clean/firmware/flash"; String FSTRIM_VM_PATH = "/vm/fstrim"; + String WRITE_VM_INSTANCE_METADATA_PATH = "/vm/metadata/write"; + String READ_VM_INSTANCE_METADATA_PATH = "/vm/metadata/read"; + String ISO_TO = "kvm.isoto"; String ANSIBLE_PLAYBOOK_NAME = "kvm.py"; String ANSIBLE_MODULE_PATH = "ansible/kvm"; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 23d7b1cfe47..86a19c60b2f 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -230,6 +230,8 @@ public class KVMHost extends HostBase implements Host { private String fileDownloadPath; private String fileUploadPath; private String fileDownloadProgressPath; + private String writeVmInstanceMetadataPath; + private String readVmInstanceMetadataPath; public KVMHost(KVMHostVO self, KVMHostContext context) { super(self); @@ -480,6 +482,14 @@ public KVMHost(KVMHostVO self, KVMHostContext context) { ub = UriComponentsBuilder.fromHttpUrl(baseUrl); ub.path(KVMConstant.KVM_HOST_FILE_DOWNLOAD_PROGRESS_PATH); fileDownloadProgressPath = ub.build().toString(); + + ub = UriComponentsBuilder.fromHttpUrl(baseUrl); + ub.path(KVMConstant.WRITE_VM_INSTANCE_METADATA_PATH); + writeVmInstanceMetadataPath = ub.build().toString(); + + ub = UriComponentsBuilder.fromHttpUrl(baseUrl); + ub.path(KVMConstant.READ_VM_INSTANCE_METADATA_PATH); + readVmInstanceMetadataPath = ub.build().toString(); } static { @@ -738,6 +748,10 @@ protected void handleLocalMessage(Message msg) { handle((GetFileDownloadProgressMsg) msg); } else if (msg instanceof RestartKvmAgentMsg) { handle((RestartKvmAgentMsg) msg); + } else if (msg instanceof UpdateVmInstanceMetadataOnHypervisorMsg) { + handle((UpdateVmInstanceMetadataOnHypervisorMsg) msg); + } else if (msg instanceof ReadVmInstanceMetadataOnHypervisorMsg) { + handle((ReadVmInstanceMetadataOnHypervisorMsg) msg); } else { super.handleLocalMessage(msg); } @@ -7309,4 +7323,77 @@ public void fail(ErrorCode errorCode) { } }); } + + private void handle(UpdateVmInstanceMetadataOnHypervisorMsg msg) { + inQueue().name(String.format("update-vmInstance-metadata-on-host-%s", self.getUuid())) + .asyncBackup(msg) + .run(chain -> updateVmInstanceMetadata(msg, new NoErrorCompletion(chain) { + @Override + public void done() { + chain.next(); + } + })); + } + + private void updateVmInstanceMetadata(final UpdateVmInstanceMetadataOnHypervisorMsg msg, final NoErrorCompletion completion) { + UpdateVmInstanceMetadataOnHypervisorReply reply = new UpdateVmInstanceMetadataOnHypervisorReply(); + + checkStatus(); + WriteVmInstanceMetadataCmd cmd = new WriteVmInstanceMetadataCmd(); + cmd.metadata = msg.getMetadata(); + cmd.metadataPath = msg.getMetadataPath(); + new Http<>(writeVmInstanceMetadataPath, cmd, WriteVmInstanceMetadataRsp.class).call(new ReturnValueCompletion(msg) { + @Override + public void success(WriteVmInstanceMetadataRsp ret) { + if (!ret.isSuccess()) { + reply.setError(operr("operation error, because:%s", ret.getError())); + } + bus.reply(msg, reply); + completion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + completion.done(); + } + }); + } + + private void handle(ReadVmInstanceMetadataOnHypervisorMsg msg) { + inQueue().name(String.format("readVmInstanceMetadata-on-host-%s", self.getUuid())) + .asyncBackup(msg) + .run(chain -> readVmInstanceMetadata(msg, new NoErrorCompletion(chain) { + @Override + public void done() { + chain.next(); + } + })); + } + + private void readVmInstanceMetadata(final ReadVmInstanceMetadataOnHypervisorMsg msg, final NoErrorCompletion completion) { + checkStatus(); + ReadVmInstanceMetadataOnHypervisorReply reply = new ReadVmInstanceMetadataOnHypervisorReply(); + ReadVmInstanceMetadataCmd cmd = new ReadVmInstanceMetadataCmd(); + cmd.metadataPath = msg.getMetadataPath(); + new Http<>(readVmInstanceMetadataPath, cmd, ReadVmInstanceMetadataRsp.class).call(new ReturnValueCompletion(msg) { + @Override + public void success(ReadVmInstanceMetadataRsp rsp) { + if (!rsp.isSuccess()) { + reply.setError(operr("operation error, because:%s", rsp.getError())); + } + reply.setMetadata(rsp.metadata); + bus.reply(msg, reply); + completion.done(); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + completion.done(); + } + }); + } } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java index d4665a86a06..08e1d0dcf1b 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java @@ -902,6 +902,8 @@ public void handleLocalMessage(Message msg) { handle((CommitVolumeSnapshotOnPrimaryStorageMsg) msg); } else if (msg instanceof PullVolumeSnapshotOnPrimaryStorageMsg) { handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg); + } else if (msg instanceof UpdateVmInstanceMetadataOnPrimaryStorageMsg) { + handle((UpdateVmInstanceMetadataOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } @@ -3329,4 +3331,23 @@ public void fail(ErrorCode errorCode) { public static class LocalStoragePhysicalCapacityUsage extends PrimaryStorageBase.PhysicalCapacityUsage { public long localStorageUsedSize; } + + private void handle(final UpdateVmInstanceMetadataOnPrimaryStorageMsg msg) { + final String hostUuid = getHostUuidByResourceUuid(msg.getRootVolumeUuid()); + LocalStorageHypervisorFactory f = getHypervisorBackendFactoryByHostUuid(hostUuid); + LocalStorageHypervisorBackend bkd = f.getHypervisorBackend(self); + bkd.handle(msg, hostUuid, new ReturnValueCompletion(msg) { + @Override + public void success(UpdateVmInstanceMetadataOnPrimaryStorageReply returnValue) { + bus.reply(msg, returnValue); + } + + @Override + public void fail(ErrorCode errorCode) { + UpdateVmInstanceMetadataOnPrimaryStorageReply reply = new UpdateVmInstanceMetadataOnPrimaryStorageReply(); + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java index 7760e28de93..7e85d562d8c 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java @@ -7,6 +7,8 @@ import org.zstack.header.image.ImageInventory; import org.zstack.header.storage.primary.*; import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.vm.UpdateVmInstanceMetadataOnPrimaryStorageMsg; +import org.zstack.header.vm.UpdateVmInstanceMetadataOnPrimaryStorageReply; import org.zstack.header.volume.*; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageMsg; import org.zstack.storage.primary.EstimateVolumeTemplateSizeOnPrimaryStorageReply; @@ -121,4 +123,6 @@ public LocalStorageHypervisorBackend(PrimaryStorageVO self) { abstract void handle(CommitVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); abstract void handle(PullVolumeSnapshotOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); + + abstract void handle(UpdateVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); } diff --git a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java index e8d268e518a..7da947744b7 100755 --- a/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java +++ b/plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java @@ -43,10 +43,8 @@ import org.zstack.header.storage.backup.*; import org.zstack.header.storage.primary.*; import org.zstack.header.storage.snapshot.*; +import org.zstack.header.vm.*; import org.zstack.header.vm.VmInstanceSpec.ImageSpec; -import org.zstack.header.vm.VmInstanceState; -import org.zstack.header.vm.VmInstanceVO; -import org.zstack.header.vm.VmInstanceVO_; import org.zstack.header.volume.*; import org.zstack.identity.AccountManager; import org.zstack.kvm.*; @@ -70,6 +68,7 @@ import static org.zstack.core.Platform.inerr; import static org.zstack.core.Platform.multiErr; import static org.zstack.core.Platform.operr; +import static org.zstack.header.vm.VmInstanceConstant.VM_META_SUFFIX; import static org.zstack.utils.CollectionDSL.list; import static org.zstack.utils.CollectionUtils.transformAndRemoveNull; @@ -3797,4 +3796,31 @@ public void fail(ErrorCode errorCode) { } }); } + + @Override + void handle(UpdateVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { + String installPath = Q.New(VolumeVO.class).eq(VolumeVO_.uuid, msg.getRootVolumeUuid()).select(VolumeVO_.installPath).findValue(); + // /vms_ds/rootVolumes/acct-36c27e8ff05c4780bf6d2fa65700f22e/vol-829a91b68e794a03865eab8a5918600a/snapshots/f2c31aeede604917aa8cee24848d8bfa.qcow2 + // /vms_ds/rootVolumes/acct-36c27e8ff05c4780bf6d2fa65700f22e/vol-829a91b68e794a03865eab8a5918600a/829a91b68e794a03865eab8a5918600a.qcow2 + + String path = installPath.replaceFirst("^(.+/vol-[^/]+/).*$", "$1"); + String metadataPath = String.format("%s%s", path, VM_META_SUFFIX); + + UpdateVmInstanceMetadataOnHypervisorMsg umsg = new UpdateVmInstanceMetadataOnHypervisorMsg(); + umsg.setMetadata(msg.getMetadata()); + umsg.setMetadataPath(metadataPath); + umsg.setHostUuid(hostUuid); + bus.makeTargetServiceIdByResourceUuid(umsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(umsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + UpdateVmInstanceMetadataOnPrimaryStorageReply reply = new UpdateVmInstanceMetadataOnPrimaryStorageReply(); + if (!r.isSuccess()) { + reply.setError(Platform.operr("failed to update vm[uuid=%s] on hypervisor.", self.getUuid()) + .withCause(r.getError())); + } + bus.reply(msg, reply); + } + }); + } } diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java index abe9ac152b6..edce3f37f50 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java @@ -40,10 +40,8 @@ import org.zstack.header.storage.snapshot.ShrinkVolumeSnapshotOnPrimaryStorageMsg; import org.zstack.header.storage.snapshot.VolumeSnapshotConstant; import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.vm.*; import org.zstack.header.vm.VmInstanceSpec.ImageSpec; -import org.zstack.header.vm.VmInstanceState; -import org.zstack.header.vm.VmInstanceVO; -import org.zstack.header.vm.VmInstanceVO_; import org.zstack.header.volume.*; import org.zstack.kvm.*; import org.zstack.storage.primary.*; @@ -131,6 +129,8 @@ protected void handleLocalMessage(Message msg) { handle((CommitVolumeSnapshotOnPrimaryStorageMsg) msg); } else if (msg instanceof PullVolumeSnapshotOnPrimaryStorageMsg) { handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg); + } else if (msg instanceof UpdateVmInstanceMetadataOnPrimaryStorageMsg) { + handle((UpdateVmInstanceMetadataOnPrimaryStorageMsg) msg); } else { super.handleLocalMessage(msg); } @@ -1924,4 +1924,30 @@ private String getHostUuidFromVolume(String volumeUuid) { return hostUuid; } + + protected void handle(UpdateVmInstanceMetadataOnPrimaryStorageMsg msg) { + UpdateVmInstanceMetadataOnPrimaryStorageReply reply = new UpdateVmInstanceMetadataOnPrimaryStorageReply(); + + String hostUuid = getHostUuidFromVolume(msg.getRootVolumeUuid()); + if (hostUuid == null || hostUuid.isEmpty()) { + reply.setError(operr("no host found for volume[uuid:%s]", msg.getRootVolumeUuid())); + bus.reply(msg, reply); + return; + } + + final NfsPrimaryStorageBackend backend = getUsableBackend(); + + backend.handle(msg, hostUuid, new ReturnValueCompletion(msg) { + @Override + public void success(UpdateVmInstanceMetadataOnPrimaryStorageReply r) { + bus.reply(msg, r); + } + + @Override + public void fail(ErrorCode errorCode) { + reply.setError(errorCode); + bus.reply(msg, reply); + } + }); + } } diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java index 459023d7c17..a19f2d1d38e 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageBackend.java @@ -7,6 +7,8 @@ import org.zstack.header.image.ImageInventory; import org.zstack.header.storage.primary.*; import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; +import org.zstack.header.vm.UpdateVmInstanceMetadataOnPrimaryStorageMsg; +import org.zstack.header.vm.UpdateVmInstanceMetadataOnPrimaryStorageReply; import org.zstack.header.volume.VolumeStats; import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageMsg; import org.zstack.header.volume.BatchSyncVolumeSizeOnPrimaryStorageReply; @@ -91,6 +93,8 @@ public interface NfsPrimaryStorageBackend { void updateMountPoint(PrimaryStorageInventory pinv, String clusterUuid, String oldMountPoint, String newMountPoint, Completion completion); + void handle(UpdateVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion); + class BitsInfo { private String installPath; private long size; diff --git a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java index 93d3d7aab99..8c1bd334400 100755 --- a/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java +++ b/plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorageKVMBackend.java @@ -35,10 +35,7 @@ import org.zstack.header.storage.primary.*; import org.zstack.header.storage.snapshot.VolumeSnapshotInventory; import org.zstack.header.storage.snapshot.VolumeSnapshotVO; -import org.zstack.header.vm.VmInstanceSpec; -import org.zstack.header.vm.VmInstanceState; -import org.zstack.header.vm.VmInstanceVO; -import org.zstack.header.vm.VmInstanceVO_; +import org.zstack.header.vm.*; import org.zstack.header.volume.*; import org.zstack.identity.AccountManager; import org.zstack.kvm.*; @@ -2051,4 +2048,22 @@ public void run(MessageReply r) { } }); } + + public void handle(UpdateVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion completion) { + UpdateVmInstanceMetadataOnHypervisorMsg umsg = new UpdateVmInstanceMetadataOnHypervisorMsg(); + umsg.setMetadata(msg.getMetadata()); + umsg.setHostUuid(hostUuid); + bus.makeTargetServiceIdByResourceUuid(umsg, HostConstant.SERVICE_ID, hostUuid); + bus.send(umsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + RevertVolumeFromSnapshotOnPrimaryStorageReply reply = new RevertVolumeFromSnapshotOnPrimaryStorageReply(); + if (!r.isSuccess()) { + reply.setError(Platform.operr("failed to update vm[uuid=%s] on hypervisor.", msg.getPrimaryStorageUuid()) + .withCause(r.getError())); + } + bus.reply(msg, reply); + } + }); + } } diff --git a/sdk/src/main/java/org/zstack/sdk/RegisterVmInstanceAction.java b/sdk/src/main/java/org/zstack/sdk/RegisterVmInstanceAction.java new file mode 100644 index 00000000000..1fb295bf5ae --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/RegisterVmInstanceAction.java @@ -0,0 +1,113 @@ +package org.zstack.sdk; + +import java.util.HashMap; +import java.util.Map; +import org.zstack.sdk.*; + +public class RegisterVmInstanceAction extends AbstractAction { + + private static final HashMap parameterMap = new HashMap<>(); + + private static final HashMap nonAPIParameterMap = new HashMap<>(); + + public static class Result { + public ErrorCode error; + public org.zstack.sdk.RegisterVmInstanceResult value; + + public Result throwExceptionIfError() { + if (error != null) { + throw new ApiException( + String.format("error[code: %s, description: %s, details: %s]", error.code, error.description, error.details) + ); + } + + return this; + } + } + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String clusterUuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String zoneUuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String primaryStorageUuid; + + @Param(required = false, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String hostUuid; + + @Param(required = true, nonempty = false, nullElements = false, emptyString = true, noTrim = false) + public java.lang.String metadataPath; + + @Param(required = false) + public java.util.List systemTags; + + @Param(required = false) + public java.util.List userTags; + + @Param(required = false) + public String sessionId; + + @Param(required = false) + public String accessKeyId; + + @Param(required = false) + public String accessKeySecret; + + @Param(required = false) + public String requestIp; + + @NonAPIParam + public long timeout = -1; + + @NonAPIParam + public long pollingInterval = -1; + + + private Result makeResult(ApiResult res) { + Result ret = new Result(); + if (res.error != null) { + ret.error = res.error; + return ret; + } + + org.zstack.sdk.RegisterVmInstanceResult value = res.getResult(org.zstack.sdk.RegisterVmInstanceResult.class); + ret.value = value == null ? new org.zstack.sdk.RegisterVmInstanceResult() : value; + + return ret; + } + + public Result call() { + ApiResult res = ZSClient.call(this); + return makeResult(res); + } + + public void call(final Completion completion) { + ZSClient.call(this, new InternalCompletion() { + @Override + public void complete(ApiResult res) { + completion.complete(makeResult(res)); + } + }); + } + + protected Map getParameterMap() { + return parameterMap; + } + + protected Map getNonAPIParameterMap() { + return nonAPIParameterMap; + } + + protected RestInfo getRestInfo() { + RestInfo info = new RestInfo(); + info.httpMethod = "POST"; + info.path = "/vm-instances/register"; + info.needSession = true; + info.needPoll = true; + info.parameterName = "params"; + return info; + } + +} diff --git a/sdk/src/main/java/org/zstack/sdk/RegisterVmInstanceResult.java b/sdk/src/main/java/org/zstack/sdk/RegisterVmInstanceResult.java new file mode 100644 index 00000000000..49510a84cb9 --- /dev/null +++ b/sdk/src/main/java/org/zstack/sdk/RegisterVmInstanceResult.java @@ -0,0 +1,14 @@ +package org.zstack.sdk; + +import org.zstack.sdk.VmInstanceInventory; + +public class RegisterVmInstanceResult { + public VmInstanceInventory inventory; + public void setInventory(VmInstanceInventory inventory) { + this.inventory = inventory; + } + public VmInstanceInventory getInventory() { + return this.inventory; + } + +} diff --git a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java index b7f8cfbc24d..869e7133c26 100755 --- a/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java +++ b/storage/src/main/java/org/zstack/storage/primary/PrimaryStorageBase.java @@ -14,6 +14,8 @@ import org.zstack.core.cloudbus.CloudBusListCallBack; import org.zstack.core.cloudbus.EventFacade; import org.zstack.core.componentloader.PluginRegistry; +import org.zstack.core.config.GlobalConfig; +import org.zstack.core.config.GlobalConfigDefinition; import org.zstack.core.db.*; import org.zstack.core.db.SimpleQuery.Op; import org.zstack.core.errorcode.ErrorFacade; @@ -27,6 +29,7 @@ import org.zstack.core.trash.TrashType; import org.zstack.core.workflow.FlowChainBuilder; import org.zstack.core.workflow.ShareFlow; +import org.zstack.core.workflow.ShareFlowChain; import org.zstack.header.apimediator.ApiMessageInterceptionException; import org.zstack.header.core.*; import org.zstack.header.core.trash.CleanTrashResult; @@ -49,16 +52,24 @@ import org.zstack.header.storage.primary.PrimaryStorageCanonicalEvent.PrimaryStorageDeletedData; import org.zstack.header.storage.primary.PrimaryStorageCanonicalEvent.PrimaryStorageStatusChangedData; import org.zstack.header.storage.snapshot.*; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupRefVO; +import org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO; +import org.zstack.header.tag.TagDefinition; import org.zstack.header.vm.*; import org.zstack.header.volume.*; +import org.zstack.resourceconfig.BindResourceConfig; import org.zstack.storage.volume.VolumeUtils; +import org.zstack.tag.SystemTag; +import org.zstack.utils.BeanUtils; import org.zstack.utils.CollectionDSL; import org.zstack.utils.DebugUtils; import org.zstack.utils.Utils; +import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.logging.CLogger; import javax.persistence.LockModeType; import javax.persistence.TypedQuery; +import java.lang.reflect.Field; import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -177,6 +188,8 @@ public void setNewAdded(boolean newAdded) { protected abstract void handle(GetVolumeSnapshotEncryptedOnPrimaryStorageMsg msg); + protected abstract void handle(GetVmInstanceMetadataFromPrimaryStorageMsg msg); + public PrimaryStorageBase(PrimaryStorageVO self) { this.self = self; } @@ -935,6 +948,10 @@ protected void handleApiMessage(APIMessage msg) { handle((APICleanUpStorageTrashOnPrimaryStorageMsg) msg); } else if (msg instanceof APIAddStorageProtocolMsg) { handle((APIAddStorageProtocolMsg) msg); + } else if (msg instanceof APIRegisterVmInstanceMsg) { + handle((APIRegisterVmInstanceMsg) msg); + } else if (msg instanceof APIGetVmInstanceMetadataFromPrimaryStorageMsg) { + handle((APIGetVmInstanceMetadataFromPrimaryStorageMsg) msg); } else { bus.dealWithUnknownMessage(msg); } @@ -1812,4 +1829,375 @@ protected ImageCacheVO createTemporaryImageCacheFromVolumeSnapshot(ImageInventor private static String getDeduplicateError(String operationName) { return String.format("an other %s task is running, cancel this operation", operationName); } + + private void handle(APIRegisterVmInstanceMsg msg) { + APIRegisterVmInstanceReply event = new APIRegisterVmInstanceReply(msg.getId()); + thdf.chainSubmit(new ChainTask(msg) { + @Override + public String getSyncSignature() { + return String.format("register-vm-from-%s", msg.getMetadataPath()); + } + + @Override + public void run(SyncTaskChain chain) { + registerVmInstance(msg, new ReturnValueCompletion(chain, msg) { + @Override + public void success(VmInstanceInventory vmInstanceInventory) { + event.setInventory(vmInstanceInventory); + bus.publish(event); + chain.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + bus.publish(event); + chain.next(); + } + }); + } + + @Override + public String getName() { + return String.format("register-vm-from-%s", msg.getMetadataPath()); + } + }); + } + + private void registerVmInstance(APIRegisterVmInstanceMsg msg, ReturnValueCompletion completion) { + FlowChain chain = new ShareFlowChain(); + chain.setName("register-vm-from-metadata"); + chain.then(new ShareFlow() { + VmMetadata vmMetadata; + VmInstanceInventory vmInstanceInventory; + + @Override + public void setup() { + flow(new NoRollbackFlow() { + String __name__ = "read-metadata"; + + @Override + public void run(FlowTrigger trigger, Map data) { + ReadVmInstanceMetadataOnHypervisorMsg umsg = new ReadVmInstanceMetadataOnHypervisorMsg(); + umsg.setHostUuid(msg.getHostUuid()); + umsg.setMetadataPath(msg.getMetadataPath()); + bus.makeTargetServiceIdByResourceUuid(umsg, HostConstant.SERVICE_ID, msg.getHostUuid()); + bus.send(umsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + trigger.fail(operr("failed to update vm[uuid=%s] on hypervisor.", + self.getUuid()).withCause(r.getError())); + return; + } + ReadVmInstanceMetadataOnHypervisorReply reply = r.castReply(); + vmMetadata = JSONObjectUtil.toObject(reply.getMetadata(), VmMetadata.class); + trigger.next(); + } + }); + } + }); + + flow(new NoRollbackFlow() { + String __name__ = "register-volume"; + + @Override + public void run(FlowTrigger trigger, Map data) { + List volumesString = vmMetadata.volumeVOs; + + List volumes = new ArrayList<>(); + volumesString.forEach(v -> volumes.add(JSONObjectUtil.toObject(v, VolumeVO.class))); + + List newVolumes = new ArrayList<>(); + volumes.forEach(v -> { + VolumeVO vo = new VolumeVO(); +// vo.setRootImageUuid(vo.getRootImageUuid()); + vo.setAccountUuid(msg.getSession().getAccountUuid()); + vo.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); + vo.setInstallPath(v.getInstallPath()); + + vo.setCreateDate(v.getCreateDate()); + vo.setDescription(v.getDescription()); + vo.setName(v.getName()); + vo.setSize(v.getSize()); + vo.setActualSize(v.getActualSize()); + vo.setState(v.getState()); + vo.setUuid(v.getUuid()); + vo.setVmInstanceUuid(v.getVmInstanceUuid()); + vo.setType(v.getType()); + vo.setCreateDate(v.getCreateDate()); + vo.setLastOpDate(v.getLastOpDate()); + vo.setDeviceId(v.getDeviceId()); + vo.setStatus(v.getStatus()); + vo.setFormat(v.getFormat()); + vo.setShareable(v.isShareable()); + vo.setVolumeQos(v.getVolumeQos()); + vo.setLastDetachDate(v.getLastDetachDate()); + vo.setLastVmInstanceUuid(v.getLastVmInstanceUuid()); + vo.setLastAttachDate(v.getLastAttachDate()); + vo.setProtocol(v.getProtocol()); + newVolumes.add(vo); + }); + dbf.persistCollection(newVolumes); + trigger.next(); + } + }); + + flow(new NoRollbackFlow() { + String __name__ = "register-snapshot"; + + @Override + public void run(FlowTrigger trigger, Map data) { + // 快照 + vmMetadata.volumeSnapshots.forEach((volumeUuid, snapshotList) -> { + // 一个 volume 有多个快照树 + // key = treeuuid + // value = snapshosts + Map> snapshotsByTreeUuid = new HashMap<>(); + snapshotList.forEach(snapshot -> { + VolumeSnapshotInventory inv = JSONObjectUtil.toObject(snapshot, VolumeSnapshotInventory.class); + if (snapshotsByTreeUuid.containsKey(inv.getTreeUuid())) { + snapshotsByTreeUuid.get(inv.getTreeUuid()).add(inv); + } else { + snapshotsByTreeUuid.put(inv.getTreeUuid(), new ArrayList<>()); + snapshotsByTreeUuid.get(inv.getTreeUuid()).add(inv); + } + }); + + // 遍历每一颗树 + snapshotsByTreeUuid.forEach((treeUuid, snapshots) -> { + //构建快照树 + VolumeSnapshotTree tree = VolumeSnapshotTree.fromInventories(snapshots); + // 层级遍历 快照 + List levelOrderTraversals = tree.levelOrderTraversal(); + // 判断当前树有没有 latest 节点 + boolean treeIsCurrent = levelOrderTraversals.stream().anyMatch(VolumeSnapshotInventory::isLatest); + + // 先创建快照树，VolumeSnapshotVO 外键依赖 VolumeSnapshotTreeVO + VolumeSnapshotTreeVO newTree = new VolumeSnapshotTreeVO(); + newTree.setCurrent(treeIsCurrent); + newTree.setVolumeUuid(volumeUuid); + newTree.setUuid(treeUuid); + newTree.setStatus(VolumeSnapshotTreeStatus.Completed); + dbf.persist(newTree); + + // 按照层级遍历的快照构建VolumeSnapshotTreeVO + levelOrderTraversals.forEach(snapshot -> { + VolumeSnapshotVO vo = new VolumeSnapshotVO(); + vo.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); + vo.setPrimaryStorageInstallPath(snapshot.getPrimaryStorageInstallPath()); + + vo.setName(snapshot.getName()); + vo.setCreateDate(snapshot.getCreateDate()); + vo.setDescription(snapshot.getDescription()); + vo.setLastOpDate(snapshot.getLastOpDate()); + vo.setParentUuid(snapshot.getParentUuid()); + vo.setState(VolumeSnapshotState.valueOf(snapshot.getState())); + vo.setType(snapshot.getType()); + vo.setVolumeUuid(snapshot.getVolumeUuid()); + vo.setFormat(snapshot.getFormat()); + vo.setUuid(snapshot.getUuid()); + vo.setStatus(VolumeSnapshotStatus.valueOf(snapshot.getStatus())); + vo.setLatest(snapshot.isLatest()); + vo.setSize(snapshot.getSize()); + vo.setVolumeType(snapshot.getVolumeType()); + vo.setTreeUuid(snapshot.getTreeUuid()); + vo.setDistance(snapshot.getDistance()); + dbf.persist(vo); + }); + }); + }); + + // 快照组 + List newGroups = new ArrayList<>(); + vmMetadata.volumeSnapshotGroupVO.forEach(group -> { + VolumeSnapshotGroupVO vo = JSONObjectUtil.toObject(group, VolumeSnapshotGroupVO.class); + vo.setAccountUuid(msg.getSession().getAccountUuid()); + newGroups.add(vo); + }); + dbf.persistCollection(newGroups); + + // 快照组ref + List newGroupRefs = new ArrayList<>(); + vmMetadata.volumeSnapshotGroupRefVO.forEach(group -> { + VolumeSnapshotGroupRefVO vo = JSONObjectUtil.toObject(group, VolumeSnapshotGroupRefVO.class); + newGroupRefs.add(vo); + }); + dbf.persistCollection(newGroupRefs); + + trigger.next(); + } + }); + + flow(new NoRollbackFlow() { + String __name__ = "register-vmInstance"; + + @Override + public void run(FlowTrigger trigger, Map data) { + VmInstanceVO metaVm = JSONObjectUtil.toObject(vmMetadata.vmInstanceVO, VmInstanceVO.class); + VmInstanceVO newVm = new VmInstanceVO(); + + newVm.setClusterUuid(msg.getClusterUuid()); + newVm.setHostUuid(msg.getHostUuid()); + // 寻找有没有cache的tag lv 构建imageCache +// newVm.setImageUuid(); + + newVm.setUuid(metaVm.getUuid()); + newVm.setName(metaVm.getName()); + newVm.setDescription(metaVm.getDescription()); + newVm.setType(metaVm.getType()); + newVm.setHypervisorType(metaVm.getHypervisorType()); + newVm.setCreateDate(metaVm.getCreateDate()); + newVm.setLastOpDate(metaVm.getLastOpDate()); + newVm.setState(metaVm.getState()); + newVm.setRootVolumeUuid(metaVm.getRootVolumeUuid()); + newVm.setInternalId(metaVm.getInternalId()); + newVm.setCpuNum(metaVm.getCpuNum()); + newVm.setCpuSpeed(metaVm.getCpuSpeed()); + newVm.setMemorySize(metaVm.getMemorySize()); + newVm.setReservedMemorySize(metaVm.getReservedMemorySize()); + newVm.setAllocatorStrategy(metaVm.getAllocatorStrategy()); + newVm.setPlatform(metaVm.getPlatform()); + newVm.setArchitecture(metaVm.getArchitecture()); + newVm.setGuestOsType(metaVm.getGuestOsType()); + dbf.persist(newVm); + vmInstanceInventory = VmInstanceInventory.valueOf(newVm); + trigger.next(); +// List vmSystemTags = vmMetadata.vmSystemTags; +// List vmResourceConfigs = vmMetadata.vmResourceConfigs; +// +// try { +// List systemTags = getResourceSystemTagFromSystem(VmInstanceVO.class.getSimpleName()); +// List resourceConfigs = getResourceConfigFromSystem(VmInstanceVO.class.getSimpleName()); +// +// List tagVOS = new ArrayList<>(); +// vmSystemTags.forEach(tag -> { +// List info = asList(tag.split("_")); +// String t = info.get(0); +// Boolean inherent = Boolean.valueOf(info.get(1)); +// String type = info.get(2); +// systemTags.forEach(it -> { +// if (!it.isMatch(t)) { +// return; +// } +// SystemTagVO vo = new SystemTagVO(); +// vo.setTag(t); +// vo.setType(TagType.valueOf(type)); +// vo.setInherent(inherent); +// vo.setResourceType(VmInstanceVO.class.getSimpleName()); +// vo.setResourceUuid(newVm.getUuid()); +// tagVOS.add(vo); +// }); +// }); +// +// List configVOS = new ArrayList<>(); +// vmResourceConfigs.forEach(tag -> { +// List info = asList(tag.split("_")); +// String identity = info.get(0); +// String value = info.get(1); +// resourceConfigs.forEach(it -> { +// if (it.getIdentity() == identity) { +// return; +// } +// ResourceConfigVO vo = new ResourceConfigVO(); +// vo.setCategory(identity); +// vo.setName(identity); +// vo.setValue(value); +// vo.setResourceType(VmInstanceVO.class.getSimpleName()); +// vo.setResourceUuid(newVm.getUuid()); +// configVOS.add(vo); +// }); +// }); +// } catch (IllegalAccessException | InstantiationException e) { +// throw new RuntimeException(e); +// } + } + }); + + done(new FlowDoneHandler(completion) { + @Override + public void handle(Map data) { + completion.success(vmInstanceInventory); + } + }); + + error(new FlowErrorHandler(msg) { + @Override + public void handle(ErrorCode errCode, Map data) { + completion.fail(errCode); + } + }); + } + }).start(); + } + + private List getResourceSystemTagFromSystem(String resourceType) throws IllegalAccessException, InstantiationException { + List systemTags = new ArrayList<>(); + + Set> classes = BeanUtils.reflections.getTypesAnnotatedWith(TagDefinition.class); + for (Class clazz : classes) { + Field[] fields = clazz.getDeclaredFields(); + for (Field field : fields) { + if (!SystemTag.class.isAssignableFrom(field.getType())) { + continue; + } + + SystemTag systemTag = (SystemTag) field.get(clazz.newInstance()); + + if (resourceType.equals(systemTag.getResourceClass().getName())) { + systemTags.add(systemTag); + } + } + } + return systemTags; + } + + private List getResourceConfigFromSystem(String resourceType) throws IllegalAccessException, InstantiationException { + List globalConfigs = new ArrayList<>(); + + Set> classes = BeanUtils.reflections.getTypesAnnotatedWith(GlobalConfigDefinition.class); + for (Class clazz : classes) { + Field[] fields = clazz.getDeclaredFields(); + for (Field field : fields) { + if (!GlobalConfig.class.isAssignableFrom(field.getType())) { + continue; + } + GlobalConfig globalConfig = (GlobalConfig) field.get(clazz.newInstance()); + + BindResourceConfig bindResourceConfig = field.getAnnotation(BindResourceConfig.class); + if (bindResourceConfig == null) { + continue; + } + + List bindResourceConfigs = Arrays.stream(bindResourceConfig.value()).map(Class::getName).collect(Collectors.toList()); + + if (bindResourceConfigs.contains(resourceType)) { + globalConfigs.add(globalConfig); + } + } + } + + return globalConfigs; + } + + private void handle(APIGetVmInstanceMetadataFromPrimaryStorageMsg msg) { + APIGetVmInstanceMetadataFromPrimaryStorageReply reply = new APIGetVmInstanceMetadataFromPrimaryStorageReply(); + + GetVmInstanceMetadataFromPrimaryStorageMsg gmsg = new GetVmInstanceMetadataFromPrimaryStorageMsg(); + gmsg.setPrimaryStorageUuid(msg.getPrimaryStorageUuid()); + bus.makeTargetServiceIdByResourceUuid(gmsg, PrimaryStorageConstant.SERVICE_ID, msg.getPrimaryStorageUuid()); + + bus.send(gmsg, new CloudBusCallBack(msg) { + @Override + public void run(MessageReply r) { + if (!r.isSuccess()) { + reply.setError(r.getError()); + bus.reply(msg, reply); + return; + } + GetVmInstanceMetadataFromPrimaryStorageReply re = r.castReply(); + reply.setVmInstanceMetadata(re.getVmInstanceMetadata()); + bus.reply(msg, reply); + } + }); + } } diff --git a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy index 07c05b73b9e..5ddb04eac93 100644 --- a/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy +++ b/testlib/src/main/java/org/zstack/testlib/ApiHelper.groovy @@ -27406,6 +27406,33 @@ abstract class ApiHelper { } + def registerVmInstance(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.RegisterVmInstanceAction.class) Closure c) { + def a = new org.zstack.sdk.RegisterVmInstanceAction() + a.sessionId = Test.currentEnvSpec?.session?.uuid + c.resolveStrategy = Closure.OWNER_FIRST + c.delegate = a + c() + + + if (System.getProperty("apipath") != null) { + if (a.apiId == null) { + a.apiId = Platform.uuid + } + + def tracker = new ApiPathTracker(a.apiId) + def out = errorOut(a.call()) + def path = tracker.getApiPath() + if (!path.isEmpty()) { + Test.apiPaths[a.class.name] = path.join(" --->\n") + } + + return out + } else { + return errorOut(a.call()) + } + } + + def reimageVmInstance(@DelegatesTo(strategy = Closure.OWNER_FIRST, value = org.zstack.sdk.ReimageVmInstanceAction.class) Closure c) { def a = new org.zstack.sdk.ReimageVmInstanceAction() a.sessionId = Test.currentEnvSpec?.session?.uuid diff --git a/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy b/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy index 94fc178245d..fcfb5a8ff78 100755 --- a/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy +++ b/testlib/src/main/java/org/zstack/testlib/KVMSimulator.groovy @@ -680,5 +680,15 @@ class KVMSimulator implements Simulator { spec.simulator(KVMConstant.KVM_UPDATE_HOSTNAME_PATH) { return new UpdateHostnameRsp() } + + spec.simulator(KVMConstant.WRITE_VM_INSTANCE_METADATA_PATH) { HttpEntity e -> + return new WriteVmInstanceMetadataRsp() + } + + spec.simulator(KVMConstant.READ_VM_INSTANCE_METADATA_PATH) { HttpEntity e -> + def rsp = new ReadVmInstanceMetadataRsp() + rsp.metadata = "{\"vmInstanceVO\":\"{\\\"vmNics\\\":[{\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"l3NetworkUuid\\\":\\\"28d3a9c8e54c48f290ab4f9e52bbb006\\\",\\\"mac\\\":\\\"fa:81:16:b2:32:00\\\",\\\"hypervisorType\\\":\\\"KVM\\\",\\\"deviceId\\\":0,\\\"internalName\\\":\\\"vnic1.0\\\",\\\"driverType\\\":\\\"virtio\\\",\\\"type\\\":\\\"VNIC\\\",\\\"state\\\":\\\"enable\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"usedIps\\\":[],\\\"uuid\\\":\\\"a77234a5a45a4a7caca46d01d746f41f\\\",\\\"resourceType\\\":\\\"VmNicVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.vm.VmNicVO\\\"}],\\\"allVolumes\\\":[{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/7832daf63d9b41d68bd1460c20ed0e0a\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":2,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"isShareable\\\":false,\\\"shadow\\\":{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/7832daf63d9b41d68bd1460c20ed0e0a\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":2,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"isShareable\\\":false},\\\"uuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"resourceName\\\":\\\"volumeName1null\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.volume.VolumeVO\\\"},{\\\"name\\\":\\\"ROOT-for-vmName\\\",\\\"description\\\":\\\"Root volume for VM[uuid:77bc3074f5f4438c836ce6c56bc5a4aa]\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"rootImageUuid\\\":\\\"575591e021b446e4b465e981da3a8d1b\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/bc5ab54cb3d04635923a2a9d0b5fc73f\\\",\\\"type\\\":\\\"Root\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":0,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\",\\\"isShareable\\\":false,\\\"shadow\\\":{\\\"name\\\":\\\"ROOT-for-vmName\\\",\\\"description\\\":\\\"Root volume for VM[uuid:77bc3074f5f4438c836ce6c56bc5a4aa]\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"rootImageUuid\\\":\\\"575591e021b446e4b465e981da3a8d1b\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/bc5ab54cb3d04635923a2a9d0b5fc73f\\\",\\\"type\\\":\\\"Root\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":0,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\",\\\"isShareable\\\":false},\\\"uuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"resourceName\\\":\\\"ROOT-for-vmName\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.volume.VolumeVO\\\"},{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/43436624dc714282913e0a141246629e\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":1,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"isShareable\\\":false,\\\"shadow\\\":{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/43436624dc714282913e0a141246629e\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":1,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"isShareable\\\":false},\\\"uuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"resourceName\\\":\\\"volumeName1null\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.volume.VolumeVO\\\"},{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b711f22ad5c045b6ad1d770d4f301d05\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":3,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"isShareable\\\":false,\\\"shadow\\\":{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b711f22ad5c045b6ad1d770d4f301d05\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":3,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"isShareable\\\":false},\\\"uuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"resourceName\\\":\\\"volumeName1null\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.volume.VolumeVO\\\"}],\\\"vmCdRoms\\\":[{\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"deviceId\\\":0,\\\"name\\\":\\\"vm-77bc3074f5f4438c836ce6c56bc5a4aa-cdRom\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"uuid\\\":\\\"e8a57f5b8c834573b4da822b672740e4\\\",\\\"resourceName\\\":\\\"vm-77bc3074f5f4438c836ce6c56bc5a4aa-cdRom\\\",\\\"resourceType\\\":\\\"VmCdRomVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.vm.cdrom.VmCdRomVO\\\"}],\\\"name\\\":\\\"vmName\\\",\\\"zoneUuid\\\":\\\"d71de3f6981d46c9a2be43e5fcf31021\\\",\\\"clusterUuid\\\":\\\"29f13acb820d4f7f8cd3593b79b742e5\\\",\\\"imageUuid\\\":\\\"575591e021b446e4b465e981da3a8d1b\\\",\\\"hostUuid\\\":\\\"e99debc09c5845fb8ed682320117f4ce\\\",\\\"internalId\\\":1,\\\"lastHostUuid\\\":\\\"e99debc09c5845fb8ed682320117f4ce\\\",\\\"rootVolumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"defaultL3NetworkUuid\\\":\\\"28d3a9c8e54c48f290ab4f9e52bbb006\\\",\\\"type\\\":\\\"UserVm\\\",\\\"hypervisorType\\\":\\\"KVM\\\",\\\"cpuNum\\\":1,\\\"cpuSpeed\\\":0,\\\"memorySize\\\":1073741824,\\\"reservedMemorySize\\\":0,\\\"platform\\\":\\\"Linux\\\",\\\"architecture\\\":\\\"x86_64\\\",\\\"guestOsType\\\":\\\"CentOS\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:45 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\",\\\"state\\\":\\\"Running\\\",\\\"uuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"resourceName\\\":\\\"vmName\\\",\\\"resourceType\\\":\\\"VmInstanceVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.vm.VmInstanceVO\\\"}\",\"vmSystemTags\":[\"{\\\"inherent\\\":false,\\\"uuid\\\":\\\"38a9b4bd1b8b3dfa829d582aafb2ec25\\\",\\\"resourceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"resourceType\\\":\\\"VmInstanceVO\\\",\\\"tag\\\":\\\"syncPorts::77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:45 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:45 AM\\\"}\",\"{\\\"inherent\\\":true,\\\"uuid\\\":\\\"3e984cdb5edb47559a3f907e1d49bfcc\\\",\\\"resourceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"resourceType\\\":\\\"VmInstanceVO\\\",\\\"tag\\\":\\\"additionalQmp\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\"}\",\"{\\\"inherent\\\":false,\\\"uuid\\\":\\\"85237d3a06133523bd84669349040ec5\\\",\\\"resourceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"resourceType\\\":\\\"VmInstanceVO\\\",\\\"tag\\\":\\\"vmPriority::Normal\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\"}\",\"{\\\"inherent\\\":true,\\\"uuid\\\":\\\"b7c5d5e94ba13159ab2c8c65c1d7bc29\\\",\\\"resourceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"resourceType\\\":\\\"VmInstanceVO\\\",\\\"tag\\\":\\\"vmSystemSerialNumber::8ed14f00-50bb-4e9e-9448-e92c0f67e1e1\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\"}\",\"{\\\"inherent\\\":false,\\\"uuid\\\":\\\"d5019730aeba3e57b2f1a3e8d74d0cbc\\\",\\\"resourceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"resourceType\\\":\\\"VmInstanceVO\\\",\\\"tag\\\":\\\"ha::None\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\"}\"],\"vmResourceConfigs\":[\"{\\\"uuid\\\":\\\"8d2f9937a28846aba03fded826c10c73\\\",\\\"resourceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"resourceType\\\":\\\"VmInstanceVO\\\",\\\"name\\\":\\\"nicMultiQueueNum\\\",\\\"description\\\":\\\"default num of queues on virtio nic\\\",\\\"category\\\":\\\"vm\\\",\\\"value\\\":\\\"1\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\"}\"],\"volumeVOs\":[\"{\\\"name\\\":\\\"ROOT-for-vmName\\\",\\\"description\\\":\\\"Root volume for VM[uuid:77bc3074f5f4438c836ce6c56bc5a4aa]\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"rootImageUuid\\\":\\\"575591e021b446e4b465e981da3a8d1b\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/bc5ab54cb3d04635923a2a9d0b5fc73f\\\",\\\"type\\\":\\\"Root\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":0,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\",\\\"isShareable\\\":false,\\\"shadow\\\":{\\\"name\\\":\\\"ROOT-for-vmName\\\",\\\"description\\\":\\\"Root volume for VM[uuid:77bc3074f5f4438c836ce6c56bc5a4aa]\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"rootImageUuid\\\":\\\"575591e021b446e4b465e981da3a8d1b\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/bc5ab54cb3d04635923a2a9d0b5fc73f\\\",\\\"type\\\":\\\"Root\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":0,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\",\\\"isShareable\\\":false},\\\"uuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"resourceName\\\":\\\"ROOT-for-vmName\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.volume.VolumeVO\\\"}\",\"{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b711f22ad5c045b6ad1d770d4f301d05\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":3,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"isShareable\\\":false,\\\"shadow\\\":{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b711f22ad5c045b6ad1d770d4f301d05\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":3,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"isShareable\\\":false},\\\"uuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"resourceName\\\":\\\"volumeName1null\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.volume.VolumeVO\\\"}\",\"{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/7832daf63d9b41d68bd1460c20ed0e0a\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":2,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"isShareable\\\":false,\\\"shadow\\\":{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/7832daf63d9b41d68bd1460c20ed0e0a\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":2,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"isShareable\\\":false},\\\"uuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"resourceName\\\":\\\"volumeName1null\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.volume.VolumeVO\\\"}\",\"{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/43436624dc714282913e0a141246629e\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":1,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"isShareable\\\":false,\\\"shadow\\\":{\\\"name\\\":\\\"volumeName1null\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"installPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/43436624dc714282913e0a141246629e\\\",\\\"type\\\":\\\"Data\\\",\\\"status\\\":\\\"Ready\\\",\\\"size\\\":1073741824,\\\"actualSize\\\":0,\\\"deviceId\\\":1,\\\"format\\\":\\\"qcow2\\\",\\\"state\\\":\\\"Enabled\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"isShareable\\\":false},\\\"uuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"resourceName\\\":\\\"volumeName1null\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.volume.VolumeVO\\\"}\"],\"volumeSystemTags\":{\"b7290c15276b4700af2c1b108b2b62e1\":[\"{\\\"inherent\\\":true,\\\"uuid\\\":\\\"b9874ec02b583538a5603e7eec8c5b69\\\",\\\"resourceUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"tag\\\":\\\"kvm::volume::0x000f59f934d14a68\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\"],\"8d1e76eca52647f5a4544b9ff2d370de\":[\"{\\\"inherent\\\":true,\\\"uuid\\\":\\\"96cb4b006708387b8318f0fd6ae6ab8b\\\",\\\"resourceUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"tag\\\":\\\"kvm::volume::0x000faad0c9ca4231\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:48 AM\\\"}\"],\"ae9f28cb5055498e8661793d204208ba\":[\"{\\\"inherent\\\":true,\\\"uuid\\\":\\\"5ceacd06bf753b0c8abe5bcef9b5a894\\\",\\\"resourceUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"tag\\\":\\\"kvm::volume::0x000fc4ffeaab6e71\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\"],\"db8251e870b14d60ace863a7598cce8b\":[\"{\\\"inherent\\\":true,\\\"uuid\\\":\\\"d53865baa675373a9bf07a6f501eab41\\\",\\\"resourceUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"resourceType\\\":\\\"VolumeVO\\\",\\\"tag\\\":\\\"kvm::volume::0x000fad154165d205\\\",\\\"type\\\":\\\"System\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"}\"]},\"volumeResourceConfigs\":{\"b7290c15276b4700af2c1b108b2b62e1\":[],\"8d1e76eca52647f5a4544b9ff2d370de\":[],\"ae9f28cb5055498e8661793d204208ba\":[],\"db8251e870b14d60ace863a7598cce8b\":[]},\"vmNicVOs\":[\"{\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"l3NetworkUuid\\\":\\\"28d3a9c8e54c48f290ab4f9e52bbb006\\\",\\\"mac\\\":\\\"fa:81:16:b2:32:00\\\",\\\"hypervisorType\\\":\\\"KVM\\\",\\\"deviceId\\\":0,\\\"internalName\\\":\\\"vnic1.0\\\",\\\"driverType\\\":\\\"virtio\\\",\\\"type\\\":\\\"VNIC\\\",\\\"state\\\":\\\"enable\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:46 AM\\\",\\\"usedIps\\\":[],\\\"uuid\\\":\\\"a77234a5a45a4a7caca46d01d746f41f\\\",\\\"resourceType\\\":\\\"VmNicVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.vm.VmNicVO\\\"}\"],\"vmNicSystemTags\":{\"a77234a5a45a4a7caca46d01d746f41f\":[]},\"vmNicResourceConfigs\":{\"a77234a5a45a4a7caca46d01d746f41f\":[]},\"volumeSnapshots\":{\"b7290c15276b4700af2c1b108b2b62e1\":[\"{\\\"uuid\\\":\\\"7832daf63d9b41d68bd1460c20ed0e0a\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"treeUuid\\\":\\\"f8042fb57bb04ebcb0f01bab2abeb5dd\\\",\\\"parentUuid\\\":\\\"a31c3de68ce246538e982e0e5c7d2d73\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a31c3de68ce246538e982e0e5c7d2d73\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":true,\\\"size\\\":1,\\\"distance\\\":4,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\"}\",\"{\\\"uuid\\\":\\\"a31c3de68ce246538e982e0e5c7d2d73\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"treeUuid\\\":\\\"f8042fb57bb04ebcb0f01bab2abeb5dd\\\",\\\"parentUuid\\\":\\\"b5d771aa83584c9c88d9b84147dfc9ad\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b5d771aa83584c9c88d9b84147dfc9ad\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":3,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\"}\",\"{\\\"uuid\\\":\\\"b5d771aa83584c9c88d9b84147dfc9ad\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"treeUuid\\\":\\\"f8042fb57bb04ebcb0f01bab2abeb5dd\\\",\\\"parentUuid\\\":\\\"bcc3ed8070984cd691c62f421aeaa44d\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/bcc3ed8070984cd691c62f421aeaa44d\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":2,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:53 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\"}\",\"{\\\"uuid\\\":\\\"bcc3ed8070984cd691c62f421aeaa44d\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"treeUuid\\\":\\\"f8042fb57bb04ebcb0f01bab2abeb5dd\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":1,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:51 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:53 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\"}\"],\"8d1e76eca52647f5a4544b9ff2d370de\":[\"{\\\"uuid\\\":\\\"04ef6d31675f4ba5816b104920dc3e2c\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"treeUuid\\\":\\\"d4a030087ed3407894c393ee81f0bc3b\\\",\\\"parentUuid\\\":\\\"79caace79a1048d58ea7c0b38815bbd0\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/79caace79a1048d58ea7c0b38815bbd0\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":0,\\\"distance\\\":3,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\"}\",\"{\\\"uuid\\\":\\\"61e2ada0170142bb8b303910a27690aa\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"treeUuid\\\":\\\"d4a030087ed3407894c393ee81f0bc3b\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":0,\\\"distance\\\":1,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:51 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:53 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\"}\",\"{\\\"uuid\\\":\\\"79caace79a1048d58ea7c0b38815bbd0\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"treeUuid\\\":\\\"d4a030087ed3407894c393ee81f0bc3b\\\",\\\"parentUuid\\\":\\\"61e2ada0170142bb8b303910a27690aa\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/61e2ada0170142bb8b303910a27690aa\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":0,\\\"distance\\\":2,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:53 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\"}\",\"{\\\"uuid\\\":\\\"bc5ab54cb3d04635923a2a9d0b5fc73f\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"treeUuid\\\":\\\"d4a030087ed3407894c393ee81f0bc3b\\\",\\\"parentUuid\\\":\\\"04ef6d31675f4ba5816b104920dc3e2c\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/04ef6d31675f4ba5816b104920dc3e2c\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":true,\\\"size\\\":0,\\\"distance\\\":4,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\"}\"],\"ae9f28cb5055498e8661793d204208ba\":[\"{\\\"uuid\\\":\\\"1aaa92b2d1eb4c36bb8951b8e1521b34\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"treeUuid\\\":\\\"055b80b0727e4117b246b1b29f2d58b6\\\",\\\"parentUuid\\\":\\\"aefbe47465c047d1b118321c34425869\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/aefbe47465c047d1b118321c34425869\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":3,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\"}\",\"{\\\"uuid\\\":\\\"69e85ac72fea4263a55cbcd21785006e\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"treeUuid\\\":\\\"055b80b0727e4117b246b1b29f2d58b6\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":1,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:51 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:53 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\"}\",\"{\\\"uuid\\\":\\\"aefbe47465c047d1b118321c34425869\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"treeUuid\\\":\\\"055b80b0727e4117b246b1b29f2d58b6\\\",\\\"parentUuid\\\":\\\"69e85ac72fea4263a55cbcd21785006e\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/69e85ac72fea4263a55cbcd21785006e\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":2,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:53 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\"}\",\"{\\\"uuid\\\":\\\"b711f22ad5c045b6ad1d770d4f301d05\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"treeUuid\\\":\\\"055b80b0727e4117b246b1b29f2d58b6\\\",\\\"parentUuid\\\":\\\"1aaa92b2d1eb4c36bb8951b8e1521b34\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/1aaa92b2d1eb4c36bb8951b8e1521b34\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":true,\\\"size\\\":1,\\\"distance\\\":4,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\"}\"],\"db8251e870b14d60ace863a7598cce8b\":[\"{\\\"uuid\\\":\\\"43436624dc714282913e0a141246629e\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"treeUuid\\\":\\\"1c0773fa98f4465b8e535ba3c00dc039\\\",\\\"parentUuid\\\":\\\"a70bb2be871644b6ad12ac8d6e9524d0\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a70bb2be871644b6ad12ac8d6e9524d0\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":true,\\\"size\\\":1,\\\"distance\\\":4,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\"}\",\"{\\\"uuid\\\":\\\"791f523bc99a4f08bd70b5a59d8ed5c8\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"treeUuid\\\":\\\"1c0773fa98f4465b8e535ba3c00dc039\\\",\\\"parentUuid\\\":\\\"a35b7ae1616a4974b2f80654c5527fbb\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a35b7ae1616a4974b2f80654c5527fbb\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":2,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:53 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\"}\",\"{\\\"uuid\\\":\\\"a35b7ae1616a4974b2f80654c5527fbb\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"treeUuid\\\":\\\"1c0773fa98f4465b8e535ba3c00dc039\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":1,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:51 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:53 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\"}\",\"{\\\"uuid\\\":\\\"a70bb2be871644b6ad12ac8d6e9524d0\\\",\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"type\\\":\\\"Hypervisor\\\",\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"treeUuid\\\":\\\"1c0773fa98f4465b8e535ba3c00dc039\\\",\\\"parentUuid\\\":\\\"791f523bc99a4f08bd70b5a59d8ed5c8\\\",\\\"primaryStorageUuid\\\":\\\"e121a11157bb4746ad3c8d56c3760a3e\\\",\\\"primaryStorageInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/791f523bc99a4f08bd70b5a59d8ed5c8\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"format\\\":\\\"qcow2\\\",\\\"latest\\\":false,\\\"size\\\":1,\\\"distance\\\":3,\\\"state\\\":\\\"Enabled\\\",\\\"status\\\":\\\"Ready\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"backupStorageRefs\\\":[],\\\"groupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\"}\"]},\"volumeSnapshotGroupVO\":[\"{\\\"snapshotCount\\\":4,\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeSnapshotRefs\\\":[{\\\"volumeSnapshotUuid\\\":\\\"a35b7ae1616a4974b2f80654c5527fbb\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":1,\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"69e85ac72fea4263a55cbcd21785006e\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":3,\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"61e2ada0170142bb8b303910a27690aa\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":0,\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeName\\\":\\\"ROOT-for-vmName\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"bcc3ed8070984cd691c62f421aeaa44d\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":2,\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}],\\\"uuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"resourceName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"resourceType\\\":\\\"VolumeSnapshotGroupVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO\\\"}\",\"{\\\"snapshotCount\\\":4,\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeSnapshotRefs\\\":[{\\\"volumeSnapshotUuid\\\":\\\"b5d771aa83584c9c88d9b84147dfc9ad\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":2,\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/bcc3ed8070984cd691c62f421aeaa44d\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"79caace79a1048d58ea7c0b38815bbd0\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":0,\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeName\\\":\\\"ROOT-for-vmName\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/61e2ada0170142bb8b303910a27690aa\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"aefbe47465c047d1b118321c34425869\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":3,\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/69e85ac72fea4263a55cbcd21785006e\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"791f523bc99a4f08bd70b5a59d8ed5c8\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":1,\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a35b7ae1616a4974b2f80654c5527fbb\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"}],\\\"uuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"resourceName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"resourceType\\\":\\\"VolumeSnapshotGroupVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO\\\"}\",\"{\\\"snapshotCount\\\":4,\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeSnapshotRefs\\\":[{\\\"volumeSnapshotUuid\\\":\\\"1aaa92b2d1eb4c36bb8951b8e1521b34\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":3,\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/aefbe47465c047d1b118321c34425869\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"a70bb2be871644b6ad12ac8d6e9524d0\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":1,\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/791f523bc99a4f08bd70b5a59d8ed5c8\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"04ef6d31675f4ba5816b104920dc3e2c\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":0,\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeName\\\":\\\"ROOT-for-vmName\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/79caace79a1048d58ea7c0b38815bbd0\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"a31c3de68ce246538e982e0e5c7d2d73\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":2,\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b5d771aa83584c9c88d9b84147dfc9ad\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}],\\\"uuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"resourceName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"resourceType\\\":\\\"VolumeSnapshotGroupVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO\\\"}\",\"{\\\"snapshotCount\\\":4,\\\"name\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"vmInstanceUuid\\\":\\\"77bc3074f5f4438c836ce6c56bc5a4aa\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeSnapshotRefs\\\":[{\\\"volumeSnapshotUuid\\\":\\\"43436624dc714282913e0a141246629e\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":1,\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a70bb2be871644b6ad12ac8d6e9524d0\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"bc5ab54cb3d04635923a2a9d0b5fc73f\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":0,\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeName\\\":\\\"ROOT-for-vmName\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/04ef6d31675f4ba5816b104920dc3e2c\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"b711f22ad5c045b6ad1d770d4f301d05\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":3,\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/1aaa92b2d1eb4c36bb8951b8e1521b34\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"},{\\\"volumeSnapshotUuid\\\":\\\"7832daf63d9b41d68bd1460c20ed0e0a\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":2,\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a31c3de68ce246538e982e0e5c7d2d73\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}],\\\"uuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"resourceName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"resourceType\\\":\\\"VolumeSnapshotGroupVO\\\",\\\"concreteResourceType\\\":\\\"org.zstack.header.storage.snapshot.group.VolumeSnapshotGroupVO\\\"}\"],\"volumeSnapshotGroupRefVO\":[\"{\\\"volumeSnapshotUuid\\\":\\\"04ef6d31675f4ba5816b104920dc3e2c\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":0,\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeName\\\":\\\"ROOT-for-vmName\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/79caace79a1048d58ea7c0b38815bbd0\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"1aaa92b2d1eb4c36bb8951b8e1521b34\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":3,\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/aefbe47465c047d1b118321c34425869\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"43436624dc714282913e0a141246629e\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":1,\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a70bb2be871644b6ad12ac8d6e9524d0\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"61e2ada0170142bb8b303910a27690aa\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":0,\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeName\\\":\\\"ROOT-for-vmName\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"69e85ac72fea4263a55cbcd21785006e\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":3,\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"7832daf63d9b41d68bd1460c20ed0e0a\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":2,\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a31c3de68ce246538e982e0e5c7d2d73\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"791f523bc99a4f08bd70b5a59d8ed5c8\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":1,\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/a35b7ae1616a4974b2f80654c5527fbb\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"79caace79a1048d58ea7c0b38815bbd0\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":0,\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeName\\\":\\\"ROOT-for-vmName\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/61e2ada0170142bb8b303910a27690aa\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"a31c3de68ce246538e982e0e5c7d2d73\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":2,\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b5d771aa83584c9c88d9b84147dfc9ad\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"a35b7ae1616a4974b2f80654c5527fbb\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":1,\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"a70bb2be871644b6ad12ac8d6e9524d0\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"7648a93930db473785b0abc0e0716c1a\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":1,\\\"volumeUuid\\\":\\\"db8251e870b14d60ace863a7598cce8b\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/791f523bc99a4f08bd70b5a59d8ed5c8\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:56 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:49 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"aefbe47465c047d1b118321c34425869\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":3,\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/69e85ac72fea4263a55cbcd21785006e\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"b5d771aa83584c9c88d9b84147dfc9ad\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"6db066c890d141008e8ff18bd5940d77\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":2,\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/bcc3ed8070984cd691c62f421aeaa44d\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:54 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"b711f22ad5c045b6ad1d770d4f301d05\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":3,\\\"volumeUuid\\\":\\\"ae9f28cb5055498e8661793d204208ba\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/1aaa92b2d1eb4c36bb8951b8e1521b34\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"bc5ab54cb3d04635923a2a9d0b5fc73f\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"d2d486455d6c472cbb7391958edebea5\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":0,\\\"volumeUuid\\\":\\\"8d1e76eca52647f5a4544b9ff2d370de\\\",\\\"volumeName\\\":\\\"ROOT-for-vmName\\\",\\\"volumeType\\\":\\\"Root\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/04ef6d31675f4ba5816b104920dc3e2c\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-ROOT-for-vmName\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:57 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:47 AM\\\"}\",\"{\\\"volumeSnapshotUuid\\\":\\\"bcc3ed8070984cd691c62f421aeaa44d\\\",\\\"volumeSnapshotGroupUuid\\\":\\\"01171364e6704dbe83c36167de52d719\\\",\\\"snapshotDeleted\\\":false,\\\"deviceId\\\":2,\\\"volumeUuid\\\":\\\"b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeName\\\":\\\"volumeName1null\\\",\\\"volumeType\\\":\\\"Data\\\",\\\"volumeSnapshotInstallPath\\\":\\\"sharedblock://e121a11157bb4746ad3c8d56c3760a3e/b7290c15276b4700af2c1b108b2b62e1\\\",\\\"volumeSnapshotName\\\":\\\"group-8d1e76eca52647f5a4544b9ff2d370de-volumeName1null\\\",\\\"createDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"lastOpDate\\\":\\\"Jan 28, 2026 4:15:52 AM\\\",\\\"volumeLastAttachDate\\\":\\\"Jan 28, 2026 4:15:50 AM\\\"}\"],\"volumeSnapshotReferenceVO\":{},\"volumeSnapshotReferenceTreeVO\":{},\"EncryptedResourceKeyRefVO\":{}}" + return rsp + } } }