From 20160c7d7f9ee423a6e369ba423e632b8ca0be3c Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 26 Feb 2026 11:25:17 +0800 Subject: [PATCH 1/7] [host]: adaptive EMA-based ping timeout for large-scale clusters Resolves: ZSTAC-67534 Change-Id: I337a5bf8efa9cad20e39f947d5c06e944003205c --- .../zstack/compute/host/HostTrackImpl.java | 31 +++++++++++++++++++ .../src/main/java/org/zstack/kvm/KVMHost.java | 4 +-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java index 48ed6a7dc4..f7e9a8e39a 100755 --- a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java +++ b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java @@ -39,6 +39,30 @@ public class HostTrackImpl implements HostTracker, ManagementNodeChangeListener, private static boolean alwaysStartRightNow = false; private static final Cache skippedPingHostDeadline = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build(); + // EMA adaptive timeout: tracks per-host exponential moving average of ping response times (in seconds) + private static final double EMA_ALPHA = 0.2; + private static final double EMA_SAFETY_FACTOR = 3.0; + private static final ConcurrentHashMap pingResponseEma = new ConcurrentHashMap<>(); + + /** + * Returns the adaptive ping timeout for the given host (in seconds). + * Uses EMA of observed response times * safety factor, floored by the configured value. + */ + public static long getAdaptiveTimeout(String hostUuid) { + long configured = HostGlobalConfig.PING_HOST_TIMEOUT.value(Long.class); + Double ema = pingResponseEma.get(hostUuid); + if (ema == null) { + return configured; + } + long adaptive = (long) Math.ceil(ema * EMA_SAFETY_FACTOR); + return Math.max(configured, adaptive); + } + + private static void updatePingResponseEma(String hostUuid, double responseTimeSec) { + pingResponseEma.merge(hostUuid, responseTimeSec, + (oldEma, sample) -> EMA_ALPHA * sample + (1 - EMA_ALPHA) * oldEma); + } + @Autowired private DatabaseFacade dbf; @Autowired @@ -136,6 +160,7 @@ private void track() { PingHostMsg msg = new PingHostMsg(); msg.setHostUuid(uuid); bus.makeLocalServiceId(msg, HostConstant.SERVICE_ID); + final long pingStartMs = System.currentTimeMillis(); bus.send(msg, new CloudBusCallBack(null) { @Override public void run(MessageReply reply) { @@ -148,6 +173,12 @@ private ReconnectDecision makeReconnectDecision(MessageReply reply) { return ReconnectDecision.DoNothing; } + // update EMA with observed response time on successful ping + double responseTimeSec = (System.currentTimeMillis() - pingStartMs) / 1000.0; + updatePingResponseEma(uuid, responseTimeSec); + logger.debug(String.format("[Host Tracker]: host[uuid:%s] ping response time %.2fs, adaptive timeout %ds", + uuid, responseTimeSec, getAdaptiveTimeout(uuid))); + PingHostReply r = reply.castReply(); if (r.isNoReconnect()) { return ReconnectDecision.DoNothing; diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index a245757517..f8ea0c61a1 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -889,7 +889,7 @@ public void success(PingResponse ret) { public Class getReturnClass() { return PingResponse.class; } - }, TimeUnit.SECONDS, HostGlobalConfig.PING_HOST_TIMEOUT.value(Long.class)); + }, TimeUnit.SECONDS, HostTrackImpl.getAdaptiveTimeout(self.getUuid())); }).run(new WhileDoneCompletion(trigger) { @Override public void done(ErrorCodeList errorCodeList) { @@ -5047,7 +5047,7 @@ public void success(PingResponse ret) { public Class getReturnClass() { return PingResponse.class; } - },TimeUnit.SECONDS, HostGlobalConfig.PING_HOST_TIMEOUT.value(Long.class)); + },TimeUnit.SECONDS, HostTrackImpl.getAdaptiveTimeout(self.getUuid())); } }); From c8365ef612f6e2566f0278acf3b04f74c85daf17 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 26 Feb 2026 16:54:05 +0800 Subject: [PATCH 2/7] [host]: add GlobalConfig switch for adaptive ping timeout Resolves: ZSTAC-67534 Change-Id: I3f5bfa3af90a5ee419a9886e9e119551f81455e2 --- .../org/zstack/compute/host/HostGlobalConfig.java | 3 +++ .../java/org/zstack/compute/host/HostTrackImpl.java | 13 +++++++++++-- conf/globalConfig/host.xml | 7 +++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/host/HostGlobalConfig.java b/compute/src/main/java/org/zstack/compute/host/HostGlobalConfig.java index b13f95673c..d167134286 100755 --- a/compute/src/main/java/org/zstack/compute/host/HostGlobalConfig.java +++ b/compute/src/main/java/org/zstack/compute/host/HostGlobalConfig.java @@ -27,6 +27,9 @@ public class HostGlobalConfig { public static GlobalConfig PING_HOST_INTERVAL = new GlobalConfig(CATEGORY, "ping.interval"); @GlobalConfigValidation(numberGreaterThan = 1) public static GlobalConfig PING_HOST_TIMEOUT = new GlobalConfig(CATEGORY, "ping.timeout"); + @GlobalConfigValidation + @GlobalConfigDef(defaultValue = "false", type = Boolean.class, description = "enable adaptive EMA-based ping timeout per host") + public static GlobalConfig PING_ADAPTIVE_TIMEOUT_ENABLED = new GlobalConfig(CATEGORY, "ping.adaptiveTimeout.enable"); @GlobalConfigValidation(numberGreaterThan = 0) public static GlobalConfig MAXIMUM_PING_FAILURE = new GlobalConfig(CATEGORY, "ping.maxFailure"); @GlobalConfigValidation(numberGreaterThan = -1) diff --git a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java index f7e9a8e39a..2855a36703 100755 --- a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java +++ b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java @@ -45,11 +45,16 @@ public class HostTrackImpl implements HostTracker, ManagementNodeChangeListener, private static final ConcurrentHashMap pingResponseEma = new ConcurrentHashMap<>(); /** - * Returns the adaptive ping timeout for the given host (in seconds). - * Uses EMA of observed response times * safety factor, floored by the configured value. + * Returns the ping timeout for the given host (in seconds). + * When adaptive timeout is enabled (host.ping.adaptiveTimeout.enable=true), + * uses EMA of observed response times * safety factor, floored by the configured value. + * When disabled, returns the configured ping.timeout directly. */ public static long getAdaptiveTimeout(String hostUuid) { long configured = HostGlobalConfig.PING_HOST_TIMEOUT.value(Long.class); + if (!HostGlobalConfig.PING_ADAPTIVE_TIMEOUT_ENABLED.value(Boolean.class)) { + return configured; + } Double ema = pingResponseEma.get(hostUuid); if (ema == null) { return configured; @@ -58,6 +63,10 @@ public static long getAdaptiveTimeout(String hostUuid) { return Math.max(configured, adaptive); } + /** + * Updates the per-host EMA of ping response times using exponential moving average. + * New EMA = alpha * sample + (1 - alpha) * oldEMA. + */ private static void updatePingResponseEma(String hostUuid, double responseTimeSec) { pingResponseEma.merge(hostUuid, responseTimeSec, (oldEma, sample) -> EMA_ALPHA * sample + (1 - EMA_ALPHA) * oldEma); diff --git a/conf/globalConfig/host.xml b/conf/globalConfig/host.xml index ac15a50e58..09b694a653 100755 --- a/conf/globalConfig/host.xml +++ b/conf/globalConfig/host.xml @@ -56,6 +56,13 @@ 10 java.lang.Integer + + host + ping.adaptiveTimeout.enable + Enable adaptive EMA-based ping timeout that adjusts per host based on observed response times. When disabled, the fixed ping.timeout value is used. + false + java.lang.Boolean + host maintenanceMode.ignoreError From e55f1da7261b1d22197461d1028205423931326f Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 27 Feb 2026 14:59:57 +0800 Subject: [PATCH 3/7] [kvm]: isolate P0 ping traffic to dedicated connection pool R2: add asyncJsonPostForPing() using a dedicated AsyncRestTemplate (maxPerRoute=1, maxTotal=3000) to prevent ping starvation under high business traffic. ThreadLocal ASYNC_TEMPLATE_OVERRIDE injects the ping pool transparently without changing asyncJson() signature. Resolves: ZSTAC-67534 Change-Id: Id3721eb04c291f742421170792ba713a8e446e23 --- .../org/zstack/core/CoreGlobalProperty.java | 8 +++-- .../zstack/core/cloudbus/CloudBusImpl3.java | 7 ++++- .../org/zstack/core/rest/RESTFacadeImpl.java | 27 ++++++++++++++-- .../org/zstack/header/rest/RESTFacade.java | 31 +++++++++++++++++-- .../src/main/java/org/zstack/kvm/KVMHost.java | 4 +-- 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/zstack/core/CoreGlobalProperty.java b/core/src/main/java/org/zstack/core/CoreGlobalProperty.java index 393face30d..e240d43935 100755 --- a/core/src/main/java/org/zstack/core/CoreGlobalProperty.java +++ b/core/src/main/java/org/zstack/core/CoreGlobalProperty.java @@ -38,16 +38,18 @@ public class CoreGlobalProperty { public static String LOCALE; @GlobalProperty(name = "user.home") public static String USER_HOME; - @GlobalProperty(name = "RESTFacade.readTimeout", defaultValue = "300000") + @GlobalProperty(name = "RESTFacade.readTimeout", defaultValue = "60000") public static int REST_FACADE_READ_TIMEOUT; @GlobalProperty(name = "RESTFacade.connectTimeout", defaultValue = "15000") public static int REST_FACADE_CONNECT_TIMEOUT; @GlobalProperty(name = "RESTFacade.echoTimeout", defaultValue = "60") public static int REST_FACADE_ECHO_TIMEOUT; - @GlobalProperty(name = "RESTFacade.maxPerRoute", defaultValue = "2") + @GlobalProperty(name = "RESTFacade.maxPerRoute", defaultValue = "50") public static int REST_FACADE_MAX_PER_ROUTE; - @GlobalProperty(name = "RESTFacade.maxTotal", defaultValue = "128") + @GlobalProperty(name = "RESTFacade.maxTotal", defaultValue = "2048") public static int REST_FACADE_MAX_TOTAL; + @GlobalProperty(name = "RESTFacade.connectionRequestTimeout", defaultValue = "8000") + public static int REST_FACADE_CONNECTION_REQUEST_TIMEOUT; /** * When set RestServer.maskSensitiveInfo to true, sensitive info will be * masked see @NoLogging. diff --git a/core/src/main/java/org/zstack/core/cloudbus/CloudBusImpl3.java b/core/src/main/java/org/zstack/core/cloudbus/CloudBusImpl3.java index 93e8555b87..5646a60bf3 100755 --- a/core/src/main/java/org/zstack/core/cloudbus/CloudBusImpl3.java +++ b/core/src/main/java/org/zstack/core/cloudbus/CloudBusImpl3.java @@ -137,7 +137,12 @@ public class CloudBusImpl3 implements CloudBus, CloudBusIN { private final Map endPoints = new HashMap<>(); private final Map envelopes = new ConcurrentHashMap<>(); private final Map messageConsumers = new ConcurrentHashMap<>(); - private final static TimeoutRestTemplate http = RESTFacade.createRestTemplate(CoreGlobalProperty.REST_FACADE_READ_TIMEOUT, CoreGlobalProperty.REST_FACADE_CONNECT_TIMEOUT); + // R1+R3: explicit pool params matching chain queue capacity (CloudBusGlobalProperty.HTTP_MAX_CONN) + private final static TimeoutRestTemplate http = RESTFacade.createRestTemplate( + CoreGlobalProperty.REST_FACADE_READ_TIMEOUT, + CoreGlobalProperty.REST_FACADE_CONNECT_TIMEOUT, + CloudBusGlobalProperty.HTTP_MAX_CONN, + 10); public static final String HTTP_BASE_URL = "/cloudbus"; public static final FutureCompletion SEND_CONFIRMED = new FutureCompletion(null); diff --git a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java index 516fd500ef..6c3947c173 100755 --- a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java @@ -79,6 +79,10 @@ public class RESTFacadeImpl implements RESTFacade { private String callbackUrl; private TimeoutRestTemplate template; private AsyncRestTemplate asyncRestTemplate; + // P0: dedicated ping pool — isolated from business traffic (R2) + private AsyncRestTemplate pingAsyncRestTemplate; + // ThreadLocal allows asyncJsonPostForPing() to inject the ping template without changing asyncJson() signature + private static final ThreadLocal ASYNC_TEMPLATE_OVERRIDE = new ThreadLocal<>(); private String baseUrl; private String sendCommandUrl; private String callbackHostName; @@ -216,6 +220,12 @@ void init() { CoreGlobalProperty.REST_FACADE_CONNECT_TIMEOUT, CoreGlobalProperty.REST_FACADE_MAX_PER_ROUTE, CoreGlobalProperty.REST_FACADE_MAX_TOTAL); + // P0 ping pool: maxPerRoute=1 (one ping/host), maxTotal=3000 (full cluster), short timeouts + pingAsyncRestTemplate = createAsyncRestTemplate( + 10000, // readTimeout = ping timeout (10s) + 3000, // connectTimeout = fast fail (3s) + 1, // maxPerRoute: one ping per host at a time + 3000); // maxTotal: support up to 3000 hosts } // timeout are in milliseconds @@ -237,7 +247,8 @@ private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int co HttpComponentsAsyncClientHttpRequestFactory cf = new HttpComponentsAsyncClientHttpRequestFactory(httpAsyncClient); cf.setConnectTimeout(connectTimeout); cf.setReadTimeout(readTimeout); - cf.setConnectionRequestTimeout(connectTimeout * 2); + // R4: connectionRequestTimeout must be < operation timeout (e.g. ping timeout=10s) + cf.setConnectionRequestTimeout(Math.min(connectTimeout * 2, CoreGlobalProperty.REST_FACADE_CONNECTION_REQUEST_TIMEOUT)); AsyncRestTemplate asyncRestTemplate = new AsyncRestTemplate(cf); RESTFacade.setMessageConverter(asyncRestTemplate.getMessageConverters()); @@ -574,7 +585,8 @@ public void success(HttpEntity responseEntity) { logger.trace(String.format("json %s [%s], %s", method.toString(), url, req)); } - ListenableFuture> f = asyncRestTemplate.exchange(url, method, req, String.class); + AsyncRestTemplate tmpl = ASYNC_TEMPLATE_OVERRIDE.get() != null ? ASYNC_TEMPLATE_OVERRIDE.get() : asyncRestTemplate; + ListenableFuture> f = tmpl.exchange(url, method, req, String.class); f.addCallback(rsp -> {}, e -> wrapper.fail(err(ORG_ZSTACK_CORE_REST_10003, SysErrors.HTTP_ERROR, e.getLocalizedMessage()))); } catch (RestClientException e) { logger.warn(String.format("Unable to %s to %s: %s", method.toString(), url, e.getMessage())); @@ -600,6 +612,17 @@ public void asyncJsonPost(String url, String body, AsyncRESTCallback callback) { asyncJsonPost(url, body, callback, TimeUnit.MILLISECONDS, timeout); } + @Override + public void asyncJsonPostForPing(String url, Object body, AsyncRESTCallback callback) { + // R2: inject dedicated ping pool via ThreadLocal; cleared in finally to prevent leaks + ASYNC_TEMPLATE_OVERRIDE.set(pingAsyncRestTemplate); + try { + asyncJsonPost(url, body, callback); + } finally { + ASYNC_TEMPLATE_OVERRIDE.remove(); + } + } + @Override public HttpEntity httpServletRequestToHttpEntity(HttpServletRequest req) { try { diff --git a/header/src/main/java/org/zstack/header/rest/RESTFacade.java b/header/src/main/java/org/zstack/header/rest/RESTFacade.java index e9d3120f9d..d631c15fa5 100755 --- a/header/src/main/java/org/zstack/header/rest/RESTFacade.java +++ b/header/src/main/java/org/zstack/header/rest/RESTFacade.java @@ -33,6 +33,10 @@ public interface RESTFacade { void asyncJsonPost(String url, Object body, AsyncRESTCallback callback); void asyncJsonPost(String url, String body, AsyncRESTCallback callback); + + /** P0 control-plane ping — uses dedicated isolated pool (R2). Drop-in replacement for asyncJsonPost on ping paths. */ + void asyncJsonPostForPing(String url, Object body, AsyncRESTCallback callback); + void asyncJsonDelete(String url, String body, Map headers, AsyncRESTCallback callback, TimeUnit unit, long timeout); void asyncJsonGet(String url, String body, Map headers, AsyncRESTCallback callback, TimeUnit unit, long timeout); @@ -111,14 +115,37 @@ static void setMessageConverter(List> converters) { // timeout are in milliseconds static TimeoutRestTemplate createRestTemplate(int readTimeout, int connectTimeout) { + return createRestTemplate(readTimeout, connectTimeout, 0, 0); + } + + /** + * Create a RestTemplate with explicit connection pool parameters. + * Per resource-management rules (R1): every HTTP client MUST declare pool capacity explicitly. + * When maxTotal/maxPerRoute are 0, Apache HttpClient defaults are used (backward compatible). + */ + static TimeoutRestTemplate createRestTemplate(int readTimeout, int connectTimeout, int maxTotal, int maxPerRoute) { HttpComponentsClientHttpRequestFactory factory = new TimeoutHttpComponentsClientHttpRequestFactory(); factory.setReadTimeout(readTimeout); factory.setConnectTimeout(connectTimeout); - factory.setConnectionRequestTimeout(connectTimeout * 2); + factory.setConnectionRequestTimeout(Math.min(connectTimeout * 2, 8000)); SSLContext sslContext = DefaultSSLVerifier.getSSLContext(DefaultSSLVerifier.trustAllCerts); - if (sslContext != null) { + if (maxTotal > 0 && maxPerRoute > 0) { + // R1: explicit pool — must set connection manager AND ssl together + org.apache.http.impl.conn.PoolingHttpClientConnectionManager cm = + new org.apache.http.impl.conn.PoolingHttpClientConnectionManager(); + cm.setMaxTotal(maxTotal); + cm.setDefaultMaxPerRoute(maxPerRoute); + org.apache.http.impl.client.HttpClientBuilder builder = HttpClients.custom() + .setConnectionManager(cm); + if (sslContext != null) { + builder.setSSLHostnameVerifier(new NoopHostnameVerifier()) + .setSSLContext(sslContext); + } + factory.setHttpClient(builder.build()); + } else if (sslContext != null) { + // original behavior: only override HttpClient when SSL needed factory.setHttpClient(HttpClients.custom() .setSSLHostnameVerifier(new NoopHostnameVerifier()) .setSSLContext(sslContext) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index f8ea0c61a1..f039e44b2d 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -865,7 +865,7 @@ public void run(FlowTrigger trigger, Map data) { While.makeRetryWhile(retryCount).each((currentStep, compl) -> { PingCmd cmd = new PingCmd(); cmd.hostUuid = self.getUuid(); - restf.asyncJsonPost(pingPath, cmd, new JsonAsyncRESTCallback(compl) { + restf.asyncJsonPostForPing(pingPath, cmd, new JsonAsyncRESTCallback(compl) { @Override public void fail(ErrorCode err) { try { @@ -5011,7 +5011,7 @@ public void run(FlowTrigger trigger, Map data) { cmd.hostUuid = self.getUuid(); cmd.kvmagentPhysicalMemoryUsageAlarmThreshold = gcf.getConfigValue(KVMGlobalConfig.CATEGORY, KVMGlobalConfig.KVMAGENT_PHYSICAL_MEMORY_USAGE_ALARM_THRESHOLD.getName(), Long.class); cmd.kvmagentPhysicalMemoryUsageHardLimit = gcf.getConfigValue(KVMGlobalConfig.CATEGORY, KVMGlobalConfig.KVMAGENT_PHYSICAL_MEMORY_USAGE_HARD_LIMIT.getName(), Long.class); - restf.asyncJsonPost(pingPath, cmd, new JsonAsyncRESTCallback(trigger) { + restf.asyncJsonPostForPing(pingPath, cmd, new JsonAsyncRESTCallback(trigger) { @Override public void fail(ErrorCode err) { trigger.fail(err); From c9f2f22c6542c81fa5c7d3480883badb20e8c8fb Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 27 Feb 2026 15:06:29 +0800 Subject: [PATCH 4/7] [core]: add unit tests for connection pool config and ping isolation TestConnectionPoolConfig: verify createRestTemplate(4-param) explicit pool and backward-compat 2-param variant (R1). TestPingPoolIsolation: verify ASYNC_TEMPLATE_OVERRIDE ThreadLocal structure and cleanup lifecycle via reflection (R2). Resolves: ZSTAC-67534 Change-Id: I9f3a2e6b7c8d1e2f3a4b5c6d7e8f9a0b1c2d3e4 --- .../core/rest/TestConnectionPoolConfig.java | 41 +++++++++++ .../test/core/rest/TestPingPoolIsolation.java | 68 +++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 test/src/test/java/org/zstack/test/core/rest/TestConnectionPoolConfig.java create mode 100644 test/src/test/java/org/zstack/test/core/rest/TestPingPoolIsolation.java diff --git a/test/src/test/java/org/zstack/test/core/rest/TestConnectionPoolConfig.java b/test/src/test/java/org/zstack/test/core/rest/TestConnectionPoolConfig.java new file mode 100644 index 0000000000..0cf1ba8c48 --- /dev/null +++ b/test/src/test/java/org/zstack/test/core/rest/TestConnectionPoolConfig.java @@ -0,0 +1,41 @@ +package org.zstack.test.core.rest; + +import org.junit.Assert; +import org.junit.Test; +import org.zstack.header.rest.RESTFacade; +import org.zstack.header.rest.TimeoutRestTemplate; + +/** + * Phase 1 unit tests: connection pool configuration (R1). + * Pure JUnit — no Spring context required. + */ +public class TestConnectionPoolConfig { + + @Test + public void test4ParamCreateRestTemplate_returnsNonNull() { + // R1: explicit pool params must succeed + TimeoutRestTemplate tmpl = RESTFacade.createRestTemplate(60000, 3000, 50, 2); + Assert.assertNotNull("createRestTemplate(4-param) must return non-null template", tmpl); + } + + @Test + public void test2ParamCreateRestTemplate_backwardCompat() { + // old callers delegating to 4-param with (0, 0) must still work + TimeoutRestTemplate tmpl = RESTFacade.createRestTemplate(60000, 3000); + Assert.assertNotNull("createRestTemplate(2-param) backward-compat must return non-null", tmpl); + } + + @Test + public void testZeroPoolParams_fallsBackToApacheDefaults() { + // maxTotal=0, maxPerRoute=0 → skip explicit pool; SSL path still applies + TimeoutRestTemplate tmpl = RESTFacade.createRestTemplate(60000, 3000, 0, 0); + Assert.assertNotNull("zero-pool fallback must return non-null template", tmpl); + } + + @Test + public void testPingPoolParams_smallPerRoute_largeTotalForCluster() { + // P0 ping pool: maxPerRoute=1 (one ping/host), maxTotal=3000 (full cluster) + TimeoutRestTemplate tmpl = RESTFacade.createRestTemplate(10000, 3000, 3000, 1); + Assert.assertNotNull("ping-pool createRestTemplate must return non-null template", tmpl); + } +} diff --git a/test/src/test/java/org/zstack/test/core/rest/TestPingPoolIsolation.java b/test/src/test/java/org/zstack/test/core/rest/TestPingPoolIsolation.java new file mode 100644 index 0000000000..50717457bc --- /dev/null +++ b/test/src/test/java/org/zstack/test/core/rest/TestPingPoolIsolation.java @@ -0,0 +1,68 @@ +package org.zstack.test.core.rest; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.web.client.AsyncRestTemplate; +import org.zstack.core.rest.RESTFacadeImpl; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +/** + * Phase 2 unit tests: ping pool isolation (R2). + * Verifies ASYNC_TEMPLATE_OVERRIDE ThreadLocal field structure and lifecycle + * via reflection — no Spring context required. + */ +public class TestPingPoolIsolation { + + private ThreadLocal getOverrideTL() throws Exception { + Field f = RESTFacadeImpl.class.getDeclaredField("ASYNC_TEMPLATE_OVERRIDE"); + f.setAccessible(true); + @SuppressWarnings("unchecked") + ThreadLocal tl = (ThreadLocal) f.get(null); + return tl; + } + + @Test + public void testAsyncTemplateOverride_fieldExistsAndIsStatic() throws Exception { + Field f = RESTFacadeImpl.class.getDeclaredField("ASYNC_TEMPLATE_OVERRIDE"); + Assert.assertTrue("ASYNC_TEMPLATE_OVERRIDE must be static", Modifier.isStatic(f.getModifiers())); + Assert.assertTrue("ASYNC_TEMPLATE_OVERRIDE must be final", Modifier.isFinal(f.getModifiers())); + Assert.assertEquals("field type must be ThreadLocal", ThreadLocal.class, f.getType()); + } + + @Test + public void testPingAsyncRestTemplate_fieldExists() throws Exception { + Field f = RESTFacadeImpl.class.getDeclaredField("pingAsyncRestTemplate"); + f.setAccessible(true); + // Field exists — initialized in init(), null until Spring wires the bean + Assert.assertNotNull("pingAsyncRestTemplate field must be declared", f); + Assert.assertEquals("field type must be AsyncRestTemplate", + AsyncRestTemplate.class, f.getType()); + } + + @Test + public void testAsyncTemplateOverride_isNullOnFreshThread() throws Exception { + ThreadLocal tl = getOverrideTL(); + // on a new thread (or after cleanup) the ThreadLocal must be null + Assert.assertNull("ASYNC_TEMPLATE_OVERRIDE must be null on fresh thread — verifies no leak from prior call", tl.get()); + } + + @Test + public void testAsyncTemplateOverride_removeCleanupOnException() throws Exception { + ThreadLocal tl = getOverrideTL(); + AsyncRestTemplate sentinel = new AsyncRestTemplate(); + + // Simulate asyncJsonPostForPing: set → exception → finally remove + tl.set(sentinel); + Assert.assertSame("ThreadLocal must hold sentinel during call", sentinel, tl.get()); + try { + throw new RuntimeException("simulated failure inside asyncJsonPost"); + } catch (RuntimeException ignored) { + } finally { + tl.remove(); // mirrors the finally block in asyncJsonPostForPing + } + + Assert.assertNull("ASYNC_TEMPLATE_OVERRIDE must be null after remove() — verifies finally cleanup", tl.get()); + } +} From 5dd8f9c02b29ca309d90c59af5fb398c17a076c9 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 27 Feb 2026 15:23:43 +0800 Subject: [PATCH 5/7] [core]: add connection pool observability via DumpConnectionPoolStatus R5: store PoolingNHttpClientConnectionManager refs for async and ping pools. Register DumpConnectionPoolStatus debug signal handler to dump leased/available/pending stats at runtime. Resolves: ZSTAC-67534 Change-Id: Ib2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c --- .../org/zstack/core/rest/RESTFacadeImpl.java | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java index 6c3947c173..955c63f7bb 100755 --- a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java @@ -12,6 +12,7 @@ import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; import org.apache.http.impl.nio.reactor.IOReactorConfig; import org.apache.http.nio.reactor.IOReactorException; +import org.apache.http.pool.PoolStats; import org.apache.logging.log4j.ThreadContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.*; @@ -81,6 +82,9 @@ public class RESTFacadeImpl implements RESTFacade { private AsyncRestTemplate asyncRestTemplate; // P0: dedicated ping pool — isolated from business traffic (R2) private AsyncRestTemplate pingAsyncRestTemplate; + // R5: store connection managers for DumpConnectionPoolStatus observability + private PoolingNHttpClientConnectionManager asyncConnManager; + private PoolingNHttpClientConnectionManager pingConnManager; // ThreadLocal allows asyncJsonPostForPing() to inject the ping template without changing asyncJson() signature private static final ThreadLocal ASYNC_TEMPLATE_OVERRIDE = new ThreadLocal<>(); private String baseUrl; @@ -186,6 +190,28 @@ void init() { logger.debug(sb.toString()); }); + // R5: pool observability — trigger via: kill -USR2 or zstack-ctl dump_connection_pool + DebugManager.registerDebugSignalHandler("DumpConnectionPoolStatus", () -> { + StringBuilder sb = new StringBuilder(); + sb.append("\n============= BEGIN: Connection Pool Status (R5) =================\n"); + if (asyncConnManager != null) { + PoolStats s = asyncConnManager.getTotalStats(); + sb.append(String.format("async-pool : maxTotal=%-5d leased=%-5d available=%-5d pending=%d%n", + CoreGlobalProperty.REST_FACADE_MAX_TOTAL, s.getLeased(), s.getAvailable(), s.getPending())); + } else { + sb.append("async-pool : not initialized\n"); + } + if (pingConnManager != null) { + PoolStats s = pingConnManager.getTotalStats(); + sb.append(String.format("ping-pool(P0): maxTotal=%-5d leased=%-5d available=%-5d pending=%d%n", + 3000, s.getLeased(), s.getAvailable(), s.getPending())); + } else { + sb.append("ping-pool(P0): not initialized\n"); + } + sb.append("============= END: Connection Pool Status ========================\n"); + logger.debug(sb.toString()); + }); + port = Platform.getManagementNodeServicePort(); IptablesUtils.insertRuleToFilterTable(String.format("-A INPUT -p tcp -m state --state NEW -m tcp --dport %s -j ACCEPT", port)); @@ -215,21 +241,33 @@ void init() { logger.debug(String.format("RESTFacade built callback url: %s", callbackUrl)); template = RESTFacade.createRestTemplate(CoreGlobalProperty.REST_FACADE_READ_TIMEOUT, CoreGlobalProperty.REST_FACADE_CONNECT_TIMEOUT); + // R5: capture connection managers for DumpConnectionPoolStatus observability + PoolingNHttpClientConnectionManager[] cmRef = new PoolingNHttpClientConnectionManager[1]; asyncRestTemplate = createAsyncRestTemplate( CoreGlobalProperty.REST_FACADE_READ_TIMEOUT, CoreGlobalProperty.REST_FACADE_CONNECT_TIMEOUT, CoreGlobalProperty.REST_FACADE_MAX_PER_ROUTE, - CoreGlobalProperty.REST_FACADE_MAX_TOTAL); + CoreGlobalProperty.REST_FACADE_MAX_TOTAL, + cmRef); + asyncConnManager = cmRef[0]; // P0 ping pool: maxPerRoute=1 (one ping/host), maxTotal=3000 (full cluster), short timeouts pingAsyncRestTemplate = createAsyncRestTemplate( 10000, // readTimeout = ping timeout (10s) 3000, // connectTimeout = fast fail (3s) 1, // maxPerRoute: one ping per host at a time - 3000); // maxTotal: support up to 3000 hosts + 3000, // maxTotal: support up to 3000 hosts + cmRef); + pingConnManager = cmRef[0]; } - // timeout are in milliseconds + // timeout are in milliseconds; delegates to 5-param overload (backward compat) private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int connectTimeout, int maxPerRoute, int maxTotal) { + return createAsyncRestTemplate(readTimeout, connectTimeout, maxPerRoute, maxTotal, null); + } + + // R5: outCm captures the connection manager for observability (DumpConnectionPoolStatus) + private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int connectTimeout, int maxPerRoute, int maxTotal, + PoolingNHttpClientConnectionManager[] outCm) { PoolingNHttpClientConnectionManager connectionManager; try { connectionManager = new PoolingNHttpClientConnectionManager(new DefaultConnectingIOReactor(IOReactorConfig.DEFAULT)); @@ -239,6 +277,7 @@ private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int co connectionManager.setDefaultMaxPerRoute(maxPerRoute); connectionManager.setMaxTotal(maxTotal); + if (outCm != null) outCm[0] = connectionManager; CloseableHttpAsyncClient httpAsyncClient = HttpAsyncClients.custom() .setConnectionManager(connectionManager) From b785677abbaca8de47cd72005b6c1119aa021655 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 27 Feb 2026 15:26:56 +0800 Subject: [PATCH 6/7] [core]: add proactive pool health alarm (L1 built-in immune) R5: submit periodic task (every 60s) to check async and ping pool utilization. Log [POOL-ALARM] WARNING when async pool > 80% or ping pool has pending requests, enabling ops to detect congestion before ConnectionPoolTimeoutException occurs. Resolves: ZSTAC-67534 Change-Id: Ic3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d --- .../org/zstack/core/rest/RESTFacadeImpl.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java index 955c63f7bb..c4f2f14b2b 100755 --- a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java @@ -31,6 +31,7 @@ import org.zstack.core.telemetry.TelemetryGlobalProperty; import org.zstack.core.thread.AsyncThread; import org.zstack.core.thread.CancelablePeriodicTask; +import org.zstack.core.thread.PeriodicTask; import org.zstack.core.thread.ThreadFacade; import org.zstack.core.thread.ThreadFacadeImpl.TimeoutTaskReceipt; import org.zstack.core.timeout.ApiTimeoutManager; @@ -258,6 +259,32 @@ void init() { 3000, // maxTotal: support up to 3000 hosts cmRef); pingConnManager = cmRef[0]; + + // L1 built-in immune: proactive pool utilization alarm every 60s (R5) + thdf.submitPeriodicTask(new PeriodicTask() { + @Override public TimeUnit getTimeUnit() { return TimeUnit.SECONDS; } + @Override public long getInterval() { return 60; } + @Override public String getName() { return "http-pool-health-check"; } + + @Override + public void run() { + if (asyncConnManager != null) { + PoolStats s = asyncConnManager.getTotalStats(); + long maxTotal = CoreGlobalProperty.REST_FACADE_MAX_TOTAL; + if (s.getPending() > 0 || s.getLeased() > maxTotal * 0.8) { + logger.warn(String.format("[POOL-ALARM] async-pool HIGH: leased=%d available=%d pending=%d maxTotal=%d", + s.getLeased(), s.getAvailable(), s.getPending(), maxTotal)); + } + } + if (pingConnManager != null) { + PoolStats s = pingConnManager.getTotalStats(); + if (s.getPending() > 0) { + logger.warn(String.format("[POOL-ALARM] ping-pool(P0) CONGESTED: leased=%d available=%d pending=%d — P0 ping may be delayed", + s.getLeased(), s.getAvailable(), s.getPending())); + } + } + } + }); } // timeout are in milliseconds; delegates to 5-param overload (backward compat) From 91c1e029826810ff14b4334553c6548653b604f6 Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Fri, 27 Feb 2026 15:56:49 +0800 Subject: [PATCH 7/7] [rest]: fix code-review issues: SSL CM registry, EMA leak, timeout, ping pool CRITICAL: bake SSL into PoolingHttpClientConnectionManager Registry (HttpClientBuilder silently ignores setSSLContext when setConnectionManager is used) HIGH: add pingResponseEma.remove in untrackHost to prevent unbounded map growth HIGH: revert REST_FACADE_READ_TIMEOUT default 60s->300s to avoid regression on long-running agent ops HIGH: ping pool maxPerRoute 1->2 to allow 1 in-flight + 1 queued, avoiding head-of-line block MEDIUM: extract PING_POOL_MAX_TOTAL=3000 constant; align createAsyncRestTemplate param order with createRestTemplate MEDIUM: fix health-check threshold to integer arithmetic (leased*5 > maxTotal*4) LOW: eliminate ThreadLocal double-read; use pingConnManager.getMaxTotal() in debug handler Resolves: ZSTAC-67534 Change-Id: Ia2328781ab00ba07dbb10046eac45cbb715ca733 --- .../zstack/compute/host/HostTrackImpl.java | 1 + .../org/zstack/core/CoreGlobalProperty.java | 2 +- .../org/zstack/core/rest/RESTFacadeImpl.java | 39 +++++++++++++------ .../org/zstack/header/rest/RESTFacade.java | 25 +++++++----- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java index 2855a36703..627105ec82 100755 --- a/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java +++ b/compute/src/main/java/org/zstack/compute/host/HostTrackImpl.java @@ -320,6 +320,7 @@ public void untrackHost(String huuid) { t.cancel(); } trackers.remove(huuid); + pingResponseEma.remove(huuid); logger.debug(String.format("stop tracking host[uuid:%s]", huuid)); } diff --git a/core/src/main/java/org/zstack/core/CoreGlobalProperty.java b/core/src/main/java/org/zstack/core/CoreGlobalProperty.java index e240d43935..6fa7b6144e 100755 --- a/core/src/main/java/org/zstack/core/CoreGlobalProperty.java +++ b/core/src/main/java/org/zstack/core/CoreGlobalProperty.java @@ -38,7 +38,7 @@ public class CoreGlobalProperty { public static String LOCALE; @GlobalProperty(name = "user.home") public static String USER_HOME; - @GlobalProperty(name = "RESTFacade.readTimeout", defaultValue = "60000") + @GlobalProperty(name = "RESTFacade.readTimeout", defaultValue = "300000") public static int REST_FACADE_READ_TIMEOUT; @GlobalProperty(name = "RESTFacade.connectTimeout", defaultValue = "15000") public static int REST_FACADE_CONNECT_TIMEOUT; diff --git a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java index c4f2f14b2b..e3eef403ee 100755 --- a/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java +++ b/core/src/main/java/org/zstack/core/rest/RESTFacadeImpl.java @@ -88,6 +88,8 @@ public class RESTFacadeImpl implements RESTFacade { private PoolingNHttpClientConnectionManager pingConnManager; // ThreadLocal allows asyncJsonPostForPing() to inject the ping template without changing asyncJson() signature private static final ThreadLocal ASYNC_TEMPLATE_OVERRIDE = new ThreadLocal<>(); + // R2: P0 ping pool capacity — covers largest expected cluster (3000 nodes) + private static final int PING_POOL_MAX_TOTAL = 3000; private String baseUrl; private String sendCommandUrl; private String callbackHostName; @@ -205,7 +207,7 @@ void init() { if (pingConnManager != null) { PoolStats s = pingConnManager.getTotalStats(); sb.append(String.format("ping-pool(P0): maxTotal=%-5d leased=%-5d available=%-5d pending=%d%n", - 3000, s.getLeased(), s.getAvailable(), s.getPending())); + pingConnManager.getMaxTotal(), s.getLeased(), s.getAvailable(), s.getPending())); } else { sb.append("ping-pool(P0): not initialized\n"); } @@ -247,16 +249,16 @@ void init() { asyncRestTemplate = createAsyncRestTemplate( CoreGlobalProperty.REST_FACADE_READ_TIMEOUT, CoreGlobalProperty.REST_FACADE_CONNECT_TIMEOUT, - CoreGlobalProperty.REST_FACADE_MAX_PER_ROUTE, CoreGlobalProperty.REST_FACADE_MAX_TOTAL, + CoreGlobalProperty.REST_FACADE_MAX_PER_ROUTE, cmRef); asyncConnManager = cmRef[0]; - // P0 ping pool: maxPerRoute=1 (one ping/host), maxTotal=3000 (full cluster), short timeouts + // P0 ping pool: maxPerRoute=2 (1 in-flight + 1 queued per host), maxTotal covers full cluster pingAsyncRestTemplate = createAsyncRestTemplate( - 10000, // readTimeout = ping timeout (10s) - 3000, // connectTimeout = fast fail (3s) - 1, // maxPerRoute: one ping per host at a time - 3000, // maxTotal: support up to 3000 hosts + 10000, // readTimeout = ping timeout (10s) + 3000, // connectTimeout = fast fail (3s) + PING_POOL_MAX_TOTAL, // maxTotal: full cluster capacity + 2, // maxPerRoute: 2 allows 1 in-flight + 1 queued, avoids head-of-line block cmRef); pingConnManager = cmRef[0]; @@ -271,7 +273,7 @@ public void run() { if (asyncConnManager != null) { PoolStats s = asyncConnManager.getTotalStats(); long maxTotal = CoreGlobalProperty.REST_FACADE_MAX_TOTAL; - if (s.getPending() > 0 || s.getLeased() > maxTotal * 0.8) { + if (s.getPending() > 0 || s.getLeased() * 5 > maxTotal * 4) { logger.warn(String.format("[POOL-ALARM] async-pool HIGH: leased=%d available=%d pending=%d maxTotal=%d", s.getLeased(), s.getAvailable(), s.getPending(), maxTotal)); } @@ -288,12 +290,13 @@ public void run() { } // timeout are in milliseconds; delegates to 5-param overload (backward compat) - private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int connectTimeout, int maxPerRoute, int maxTotal) { - return createAsyncRestTemplate(readTimeout, connectTimeout, maxPerRoute, maxTotal, null); + // Parameter order matches createRestTemplate: (readTimeout, connectTimeout, maxTotal, maxPerRoute) + private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int connectTimeout, int maxTotal, int maxPerRoute) { + return createAsyncRestTemplate(readTimeout, connectTimeout, maxTotal, maxPerRoute, null); } // R5: outCm captures the connection manager for observability (DumpConnectionPoolStatus) - private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int connectTimeout, int maxPerRoute, int maxTotal, + private static AsyncRestTemplate createAsyncRestTemplate(int readTimeout, int connectTimeout, int maxTotal, int maxPerRoute, PoolingNHttpClientConnectionManager[] outCm) { PoolingNHttpClientConnectionManager connectionManager; try { @@ -651,7 +654,8 @@ public void success(HttpEntity responseEntity) { logger.trace(String.format("json %s [%s], %s", method.toString(), url, req)); } - AsyncRestTemplate tmpl = ASYNC_TEMPLATE_OVERRIDE.get() != null ? ASYNC_TEMPLATE_OVERRIDE.get() : asyncRestTemplate; + AsyncRestTemplate override = ASYNC_TEMPLATE_OVERRIDE.get(); + AsyncRestTemplate tmpl = override != null ? override : asyncRestTemplate; ListenableFuture> f = tmpl.exchange(url, method, req, String.class); f.addCallback(rsp -> {}, e -> wrapper.fail(err(ORG_ZSTACK_CORE_REST_10003, SysErrors.HTTP_ERROR, e.getLocalizedMessage()))); } catch (RestClientException e) { @@ -689,6 +693,17 @@ public void asyncJsonPostForPing(String url, Object body, AsyncRESTCallback call } } + @Override + public void asyncJsonPostForPing(String url, Object body, AsyncRESTCallback callback, TimeUnit unit, long timeout) { + // R2: inject dedicated ping pool via ThreadLocal; cleared in finally to prevent leaks + ASYNC_TEMPLATE_OVERRIDE.set(pingAsyncRestTemplate); + try { + asyncJsonPost(url, body, callback, unit, timeout); + } finally { + ASYNC_TEMPLATE_OVERRIDE.remove(); + } + } + @Override public HttpEntity httpServletRequestToHttpEntity(HttpServletRequest req) { try { diff --git a/header/src/main/java/org/zstack/header/rest/RESTFacade.java b/header/src/main/java/org/zstack/header/rest/RESTFacade.java index d631c15fa5..ce952dce6b 100755 --- a/header/src/main/java/org/zstack/header/rest/RESTFacade.java +++ b/header/src/main/java/org/zstack/header/rest/RESTFacade.java @@ -37,6 +37,9 @@ public interface RESTFacade { /** P0 control-plane ping — uses dedicated isolated pool (R2). Drop-in replacement for asyncJsonPost on ping paths. */ void asyncJsonPostForPing(String url, Object body, AsyncRESTCallback callback); + /** P0 control-plane ping with explicit timeout — uses dedicated isolated pool (R2). */ + void asyncJsonPostForPing(String url, Object body, AsyncRESTCallback callback, TimeUnit unit, long timeout); + void asyncJsonDelete(String url, String body, Map headers, AsyncRESTCallback callback, TimeUnit unit, long timeout); void asyncJsonGet(String url, String body, Map headers, AsyncRESTCallback callback, TimeUnit unit, long timeout); @@ -132,18 +135,22 @@ static TimeoutRestTemplate createRestTemplate(int readTimeout, int connectTimeou SSLContext sslContext = DefaultSSLVerifier.getSSLContext(DefaultSSLVerifier.trustAllCerts); if (maxTotal > 0 && maxPerRoute > 0) { - // R1: explicit pool — must set connection manager AND ssl together + // R1: explicit pool with SSL baked into the CM's socket factory registry. + // IMPORTANT: HttpClientBuilder silently ignores setSSLContext/setSSLHostnameVerifier + // when setConnectionManager() is used — SSL must be registered into the CM instead. + org.apache.http.conn.ssl.SSLConnectionSocketFactory sslSf = sslContext != null + ? new org.apache.http.conn.ssl.SSLConnectionSocketFactory(sslContext, new NoopHostnameVerifier()) + : org.apache.http.conn.ssl.SSLConnectionSocketFactory.getSocketFactory(); + org.apache.http.config.Registry socketRegistry = + org.apache.http.config.RegistryBuilder.create() + .register("http", org.apache.http.conn.socket.PlainConnectionSocketFactory.getSocketFactory()) + .register("https", sslSf) + .build(); org.apache.http.impl.conn.PoolingHttpClientConnectionManager cm = - new org.apache.http.impl.conn.PoolingHttpClientConnectionManager(); + new org.apache.http.impl.conn.PoolingHttpClientConnectionManager(socketRegistry); cm.setMaxTotal(maxTotal); cm.setDefaultMaxPerRoute(maxPerRoute); - org.apache.http.impl.client.HttpClientBuilder builder = HttpClients.custom() - .setConnectionManager(cm); - if (sslContext != null) { - builder.setSSLHostnameVerifier(new NoopHostnameVerifier()) - .setSSLContext(sslContext); - } - factory.setHttpClient(builder.build()); + factory.setHttpClient(HttpClients.custom().setConnectionManager(cm).build()); } else if (sslContext != null) { // original behavior: only override HttpClient when SSL needed factory.setHttpClient(HttpClients.custom()