Skip to content

Commit 3a45a84

Browse files
kgeiszkgeisz
committed
HBASE-29761: The HBase UI's Debug Dump is not redacting sensitive information
Change-Id: I7f0cf9f096727272764252d8e7f6b8c6f5fc91c0
1 parent 8b17149 commit 3a45a84

File tree

6 files changed

+239
-63
lines changed

6 files changed

+239
-63
lines changed

‎hbase-server/src/main/java/org/apache/hadoop/hbase/master/http/MasterDumpServlet.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,10 @@
1818
package org.apache.hadoop.hbase.master.http;
1919

2020
import java.io.IOException;
21-
import java.io.OutputStream;
21+
import java.io.OutputStreamWriter;
2222
import java.io.PrintStream;
2323
import java.io.PrintWriter;
24+
import java.nio.charset.StandardCharsets;
2425
import java.util.Date;
2526
import java.util.Map;
2627
import javax.servlet.http.HttpServletRequest;
@@ -53,7 +54,8 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
5354
assert master != null : "No Master in context!";
5455

5556
response.setContentType("text/plain");
56-
OutputStream os = response.getOutputStream();
57+
OutputStreamWriter os =
58+
new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8);
5759
try (PrintWriter out = new PrintWriter(os)) {
5860

5961
out.println("Master status for " + master.getServerName() + " as of " + new Date());
@@ -81,15 +83,15 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
8183
out.println("\n\nStacks:");
8284
out.println(LINE);
8385
out.flush();
84-
PrintStream ps = new PrintStream(response.getOutputStream(), false, "UTF-8");
86+
PrintStream ps = new PrintStream(response.getOutputStream(), false, StandardCharsets.UTF_8);
8587
Threads.printThreadInfo(ps, "");
8688
ps.flush();
8789

8890
out.println("\n\nMaster configuration:");
8991
out.println(LINE);
9092
Configuration conf = master.getConfiguration();
9193
out.flush();
92-
conf.writeXml(os);
94+
conf.writeXml(null, os, conf);
9395
os.flush();
9496

9597
out.println("\n\nRecent regionserver aborts:");

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/http/RSDumpServlet.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,10 @@
1818
package org.apache.hadoop.hbase.regionserver.http;
1919

2020
import java.io.IOException;
21-
import java.io.OutputStream;
21+
import java.io.OutputStreamWriter;
2222
import java.io.PrintStream;
2323
import java.io.PrintWriter;
24+
import java.nio.charset.StandardCharsets;
2425
import java.util.Date;
2526
import javax.servlet.http.HttpServletRequest;
2627
import javax.servlet.http.HttpServletResponse;
@@ -58,7 +59,8 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
5859
return;
5960
}
6061

61-
OutputStream os = response.getOutputStream();
62+
OutputStreamWriter os =
63+
new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8);
6264
try (PrintWriter out = new PrintWriter(os)) {
6365

6466
out.println("RegionServer status for " + hrs.getServerName() + " as of " + new Date());
@@ -81,15 +83,15 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
8183

8284
out.println("\n\nStacks:");
8385
out.println(LINE);
84-
PrintStream ps = new PrintStream(response.getOutputStream(), false, "UTF-8");
86+
PrintStream ps = new PrintStream(response.getOutputStream(), false, StandardCharsets.UTF_8);
8587
Threads.printThreadInfo(ps, "");
8688
ps.flush();
8789

8890
out.println("\n\nRS Configuration:");
8991
out.println(LINE);
9092
Configuration conf = hrs.getConfiguration();
9193
out.flush();
92-
conf.writeXml(os);
94+
conf.writeXml(null, os, conf);
9395
os.flush();
9496

9597
out.println("\n\nLogs");

‎hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestDebugDumpRedaction.java‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.hbase.http;
19+
20+
import static org.junit.Assert.assertEquals;
21+
import static org.junit.Assert.assertTrue;
22+
23+
import java.io.ByteArrayInputStream;
24+
import java.io.IOException;
25+
import java.io.InputStream;
26+
import java.net.URL;
27+
import java.nio.charset.StandardCharsets;
28+
import java.util.List;
29+
import org.apache.hadoop.conf.Configuration;
30+
import org.apache.hadoop.fs.Path;
31+
import org.apache.hadoop.hbase.HBaseClassTestRule;
32+
import org.apache.hadoop.hbase.HBaseTestingUtil;
33+
import org.apache.hadoop.hbase.HConstants;
34+
import org.apache.hadoop.hbase.LocalHBaseCluster;
35+
import org.apache.hadoop.hbase.ServerName;
36+
import org.apache.hadoop.hbase.master.HMaster;
37+
import org.apache.hadoop.hbase.master.ServerManager;
38+
import org.apache.hadoop.hbase.testclassification.MiscTests;
39+
import org.apache.hadoop.hbase.testclassification.SmallTests;
40+
import org.apache.hadoop.hbase.util.CommonFSUtils;
41+
import org.apache.hadoop.hbase.util.TestServerHttpUtils;
42+
import org.junit.AfterClass;
43+
import org.junit.BeforeClass;
44+
import org.junit.ClassRule;
45+
import org.junit.Test;
46+
import org.junit.experimental.categories.Category;
47+
48+
/**
49+
* This class performs tests that ensure sensitive config values found in the HBase UI's Debug Dump
50+
* are redacted. A config property name must follow one of the regex patterns specified in
51+
* hadoop.security.sensitive-config-keys in order to have its value redacted.
52+
*/
53+
@Category({ MiscTests.class, SmallTests.class })
54+
public class TestDebugDumpRedaction {
55+
private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
56+
private static final String XML_CONFIGURATION_START_TAG = "<configuration>";
57+
private static final String XML_CONFIGURATION_END_TAG = "</configuration>";
58+
private static final int SUBSTRING_OFFSET = XML_CONFIGURATION_END_TAG.length();
59+
private static final String PLAIN_TEXT_UTF8 = "text/plain;charset=utf-8";
60+
private static final String REDACTED = "******";
61+
private static final List<String> SENSITIVE_CONF_PROPERTIES =
62+
List.of("hbase.zookeeper.property.ssl.trustStore.password", "ssl.client.truststore.password",
63+
"hbase.rpc.tls.truststore.password", "ssl.server.keystore.password",
64+
"fs.s3a.server-side-encryption.key", "fs.s3a.encryption.algorithm", "fs.s3a.encryption.key",
65+
"fs.s3a.secret.key", "fs.s3a.important.secret.key", "fs.s3a.session.key",
66+
"fs.s3a.secret.session.key", "fs.s3a.session.token", "fs.s3a.secret.session.token",
67+
"fs.azure.account.key.importantKey", "fs.azure.oauth2.token", "fs.adl.oauth2.token",
68+
"fs.gs.encryption.sensitive", "fs.gs.proxy.important", "fs.gs.auth.sensitive.info",
69+
"sensitive.credential", "oauth.important.secret", "oauth.important.password",
70+
"oauth.important.token");
71+
private static LocalHBaseCluster CLUSTER;
72+
73+
@ClassRule
74+
public static final HBaseClassTestRule CLASS_RULE =
75+
HBaseClassTestRule.forClass(TestDebugDumpRedaction.class);
76+
77+
@BeforeClass
78+
public static void beforeClass() throws Exception {
79+
Configuration conf = UTIL.getConfiguration();
80+
81+
// Add various config properties with sensitive information that should be redacted
82+
// when the Debug Dump is performed in the UI. These properties are following the
83+
// regexes specified by the hadoop.security.sensitive-config-keys property.
84+
for (String property : SENSITIVE_CONF_PROPERTIES) {
85+
conf.set(property, "testPassword");
86+
}
87+
88+
UTIL.startMiniZKCluster();
89+
90+
UTIL.startMiniDFSCluster(1);
91+
Path rootdir = UTIL.getDataTestDirOnTestFS("TestDebugDumpServlet");
92+
CommonFSUtils.setRootDir(conf, rootdir);
93+
94+
// The info servers do not run in tests by default.
95+
// Set them to ephemeral ports so they will start
96+
// setup configuration
97+
conf.setInt(HConstants.MASTER_INFO_PORT, 0);
98+
conf.setInt(HConstants.REGIONSERVER_INFO_PORT, 0);
99+
100+
CLUSTER = new LocalHBaseCluster(conf, 1);
101+
CLUSTER.startup();
102+
CLUSTER.getActiveMaster().waitForMetaOnline();
103+
}
104+
105+
@AfterClass
106+
public static void afterClass() throws Exception {
107+
if (CLUSTER != null) {
108+
CLUSTER.shutdown();
109+
CLUSTER.join();
110+
}
111+
UTIL.shutdownMiniCluster();
112+
}
113+
114+
@Test
115+
public void testMasterPasswordsAreRedacted() throws IOException {
116+
URL debugDumpUrl =
117+
new URL(TestServerHttpUtils.getMasterInfoServerHostAndPort(CLUSTER) + "/dump");
118+
String response = TestServerHttpUtils.getPageContent(debugDumpUrl, PLAIN_TEXT_UTF8);
119+
120+
// Verify this is the master server's debug dump
121+
assertTrue(
122+
response.startsWith("Master status for " + CLUSTER.getActiveMaster().getServerName()));
123+
124+
verifyDebugDumpResponseConfig(response);
125+
}
126+
127+
@Test
128+
public void testRegionServerPasswordsAreRedacted() throws IOException {
129+
HMaster master = CLUSTER.getActiveMaster();
130+
131+
ServerManager serverManager = master.getServerManager();
132+
List<ServerName> onlineServersList = serverManager.getOnlineServersList();
133+
134+
assertEquals(1, onlineServersList.size());
135+
136+
ServerName regionServerName = onlineServersList.get(0);
137+
int regionServerInfoPort = master.getRegionServerInfoPort(regionServerName);
138+
String regionServerHostname = regionServerName.getHostname();
139+
140+
URL debugDumpUrl =
141+
new URL("http://" + regionServerHostname + ":" + regionServerInfoPort + "/dump");
142+
String response = TestServerHttpUtils.getPageContent(debugDumpUrl, PLAIN_TEXT_UTF8);
143+
144+
// Verify this is the region server's debug dump
145+
assertTrue(response.startsWith("RegionServer status for " + regionServerName));
146+
147+
verifyDebugDumpResponseConfig(response);
148+
}
149+
150+
private void verifyDebugDumpResponseConfig(String response) throws IOException {
151+
// Grab the server's config from the Debug Dump.
152+
String xmlString = response.substring(response.indexOf(XML_CONFIGURATION_START_TAG),
153+
response.indexOf(XML_CONFIGURATION_END_TAG) + SUBSTRING_OFFSET);
154+
155+
// Convert the XML string into an InputStream
156+
Configuration conf = new Configuration(false);
157+
try (InputStream is = new ByteArrayInputStream(xmlString.getBytes(StandardCharsets.UTF_8))) {
158+
// Add the InputStream as a resource to the Configuration object
159+
conf.addResource(is, "DebugDumpXmlConfig");
160+
}
161+
162+
// Verify all sensitive properties have had their values redacted
163+
for (String property : SENSITIVE_CONF_PROPERTIES) {
164+
assertEquals(REDACTED, conf.get(property));
165+
}
166+
}
167+
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/master/http/TestMasterStatusPage.java‎

Lines changed: 4 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,7 @@
2121
import static org.junit.Assert.assertNull;
2222
import static org.junit.Assert.assertTrue;
2323

24-
import java.io.BufferedReader;
2524
import java.io.IOException;
26-
import java.io.InputStreamReader;
27-
import java.net.HttpURLConnection;
2825
import java.net.URL;
2926
import java.util.ArrayList;
3027
import java.util.List;
@@ -45,6 +42,7 @@
4542
import org.apache.hadoop.hbase.testclassification.MasterTests;
4643
import org.apache.hadoop.hbase.testclassification.MediumTests;
4744
import org.apache.hadoop.hbase.util.CommonFSUtils;
45+
import org.apache.hadoop.hbase.util.TestServerHttpUtils;
4846
import org.apache.hadoop.hbase.util.VersionInfo;
4947
import org.junit.AfterClass;
5048
import org.junit.BeforeClass;
@@ -108,7 +106,9 @@ public void testMasterStatusPage() throws Exception {
108106

109107
createTestTables(master);
110108

111-
String page = getMasterStatusPageContent();
109+
URL url =
110+
new URL(TestServerHttpUtils.getMasterInfoServerHostAndPort(CLUSTER) + "/master-status");
111+
String page = TestServerHttpUtils.getPageContent(url, "text/html;charset=utf-8");
112112

113113
String hostname = master.getServerName().getHostname();
114114
assertTrue(page.contains("<h1>Master <small>" + hostname + "</small></h1>"));
@@ -127,17 +127,6 @@ public void testMasterStatusPage() throws Exception {
127127
assertTrue(page.contains(VersionInfo.getVersion()));
128128
}
129129

130-
private String getMasterStatusPageContent() throws IOException {
131-
URL url = new URL(getInfoServerHostAndPort() + "/master-status");
132-
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
133-
conn.connect();
134-
135-
assertEquals(200, conn.getResponseCode());
136-
assertEquals("text/html;charset=utf-8", conn.getContentType());
137-
138-
return getResponseBody(conn);
139-
}
140-
141130
private static void createTestTables(HMaster master) throws IOException {
142131
ColumnFamilyDescriptor cf = ColumnFamilyDescriptorBuilder.of("CF");
143132
TableDescriptor tableDescriptor1 = TableDescriptorBuilder
@@ -149,20 +138,6 @@ private static void createTestTables(HMaster master) throws IOException {
149138
master.flushMasterStore();
150139
}
151140

152-
private String getInfoServerHostAndPort() {
153-
return "http://localhost:" + CLUSTER.getActiveMaster().getInfoServer().getPort();
154-
}
155-
156-
private static String getResponseBody(HttpURLConnection conn) throws IOException {
157-
StringBuilder sb = new StringBuilder();
158-
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
159-
String output;
160-
while ((output = br.readLine()) != null) {
161-
sb.append(output);
162-
}
163-
return sb.toString();
164-
}
165-
166141
private static void assertRegionServerLinks(HMaster master, String responseBody) {
167142
ServerManager serverManager = master.getServerManager();
168143
List<ServerName> servers = serverManager.getOnlineServersList();

‎hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/http/TestRSStatusPage.java‎

Lines changed: 3 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,7 @@
2020
import static org.junit.Assert.assertEquals;
2121
import static org.junit.Assert.assertTrue;
2222

23-
import java.io.BufferedReader;
2423
import java.io.IOException;
25-
import java.io.InputStreamReader;
26-
import java.net.HttpURLConnection;
2724
import java.net.URL;
2825
import java.util.List;
2926
import org.apache.hadoop.conf.Configuration;
@@ -43,6 +40,7 @@
4340
import org.apache.hadoop.hbase.testclassification.MediumTests;
4441
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
4542
import org.apache.hadoop.hbase.util.CommonFSUtils;
43+
import org.apache.hadoop.hbase.util.TestServerHttpUtils;
4644
import org.junit.AfterClass;
4745
import org.junit.BeforeClass;
4846
import org.junit.ClassRule;
@@ -123,7 +121,8 @@ public void testStatusPage() throws Exception {
123121
String hostname = firstServerName.getHostname();
124122
int port = firstServerName.getPort();
125123

126-
String page = getRegionServerStatusPageContent(hostname, infoPort);
124+
URL url = new URL("http://" + hostname + ":" + infoPort + "/regionserver.jsp");
125+
String page = TestServerHttpUtils.getPageContent(url, "text/html;charset=utf-8");
127126

128127
assertTrue(page.contains("<title>HBase Region Server: " + masterHostname + "</title>"));
129128

@@ -153,26 +152,4 @@ private static void createTestTables(HMaster master) throws IOException {
153152
master.createTable(tableDescriptor2, null, 0, 0);
154153
master.flushMasterStore();
155154
}
156-
157-
private static String getRegionServerStatusPageContent(String hostname, int infoPort)
158-
throws IOException {
159-
URL url = new URL("http://" + hostname + ":" + infoPort + "/regionserver.jsp");
160-
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
161-
conn.connect();
162-
163-
assertEquals(200, conn.getResponseCode());
164-
assertEquals("text/html;charset=utf-8", conn.getContentType());
165-
166-
return getResponseBody(conn);
167-
}
168-
169-
private static String getResponseBody(HttpURLConnection conn) throws IOException {
170-
StringBuilder sb = new StringBuilder();
171-
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
172-
String output;
173-
while ((output = br.readLine()) != null) {
174-
sb.append(output);
175-
}
176-
return sb.toString();
177-
}
178155
}

0 commit comments

Comments
 (0)