[HOPS-1551] System users x.509 v2

Author: kouzant

hadoop-common-project/hadoop-common/src/main/java/io/hops/security/CertificateLocalization.java

@@ -69,4 +69,6 @@ void updateX509(String username, String applicationId, ByteBuffer keyStore, Stri
   String getSuperTruststoreLocation();
 
   String getSuperTruststorePass();
+  
+  String getSuperMaterialPasswordFile();
 }

hadoop-common-project/hadoop-common/src/main/java/io/hops/security/CertificateLocalizationService.java

@@ -25,9 +25,7 @@
 import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.net.HopsSSLSocketFactory;
 import org.apache.hadoop.security.ssl.SecurityMaterial;
-import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
 import org.apache.hadoop.security.ssl.JWTSecurityMaterial;
-import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.ssl.X509SecurityMaterial;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.log4j.LogManager;
@@ -64,16 +62,10 @@ public class CertificateLocalizationService extends AbstractService
   private final Logger LOG = LogManager.getLogger
       (CertificateLocalizationService.class);
 
-  private final String SYSTEM_TMP = System.getProperty("java.io.tmpdir",
-      "/tmp");
+  private final String SYSTEM_TMP = System.getProperty("java.io.tmpdir", "/tmp");
   private final String LOCALIZATION_DIR_NAME = "certLoc";
   private Path materializeDir;
-  private String superKeystoreLocation;
-  private String superKeystorePass;
-  private String superKeyPassword;
-  private String superTrustStoreLocation;
-  private String superTruststorePass;
-
+  
   private final Map<StorageKey, SecurityMaterial> materialLocation =
       new ConcurrentHashMap<>();
   private ObjectName mbeanObjectName;
@@ -84,6 +76,8 @@ public class CertificateLocalizationService extends AbstractService
   private final BlockingQueue<LocalizationEvent> localizationEventsQ;
 
   private Thread localizationEventsHandler;
+  private X509SecurityMaterial superuserMaterial;
+  private char[] keystoresPassword;
 
   @InterfaceAudience.Private
   @VisibleForTesting
@@ -118,7 +112,8 @@ public CertificateLocalizationService(ServiceType service) {
 
   @Override
   protected void serviceInit(Configuration conf) throws Exception {
-    parseSuperuserMaterial(conf);
+    superuserMaterial = loadSuperuserMaterialInternal(conf);
+    keystoresPassword = readSupersuperPassword();
     String localizationDir = service.toString() +  "_" + LOCALIZATION_DIR_NAME;
     materializeDir = Paths.get(SYSTEM_TMP, localizationDir);
     File fileMaterializeDir = materializeDir.toFile();
@@ -143,6 +138,15 @@ protected void serviceInit(Configuration conf) throws Exception {
     super.serviceInit(conf);
   }
 
+  private X509SecurityMaterial loadSuperuserMaterialInternal(Configuration conf) throws IOException {
+    SuperuserKeystoresLoader superuserKeystoresLoader = new SuperuserKeystoresLoader(conf);
+    return superuserKeystoresLoader.loadSuperUserMaterial();
+  }
+  
+  public char[] readSupersuperPassword() throws IOException {
+    return FileUtils.readFileToString(superuserMaterial.getPasswdLocation().toFile()).toCharArray();
+  }
+  
   @Override
   protected void serviceStart() throws Exception {
     localizationEventsHandler = createLocalizationEventsHandler();
@@ -155,61 +159,45 @@ protected void serviceStart() throws Exception {
 
     super.serviceStart();
   }
-
-  private void parseSuperuserMaterial(Configuration conf) {
-    Configuration sslConf = new Configuration(false);
-    sslConf.addResource(conf.get(SSLFactory.SSL_SERVER_CONF_KEY,
-        "ssl-server.xml"));
-    superKeystoreLocation = sslConf.get(
-        FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER,
-            FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY));
-    superKeystorePass = sslConf.get(
-        FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER,
-            FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY));
-    superKeyPassword = sslConf.get(
-        FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER,
-            FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY));
-    superTrustStoreLocation = sslConf.get(
-        FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER,
-            FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY));
-    superTruststorePass = sslConf.get(
-        FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER,
-            FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY));
-  }
 
   // This method is accessible only from RM or NM. In any other case
   // CertificateLocalizationService is null
   @Override
   public String getSuperKeystoreLocation() {
-    return superKeystoreLocation;
+    return superuserMaterial.getKeyStoreLocation().toString();
   }
 
   // This method is accessible only from RM or NM. In any other case
   // CertificateLocalizationService is null
   @Override
   public String getSuperKeystorePass() {
-    return superKeystorePass;
+    return String.valueOf(keystoresPassword);
   }
 
   // This method is accessible only from RM or NM. In any other case
   // CertificateLocalizationService is null
   @Override
   public String getSuperKeyPassword() {
-    return superKeyPassword;
+    return String.valueOf(keystoresPassword);
   }
 
   // This method is accessible only from RM or NM. In any other case
   // CertificateLocalizationService is null
   @Override
   public String getSuperTruststoreLocation() {
-    return superTrustStoreLocation;
+    return superuserMaterial.getTrustStoreLocation().toString();
   }
 
   // This method is accessible only from RM or NM. In any other case
   // CertificateLocalizationService is null
   @Override
   public String getSuperTruststorePass() {
-    return superTruststorePass;
+    return String.valueOf(keystoresPassword);
+  }
+  
+  @Override
+  public String getSuperMaterialPasswordFile() {
+    return superuserMaterial.getPasswdLocation().toString();
   }
 
   @Override
@@ -226,6 +214,10 @@ protected void serviceStop() throws Exception {
       FileUtils.deleteQuietly(materializeDir.toFile());
     }
 
+    for (int i = 0; i < keystoresPassword.length; i++) {
+      keystoresPassword[i] = 'x';
+    }
+    
     LOG.debug("Stopped CertificateLocalization service");
     super.serviceStop();
   }

hadoop-common-project/hadoop-common/src/main/java/io/hops/security/HopsFileBasedKeyStoresFactory.java

@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.hops.security;
+
+import com.google.common.base.Strings;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.net.HopsSSLSocketFactory;
+import org.apache.hadoop.net.hopssslchecks.HopsSSLCryptoMaterial;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
+import org.apache.hadoop.security.ssl.KeyStoresFactory;
+import org.apache.hadoop.security.ssl.ReloadingX509KeyManager;
+import org.apache.hadoop.security.ssl.ReloadingX509TrustManager;
+import org.apache.hadoop.security.ssl.SSLFactory;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.TrustManager;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.DEFAULT_SSL_KEYSTORE_RELOAD_INTERVAL;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.DEFAULT_SSL_KEYSTORE_RELOAD_TIMEUNIT;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.DEFAULT_SSL_TRUSTSTORE_RELOAD_INTERVAL;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_KEYSTORE_RELOAD_INTERVAL_TPL_KEY;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_KEYSTORE_RELOAD_TIMEUNIT_TPL_KEY;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_PASSWORDFILE_LOCATION_TPL_KEY;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY;
+import static org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY;
+
+public class HopsFileBasedKeyStoresFactory implements KeyStoresFactory {
+  private static final Log LOG = LogFactory.getLog(HopsFileBasedKeyStoresFactory.class);
+  
+  private Configuration sslConf;
+  private Configuration systemConf;
+  private ReloadingX509KeyManager keyManager;
+  private KeyManager[] keyManagers;
+  private ReloadingX509TrustManager trustManager;
+  private TrustManager[] trustManagers;
+  
+  @Override
+  public void init(SSLFactory.Mode mode) throws IOException, GeneralSecurityException {
+    HopsSSLCryptoMaterial material = loadCryptoMaterial(mode);
+    createKeyManagers(mode, material);
+    createTrustManagers(mode, material);
+  }
+  
+  public HopsSSLCryptoMaterial loadCryptoMaterial(SSLFactory.Mode mode) throws IOException {
+    try {
+      CertificateLocalizationCtx certificateLocalizationCtx = CertificateLocalizationCtx.getInstance();
+      certificateLocalizationCtx.setProxySuperusers(systemConf);
+      Configuration x509MaterialConf = new Configuration(false);
+      x509MaterialConf.set(CommonConfigurationKeysPublic.HOPS_TLS_SUPER_MATERIAL_DIRECTORY,
+          systemConf.get(CommonConfigurationKeysPublic.HOPS_TLS_SUPER_MATERIAL_DIRECTORY, ""));
+      // Create a HopsSSLSocketFactory to use its functionality of identifying the correct security material
+      // We don't use the socket factory anywhere else in this class
+      HopsSSLSocketFactory hopsSSLSocketFactory = new HopsSSLSocketFactory();
+      hopsSSLSocketFactory.setConf(x509MaterialConf);
+      return hopsSSLSocketFactory.configureCryptoMaterial(
+          certificateLocalizationCtx.getCertificateLocalization(), certificateLocalizationCtx.getProxySuperusers());
+    } catch (Exception ex) {
+      UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+      LOG.warn("Could not locate cryptographic material for <" + currentUser.getUserName()
+        + "> Falling back to ssl-{client,server}.xml");
+      // Fallback to old-school ssl-{client,server}.xml
+      String keystoreLocationProperty = FileBasedKeyStoresFactory.resolvePropertyName(mode,
+          SSL_KEYSTORE_LOCATION_TPL_KEY);
+      String keystoreLocation = sslConf.get(keystoreLocationProperty);
+      String keystorePasswordProperty = FileBasedKeyStoresFactory.resolvePropertyName(mode,
+          SSL_KEYSTORE_PASSWORD_TPL_KEY);
+      String keystorePassword = sslConf.get(keystorePasswordProperty);
+      String keyPasswordProperty =
+          FileBasedKeyStoresFactory.resolvePropertyName(mode, SSL_KEYSTORE_KEYPASSWORD_TPL_KEY);
+      String keyPassword = sslConf.get(keyPasswordProperty, keystorePassword);
+      
+      String truststoreLocationProperty = FileBasedKeyStoresFactory.resolvePropertyName(mode,
+          SSL_TRUSTSTORE_LOCATION_TPL_KEY);
+      String truststoreLocation = sslConf.get(truststoreLocationProperty);
+      String passwordFileLocationProperty =
+          FileBasedKeyStoresFactory.resolvePropertyName(mode,
+              SSL_PASSWORDFILE_LOCATION_TPL_KEY);
+      String passwordFileLocation = sslConf.get(passwordFileLocationProperty, null);
+      if (Strings.isNullOrEmpty(keystoreLocation) || Strings.isNullOrEmpty(truststoreLocation)
+        || Strings.isNullOrEmpty(keystorePassword) || Strings.isNullOrEmpty(keyPassword)) {
+        throw new IOException("Failed to determine cryptographic material for user <" + currentUser.getUserName()
+          + ">. Exhausted all methods!");
+      }
+      return new HopsSSLCryptoMaterial(
+          keystoreLocation, keystorePassword, keyPassword,
+          truststoreLocation, keystorePassword,
+          passwordFileLocation, true);
+    }
+  }
+  
+  @Override
+  public void destroy() {
+    if (trustManager != null) {
+      trustManager.destroy();
+      trustManager = null;
+      trustManagers = null;
+    }
+    if (keyManager != null) {
+      keyManager.stop();
+      keyManager = null;
+      keyManagers = null;
+    }
+  }
+  
+  @Override
+  public KeyManager[] getKeyManagers() {
+    return keyManagers;
+  }
+  
+  @Override
+  public TrustManager[] getTrustManagers() {
+    return trustManagers;
+  }
+  
+  @Override
+  public void setConf(Configuration conf) {
+    this.sslConf = conf;
+  }
+  
+  @Override
+  public Configuration getConf() {
+    return sslConf;
+  }
+  
+  public void setSystemConf(Configuration conf) {
+    this.systemConf = conf;
+  }
+  
+  public Configuration getSystemConf() {
+    return systemConf;
+  }
+  
+  private void createKeyManagers(SSLFactory.Mode mode, HopsSSLCryptoMaterial material)
+      throws IOException, GeneralSecurityException {
+    boolean requireClientCert = sslConf.getBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY,
+        SSLFactory.SSL_REQUIRE_CLIENT_CERT_DEFAULT);
+    
+    String keystoreType = sslConf.get(
+        FileBasedKeyStoresFactory.resolvePropertyName(mode, FileBasedKeyStoresFactory.SSL_KEYSTORE_TYPE_TPL_KEY),
+        FileBasedKeyStoresFactory.DEFAULT_KEYSTORE_TYPE);
+    
+    if (requireClientCert || mode == SSLFactory.Mode.SERVER) {
+      
+      String keystoreLocation = material.getKeyStoreLocation();
+      if (Strings.isNullOrEmpty(keystoreLocation)) {
+        throw new GeneralSecurityException("Could not identify correct keystore");
+      }
+      String keystorePassword = material.getKeyStorePassword();
+      if (Strings.isNullOrEmpty(keystorePassword)) {
+        throw new GeneralSecurityException("Could not load keystore password");
+      }
+      String keyPassword = material.getKeyPassword();
+      if (Strings.isNullOrEmpty(keyPassword)) {
+        throw new GeneralSecurityException("Could not load key password");
+      }
+  
+      long keyStoreReloadInterval = sslConf.getLong(
+          FileBasedKeyStoresFactory.resolvePropertyName(mode, SSL_KEYSTORE_RELOAD_INTERVAL_TPL_KEY),
+          DEFAULT_SSL_KEYSTORE_RELOAD_INTERVAL);
+      String timeUnitStr = sslConf.get(
+          FileBasedKeyStoresFactory.resolvePropertyName(mode, SSL_KEYSTORE_RELOAD_TIMEUNIT_TPL_KEY),
+          DEFAULT_SSL_KEYSTORE_RELOAD_TIMEUNIT);
+      TimeUnit reloadTimeUnit = TimeUnit.valueOf(timeUnitStr.toUpperCase());
+      
+      String passwordFileLocation = material.getPasswordFileLocation();
+      keyManager = new ReloadingX509KeyManager(keystoreType, keystoreLocation, keystorePassword, passwordFileLocation,
+          keyPassword, keyStoreReloadInterval, reloadTimeUnit);
+      
+      keyManager.init();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(mode.toString() + " Loaded KeyStore: " + keystoreLocation);
+      }
+      keyManagers = new KeyManager[]{keyManager};
+    } else {
+      KeyStore keyStore = KeyStore.getInstance(keystoreType);
+      keyStore.load(null, null);
+      KeyManagerFactory keyMgrFactory = KeyManagerFactory.getInstance(SSLFactory.SSLCERTIFICATE);
+      keyMgrFactory.init(keyStore, null);
+      keyManagers = keyMgrFactory.getKeyManagers();
+    }
+  }
+  
+  private void createTrustManagers(SSLFactory.Mode mode, HopsSSLCryptoMaterial material)
+      throws IOException, GeneralSecurityException {
+    String truststoreType = sslConf.get(
+        FileBasedKeyStoresFactory.resolvePropertyName(mode, FileBasedKeyStoresFactory.SSL_TRUSTSTORE_TYPE_TPL_KEY),
+        FileBasedKeyStoresFactory.DEFAULT_KEYSTORE_TYPE);
+    String truststoreLocation = material.getTrustStoreLocation();
+    if (Strings.isNullOrEmpty(truststoreLocation)) {
+      throw new GeneralSecurityException("Could not identify correct truststore");
+    }
+    String truststorePassword = material.getTrustStorePassword();
+    if (Strings.isNullOrEmpty(truststorePassword)) {
+      throw new GeneralSecurityException("Could not load truststore password");
+    }
+    String passwordFileLocation = material.getPasswordFileLocation();
+    long truststoreReloadInterval =
+        sslConf.getLong(
+            FileBasedKeyStoresFactory.resolvePropertyName(mode, SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY),
+            DEFAULT_SSL_TRUSTSTORE_RELOAD_INTERVAL);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(mode.toString() + " TrustStore: " + truststoreLocation);
+    }
+    trustManager = new ReloadingX509TrustManager(truststoreType,
+        truststoreLocation, truststorePassword, passwordFileLocation, truststoreReloadInterval);
+    trustManager.init();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(mode.toString() + " Loaded TrustStore: " + truststoreLocation);
+    }
+    trustManagers = new TrustManager[]{trustManager};
+  }
+}