Ranger 0.7.2

agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java

@@ -63,6 +63,7 @@ public class HDFSAuditDestination extends AuditDestination {
 	private String logFolder;
 
 	private PrintWriter logWriter = null;
+	volatile FSDataOutputStream ostream = null; // output stream wrapped in logWriter
 
 	private String currentFileName;
 
@@ -169,6 +170,7 @@ public PrintWriter run()  throws Exception {
 				addDeferredCount(events.size());
 				out.close();
 				logWriter = null;
+				ostream = null;
 				return false;
 			}
 		} catch (Throwable t) {
@@ -178,7 +180,7 @@ public PrintWriter run()  throws Exception {
 		} finally {
 			logger.info("Flushing HDFS audit. Event Size:" + events.size());
 			if (out != null) {
-				out.flush();
+				flush();
 			}
 		}
 		addSuccessCount(events.size());
@@ -187,10 +189,22 @@ public PrintWriter run()  throws Exception {
 
 	@Override
 	public void flush() {
-		if ( logWriter != null) {
-			logWriter.flush();
-			logger.info("Flush HDFS audit logs completed.....");
-		 }
+		logger.info("Flush called. name=" + getName());
+		if (ostream != null) {
+			try {
+				synchronized (this) {
+					if (ostream != null)
+						// 1) PrinterWriter does not have bufferring of its own so
+						// we need to flush its underlying stream
+						// 2) HDFS flush() does not really flush all the way to disk.
+						ostream.hflush();
+						logger.info("Flush HDFS audit logs completed.....");
+				}
+			} catch (IOException e) {
+				logger.error("Error on flushing log writer: " + e.getMessage() +
+				 "\nException will be ignored. name=" + getName() + ", fileName=" + currentFileName);
+			}
+		}
 	}
 
 	/*
@@ -246,6 +260,7 @@ synchronized public void stop() {
 						+ getName() + ", fileName=" + currentFileName);
 			}
 			logWriter = null;
+			ostream = null;
 		}
 		logStatus();
 	}
@@ -290,7 +305,7 @@ synchronized private PrintWriter getLogFileStream() throws Exception {
 
 			// Create the file to write
 			logger.info("Creating new log file. hdfPath=" + fullPath);
-			FSDataOutputStream ostream = fileSystem.create(hdfPath);
+			ostream = fileSystem.create(hdfPath);
 			logWriter = new PrintWriter(ostream);
 			currentFileName = fullPath;
 		}
@@ -341,6 +356,7 @@ private void closeFileIfNeeded() throws FileNotFoundException, IOException {
 			}
 
 			logWriter = null;
+			ostream = null;
 			currentFileName = null;
 
 			if (!rollOverByDuration) {

agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java

@@ -23,6 +23,7 @@
 import java.util.Properties;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -73,6 +74,7 @@ public class AuditProviderFactory {
 	private AuditHandler mProvider = null;
 	private String componentAppType = "";
 	private boolean mInitDone = false;
+	private JVMShutdownHook jvmShutdownHook = null;
 
 	private AuditProviderFactory() {
 		LOG.info("AuditProviderFactory: creating..");
@@ -106,6 +108,16 @@ public boolean isInitDone() {
 		return mInitDone;
 	}
 
+	/**
+	 * call shutdown hook to provide a way to
+	 * shutdown gracefully in addition to the ShutdownHook mechanism
+	 */
+	public void shutdown() {
+		if (isInitDone() && jvmShutdownHook != null) {
+			jvmShutdownHook.run();
+		}
+	}
+
 	public synchronized void init(Properties props, String appType) {
 		LOG.info("AuditProviderFactory: initializing..");
 
@@ -463,7 +475,7 @@ private AuditHandler getDefaultProvider() {
 
 	private void installJvmSutdownHook(Properties props) {
 		int shutdownHookMaxWaitSeconds = MiscUtil.getIntProperty(props, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC, AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC_DEFAULT);
-		JVMShutdownHook jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds);
+		jvmShutdownHook = new JVMShutdownHook(mProvider, shutdownHookMaxWaitSeconds);
 		ShutdownHookManager.get().addShutdownHook(jvmShutdownHook, RANGER_AUDIT_SHUTDOWN_HOOK_PRIORITY);
 	}
 
@@ -503,6 +515,7 @@ private static class JVMShutdownHook extends Thread {
 		final Semaphore doneCleanup = new Semaphore(0);
 		final Thread cleanupThread;
 		final int maxWait;
+		final AtomicBoolean done = new AtomicBoolean(false);
 
 		public JVMShutdownHook(AuditHandler provider, int maxWait) {
 			this.maxWait = maxWait;
@@ -513,6 +526,10 @@ public JVMShutdownHook(AuditHandler provider, int maxWait) {
 		}
 
 		public void run() {
+			if (!done.compareAndSet(false, true)) {
+				LOG.info("==> JVMShutdownHook.run() already done by another thread");
+				return;
+			}
 			LOG.info("==> JVMShutdownHook.run()");
 			LOG.info("JVMShutdownHook: Signalling async audit cleanup to start.");
 			startCleanup.release();

agents-audit/src/main/java/org/apache/ranger/audit/provider/LocalFileLogBuffer.java

@@ -144,6 +144,8 @@ public void start(LogDestination<T> destination) {
 
 		mDispatcherThread = new DestinationDispatcherThread<T>(this, destination, mLogger);
 
+		mDispatcherThread.setDaemon(true);
+
 		mDispatcherThread.start();
 
 		mLogger.debug("<== LocalFileLogBuffer.start()");

agents-common/src/main/java/org/apache/hadoop/security/SecureClientLogin.java

@@ -49,6 +49,7 @@ public synchronized static Subject loginUserFromKeytab(String user, String path)
 			SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(true, user, path);
 			LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf);
 			subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login));
+			login.logout();
 			login.login();
 			return login.getSubject();
 		} catch (LoginException le) {
@@ -63,6 +64,7 @@ public synchronized static Subject loginUserFromKeytab(String user, String path,
 			LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf);
 			KerberosName.setRules(nameRules);
 			subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login));
+			login.logout();
 			login.login();
 			return login.getSubject();
 		} catch (LoginException le) {
@@ -71,16 +73,16 @@ public synchronized static Subject loginUserFromKeytab(String user, String path,
 	}
 
 	public synchronized static Subject loginUserWithPassword(String user, String password) throws IOException {
-		String tmpPass = password;
 		try {
 			Subject subject = new Subject();
 			SecureClientLoginConfiguration loginConf = new SecureClientLoginConfiguration(false, user, password);
 			LoginContext login = new LoginContext("hadoop-keytab-kerberos", subject, null, loginConf);
 			subject.getPrincipals().add(new User(user, AuthenticationMethod.KERBEROS, login));
+			login.logout();
 			login.login();
 			return login.getSubject();
 		} catch (LoginException le) {
-			throw new IOException("Login failure for " + user + " using password " + tmpPass.replaceAll(".","*"), le);
+			throw new IOException("Login failure for " + user + " using password ****", le);
 		}
 	}
 

agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java

@@ -31,6 +31,7 @@
 import org.apache.ranger.admin.client.datatype.RESTResponse;
 import org.apache.ranger.audit.provider.MiscUtil;
 import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.apache.ranger.authorization.utils.StringUtil;
 import org.apache.ranger.plugin.util.GrantRevokeRequest;
 import org.apache.ranger.plugin.util.RangerRESTClient;
 import org.apache.ranger.plugin.util.RangerRESTUtils;
@@ -80,11 +81,18 @@ public void init(String serviceName, String appId, String propertyPrefix) {
 		this.serviceName = serviceName;
 		this.pluginId    = restUtils.getPluginId(serviceName, appId);
 
-		String url               		= RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.url");
+		String url                      = "";
+		String tmpUrl                   = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.url");
 		String sslConfigFileName 		= RangerConfiguration.getInstance().get(propertyPrefix + ".policy.rest.ssl.config.file");
 		clusterName       				= RangerConfiguration.getInstance().get(propertyPrefix + ".ambari.cluster.name", "");
 		int	 restClientConnTimeOutMs	= RangerConfiguration.getInstance().getInt(propertyPrefix + ".policy.rest.client.connection.timeoutMs", 120 * 1000);
 		int	 restClientReadTimeOutMs	= RangerConfiguration.getInstance().getInt(propertyPrefix + ".policy.rest.client.read.timeoutMs", 30 * 1000);
+        if (!StringUtil.isEmpty(tmpUrl)) {
+            url = tmpUrl.trim();
+        }
+        if (url.endsWith("/")) {
+            url = url.substring(0, url.length() - 1);
+        }
 
 		init(url, sslConfigFileName, restClientConnTimeOutMs , restClientReadTimeOutMs);
 	}

agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java

@@ -21,7 +21,9 @@
 public class RangerHadoopConstants {
 
 	public static final String RANGER_ADD_HDFS_PERMISSION_PROP = "xasecure.add-hadoop-authorization";
+	public static final String RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_PROP = "ranger.optimize-subaccess-authorization" ;
 	public static final boolean RANGER_ADD_HDFS_PERMISSION_DEFAULT = false;
+	public static final boolean RANGER_OPTIMIZE_SUBACCESS_AUTHORIZATION_DEFAULT = false ;
 	public static final String READ_ACCCESS_TYPE = "read";
 	public static final String WRITE_ACCCESS_TYPE = "write";
 	public static final String EXECUTE_ACCCESS_TYPE = "execute";

agents-common/src/main/java/org/apache/ranger/authorization/utils/JsonUtils.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.authorization.utils;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class JsonUtils {
+    private static final Log LOG = LogFactory.getLog(JsonUtils.class);
+
+    private static final HashMap<String, String> MAP_STRING_STRING = new HashMap<>();
+
+    private static final Gson gson;
+
+    static {
+        gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z")
+                .create();
+    }
+
+    public static String mapToJson(Map<?, ?> map) {
+        String ret = null;
+        if (MapUtils.isNotEmpty(map)) {
+            try {
+                ret = gson.toJson(map);
+            } catch (Exception e) {
+                LOG.error("Invalid input data: ", e);
+            }
+        }
+        return ret;
+    }
+
+    public static String listToJson(List<?> list) {
+        String ret = null;
+        if (CollectionUtils.isNotEmpty(list)) {
+            try {
+                ret = gson.toJson(list);
+            } catch (Exception e) {
+                LOG.error("Invalid input data: ", e);
+            }
+        }
+        return ret;
+    }
+
+    public static String objectToJson(Object object) {
+        String ret = null;
+
+        if(object != null) {
+            try {
+                ret = gson.toJson(object);
+            } catch(Exception excp) {
+                LOG.warn("objectToJson() failed to convert object to Json", excp);
+            }
+        }
+
+        return ret;
+    }
+
+    public static <T> T jsonToObject(String jsonStr, Class<T> clz) {
+        T ret = null;
+
+        if(StringUtils.isNotEmpty(jsonStr)) {
+            try {
+                ret = gson.fromJson(jsonStr, clz);
+            } catch(Exception excp) {
+                LOG.warn("jsonToObject() failed to convert json to object: " + jsonStr, excp);
+            }
+        }
+
+        return ret;
+    }
+
+    public static Map<String, String> jsonToMapStringString(String jsonStr) {
+        Map<String, String> ret = null;
+
+        if(StringUtils.isNotEmpty(jsonStr)) {
+            try {
+                ret = gson.fromJson(jsonStr, MAP_STRING_STRING.getClass());
+            } catch(Exception excp) {
+                LOG.warn("jsonToObject() failed to convert json to object: " + jsonStr, excp);
+            }
+        }
+
+        return ret;
+    }
+
+}

agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerMultiResourceAuditHandler.java

@@ -43,7 +43,7 @@ public void logAuthzAudit(AuthzAuditEvent auditEvent) {
 
 	@Override
 	public void logAuthzAudits(Collection<AuthzAuditEvent> auditEvents) {
-		auditEvents.addAll(auditEvents);
+		this.auditEvents.addAll(auditEvents);
 	}
 
 	public void flushAudit() {

agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java

@@ -24,12 +24,14 @@
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.contextenricher.RangerTagForEval;
 import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
 
 import javax.script.Bindings;
 import javax.script.ScriptEngine;
 import javax.script.ScriptEngineManager;
 import javax.script.ScriptException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -90,11 +92,15 @@ public boolean isMatched(RangerAccessRequest request) {
 
 				RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy();
 
-				RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest);
+				RangerScriptExecutionContext context    = new RangerScriptExecutionContext(readOnlyRequest);
+				RangerTagForEval             currentTag = context.getCurrentTag();
+				Map<String, String>          tagAttribs = currentTag != null ? currentTag.getAttributes() : Collections.<String, String>emptyMap();
 
 				Bindings bindings = scriptEngine.createBindings();
 
 				bindings.put("ctx", context);
+				bindings.put("tag", currentTag);
+				bindings.put("tagAttr", tagAttribs);
 
 				if (LOG.isDebugEnabled()) {
 					LOG.debug("RangerScriptConditionEvaluator.isMatched(): script={" + script + "}");