diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b556905e58..f478f81990 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -160,9 +160,9 @@ jobs: -f docker-compose.ranger-knox.yml \ -f docker-compose.ranger-ozone.yml up -d - - name: Check status of containers and remove them + - name: Check status of containers run: | - sleep 60 + sleep 240 containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); flag=true; for container in "${containers[@]}"; do @@ -174,10 +174,23 @@ jobs: fi done - if [[ $flag == true ]]; then - echo "All required containers are up and running"; - docker stop $(docker ps -q) && docker rm $(docker ps -aq); - else - docker stop $(docker ps -q) && docker rm $(docker ps -aq); - exit 1; + if [[ $flag == false ]]; then + echo "Some required containers are NOT running. Failing pipeline." + exit 1 fi + + - name: Check Ranger plugin status + env: + PLUGIN_RETRY_COUNT: 10 + PLUGIN_RETRY_INTERVAL: 30 + run: | + cd dev-support/ranger-docker + pip install --upgrade pip + pip install apache-ranger python-dotenv + ./scripts/ranger-plugin-status.py + + - name: Clean up containers + if: always() + run: | + echo "Stopping and removing all containers..." + docker stop $(docker ps -q) && docker rm $(docker ps -aq) diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index 463b05f098..3ade1c10de 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -74,3 +74,13 @@ OZONE_PLUGIN_VERSION=3.0.0-SNAPSHOT DEBUG_ADMIN=false DEBUG_USERSYNC=false DEBUG_TAGSYNC=false + + +# using this info to check plugin status in actions file +RANGER_ADMIN_USER=admin +RANGER_ADMIN_PASS=rangerR0cks! +KNOX_USER=admin +KNOX_PASS=admin-password + + + diff --git a/dev-support/ranger-docker/scripts/ranger-plugin-status.py b/dev-support/ranger-docker/scripts/ranger-plugin-status.py new file mode 100755 index 0000000000..095c19709e --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-plugin-status.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import time +from dotenv import load_dotenv +from apache_ranger.client.ranger_client import RangerClient + +# Load environment variables from .env +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +# Path to the .env file in the parent directory +ENV_PATH = os.path.join(SCRIPT_DIR, "..", ".env") + +# Load it +load_dotenv(dotenv_path=ENV_PATH) + +RANGER_ADMIN_USER = os.getenv("RANGER_ADMIN_USER") +RANGER_ADMIN_PASS = os.getenv("RANGER_ADMIN_PASS") +KNOX_USER = os.getenv("KNOX_USER") +KNOX_PASS = os.getenv("KNOX_PASS") + +# Ranger Admin URL and credentials +ranger_url = "http://localhost:6080" +credentials = (RANGER_ADMIN_USER, RANGER_ADMIN_PASS) + +# Initialize the Ranger client +ranger = RangerClient(ranger_url, credentials) + +PLUGIN_INFO_ENDPOINT = "/service/public/v2/api/plugins/info" +KNOX_ENDPOINT = "https://localhost:8443/gateway/sandbox/webhdfs/v1/?op=LISTSTATUS" + +RETRY_COUNT = int(os.getenv("PLUGIN_RETRY_COUNT", 4)) +RETRY_INTERVAL = int(os.getenv("PLUGIN_RETRY_INTERVAL", 30)) + +expected_services = ["hdfs", "hbase", "kms", "yarn", "kafka", "ozone", "knox", "hive"] + + +def init_knox_plugin(): + print("\nInitializing Knox plugin...") + try: + response = ranger.session.get(KNOX_ENDPOINT,auth=(KNOX_USER,KNOX_PASS),verify=False,timeout=10) + print("Knox activity triggered.") + except Exception as e: + print(f"Failed to initialize Knox plugin: {e}") + +def fetch_plugin_info(): + print(f"\nFetching plugin info from {PLUGIN_INFO_ENDPOINT} ...") + try: + response = ranger.session.get(ranger_url + PLUGIN_INFO_ENDPOINT) + response.raise_for_status() + return response.json() + except Exception as e: + print(f"Error fetching plugin info: {e}") + exit(1) + +def check_plugins(plugin_data): + + failed_services = [] + print("\n<--------- Plugin Status ---------->") + for svc in expected_services: + print(f"\nChecking service type: {svc}") + entries = [entry for entry in plugin_data if entry.get("serviceType") == svc] + + if not entries: + print(f"MISSING: No plugins found for service type '{svc}'.") + failed_services.append(svc) + continue + + active_plugins = [ + entry for entry in entries + if entry.get("info", {}).get("policyActiveVersion") + ] + print(f"🟢 Active plugins: {len(active_plugins)} / {len(entries)} total plugins found.") + + if not active_plugins: + print(f"WARNING: Plugins present but NONE are active for '{svc}'.") + failed_services.append(svc) + + print("Details:") + for entry in entries: + host = entry.get("hostName", "unknown") + app_type = entry.get("appType", "unknown") + version = entry.get("info", {}).get("policyActiveVersion", "null") + print(f"- Host: {host}, AppType: {app_type}, PolicyActiveVersion: {version}") + return failed_services + + +def main(): + + print("Checking Ranger plugin status via Ranger Admin API") + + # Trigger knox activity + init_knox_plugin() + + # wait for status update + for i in range(RETRY_COUNT): #retrying upto RETRY_COUNT * RETRY_INTERVAL seconds + plugin_data = fetch_plugin_info() + if all(any(entry.get("info", {}).get("policyActiveVersion") for entry in plugin_data if entry["serviceType"] == svc) for svc in expected_services): + break + print(f"Some plugins not active yet, retrying in {RETRY_INTERVAL}s...") + time.sleep(RETRY_INTERVAL) + + else: + print("Timed out waiting for plugins to become active.") + + # fetch plugin info through admin API + plugin_data = fetch_plugin_info() + + if not plugin_data: + print("No plugin info returned from API.") + exit(1) + + # get plugin details + failed_services = check_plugins(plugin_data) + + print() + if failed_services: + print(f"❌ The following plugins are missing or inactive: {', '.join(failed_services)}") + exit(1) + else: + print("✅ All expected plugins are present and active.") + +if __name__ == "__main__": + main() + diff --git a/dev-support/ranger-docker/scripts/ranger-plugin-status.sh b/dev-support/ranger-docker/scripts/ranger-plugin-status.sh new file mode 100755 index 0000000000..099d01d069 --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-plugin-status.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +echo "Checking Ranger plugin status via Ranger Admin API" + +# Load environment variables from .env file +SCRIPT_DIR=$(dirname "$0") +source "$SCRIPT_DIR/../.env" + +RANGER_HOST="http://localhost:6080" +ENDPOINT="$RANGER_HOST/service/public/v2/api/plugins/info" + +# Trigger activity for KNOX to make plugin active +echo +echo "Triggering Knox activity to ensure plugin status is updated..." +KNOX_ENDPOINT="https://localhost:8443/gateway/sandbox/webhdfs/v1/?op=LISTSTATUS" + +curl -k -u "$KNOX_USER:$KNOX_PASS" "$KNOX_ENDPOINT" > /dev/null 2>&1 +echo "Knox activity triggered." + +sleep 60 + +echo "Fetching plugin info from $ENDPOINT ..." +response=$(curl -s -u "$RANGER_ADMIN_USER:$RANGER_ADMIN_PASS" "$ENDPOINT") + +if [[ -z "$response" || "$response" == "[]" ]]; then + echo "No plugin info returned from API." + exit 1 +fi + +expected_services=("hdfs" "hbase" "kms" "yarn" "kafka" "ozone" "knox" "hive") +failed=false + +echo +echo "<--------- Plugin Status ----------> " +for svc in "${expected_services[@]}"; do + echo + echo "Checking service type: $svc" + + entries=$(echo "$response" | jq --arg svc "$svc" '[.[] | select(.serviceType == $svc)]') + count=$(echo "$entries" | jq 'length') + + if (( count == 0 )); then + echo "MISSING: No plugins found for service type '$svc'." + failed=true + continue + fi + + active_count=$(echo "$entries" | jq '[.[] | select(.info.policyActiveVersion != null and .info.policyActiveVersion != "")] | length') + + echo "🟢 Active plugins: $active_count / $count total plugins found." + + if (( active_count == 0 )); then + echo "WARNING: Plugins present but NONE are active for '$svc'." + failed=true + fi + + # List hostnames and plugin statuses for this service type + echo "Details:" + echo "$entries" | jq -r '.[] | "- Host: \(.hostName), AppType: \(.appType), PolicyActiveVersion: \(.info.policyActiveVersion // "null")"' +done + +echo +if $failed; then + echo "❌ One or more plugins are missing or inactive." + exit 1 +else + echo "✅ All expected plugins are present and active." +fi