project-usermap requested changes

Author: williamnswanson

comanage_scripts_utils.py

@@ -6,7 +6,7 @@
 import json
 import urllib.error
 import urllib.request
-from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC
+from ldap3 import Server, Connection, ALL, SAFE_SYNC
 
 
 MIN_TIMEOUT = 5
@@ -42,6 +42,14 @@ def mkauthstr(user, passwd):
     return encodebytes(raw_authstr.encode()).decode().replace("\n", "")
 
 
+def get_ldap_authtok(ldap_authfile):
+    if ldap_authfile is not None:
+        ldap_authtok = open(ldap_authfile).readline().rstrip("\n")
+    else:
+        raise PermissionError
+    return ldap_authtok
+
+
 def mkrequest(method, target, data, endpoint, authstr, **kw):
     url = os.path.join(endpoint, target)
     if kw:
@@ -135,12 +143,35 @@ def get_ldap_groups(ldap_server, ldap_user, ldap_authtok):
     ldap_group_osggids = set()
     server = Server(ldap_server, get_info=ALL)
     connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
-    _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES)
+    _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=["gidNumber"])
     for group in response:
         ldap_group_osggids.add(group["attributes"]["gidNumber"])
     return ldap_group_osggids
 
 
+def get_ldap_group_members(ldap_gid, ldap_server, ldap_user, ldap_authtok):
+    ldap_group_members = set()
+    server = Server(ldap_server, get_info=ALL)
+    connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
+    _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", f"(&(gidNumber={ldap_gid})(cn=*))", attributes=["hasMember"])
+    for group in response:
+        ldap_group_members.update(group["attributes"]["hasMember"])
+    return ldap_group_members
+
+
+def get_ldap_active_users(ldap_server, ldap_user, ldap_authtok, filter_group_name=None):
+    ldap_active_users = set()
+    filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None 
+                  else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))")
+    server = Server(ldap_server, get_info=ALL)
+    connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
+    _, _, response, _ = connection.search("ou=people,o=OSG,o=CO,dc=cilogon,dc=org", filter_str, attributes=["employeeNumber"])
+    for person in response:
+        # the "employeeNumber" is the person's name in the first.last format
+        ldap_active_users.add(person["attributes"]["employeeNumber"])
+    return ldap_active_users
+
+
 def identifier_from_list(id_list, id_type):
     id_type_list = [id["Type"] for id in id_list]
     try:

osg-comanage-project-usermap.py

@@ -1,63 +1,32 @@
 #!/usr/bin/env python3
 
-import re
 import os
 import sys
-import json
 import time
 import getopt
-import subprocess
 import urllib.error
 import urllib.request
 import comanage_scripts_utils as utils
 
 
 SCRIPT = os.path.basename(__file__)
 ENDPOINT = "https://registry-test.cilogon.org/registry/"
+LDAP_SERVER = "ldaps://ldap-test.cilogon.org"
+LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
 OSG_CO_ID = 8
-MINTIMEOUT = 5
-MAXTIMEOUT = 625
-TIMEOUTMULTIPLE = 5
 CACHE_FILENAME = "COmanage_Projects_cache.txt"
 CACHE_LIFETIME_HOURS = 0.5
 
-LDAP_AUTH_COMMAND = [
-    "awk", "/ldap_default_authtok/ {print $3}", "/etc/sssd/conf.d/0060_domain_CILOGON.ORG.conf",
-]
-
-LDAP_GROUP_MEMBERS_COMMAND = [
-    "ldapsearch",
-    "-H",
-    "ldaps://ldap.cilogon.org",
-    "-D",
-    "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org",
-    "-w", "{auth}",
-    "-b",
-    "ou=groups,o=OSG,o=CO,dc=cilogon,dc=org",
-    "-s",
-    "one",
-    "(cn=*)",
-]
-
-LDAP_ACTIVE_USERS_COMMAND = [
-    "ldapsearch",
-    "-LLL",
-    "-H", "ldaps://ldap.cilogon.org",
-    "-D", "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org",
-    "-x",
-    "-w",  "{auth}",
-    "-b", "ou=people,o=OSG,o=CO,dc=cilogon,dc=org",
-    "{filter}", "voPersonApplicationUID",
-    "|", "grep", "voPersonApplicationUID",
-    "|", "sort",
-]
 
 _usage = f"""\
 usage: [PASS=...] {SCRIPT} [OPTIONS]
 
 OPTIONS:
   -u USER[:PASS]      specify USER and optionally PASS on command line
   -c OSG_CO_ID        specify OSG CO ID (default = {OSG_CO_ID})
+  -s LDAP_SERVER      specify LDAP server to read data from
+  -l LDAP_USER        specify LDAP user for reading data from LDAP server
+  -a ldap_authfile    specify path to file to open and read LDAP authtok
   -d passfd           specify open fd to read PASS
   -f passfile         specify path to file to open and read PASS
   -e ENDPOINT         specify REST endpoint
@@ -87,6 +56,9 @@ class Options:
     osg_co_id = OSG_CO_ID
     outfile = None
     authstr = None
+    ldap_server = LDAP_SERVER
+    ldap_user = LDAP_USER
+    ldap_authtok = None
     filtergrp = None
 
 
@@ -110,14 +82,14 @@ def co_group_is_project(gid):
 
 
 def get_co_group_osggid(gid):
-    resp_data = get_co_group_identifiers(gid)
-    data = get_datalist(resp_data, "Identifiers")
+    resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr)
+    data = utils.get_datalist(resp_data, "Identifiers")
     return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"]
 
 
 def get_co_group_osggid(gid):
-    resp_data = get_co_group_identifiers(gid)
-    data = get_datalist(resp_data, "Identifiers")
+    resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr)
+    data = utils.get_datalist(resp_data, "Identifiers")
     return list(filter(lambda x : x["Type"] == "osggid", data))[0]["Identifier"]
 
 
@@ -147,11 +119,15 @@ def parse_options(args):
 
     passfd = None
     passfile = None
+    ldap_authfile = None
 
     for op, arg in ops:
         if op == '-h': usage()
         if op == '-u': options.user       = arg
         if op == '-c': options.osg_co_id  = int(arg)
+        if op == '-s': options.ldap_server= arg
+        if op == '-l': options.ldap_user  = arg
+        if op == '-a': ldap_authfile      = arg
         if op == '-d': passfd             = int(arg)
         if op == '-f': passfile           = arg
         if op == '-e': options.endpoint   = arg
@@ -161,69 +137,20 @@ def parse_options(args):
     try:
         user, passwd = utils.getpw(options.user, passfd, passfile)
         options.authstr = utils.mkauthstr(user, passwd)
+        options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile)
     except PermissionError:
         usage("PASS required")
 
 
-def get_ldap_group_members_data():
-    gidNumber_str = "gidNumber: "
-    gidNumber_regex = re.compile(gidNumber_str)
-    member_str = "hasMember: "
-    member_regex = re.compile(member_str)
-
-    auth_str = subprocess.run(
-            LDAP_AUTH_COMMAND,
-            stdout=subprocess.PIPE
-            ).stdout.decode('utf-8').strip()
-    
-    ldap_group_members_command = LDAP_GROUP_MEMBERS_COMMAND
-    ldap_group_members_command[LDAP_GROUP_MEMBERS_COMMAND.index("{auth}")] = auth_str
-
-    data_file = subprocess.run(
-        ldap_group_members_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n')
-
-    search_results = list(filter( 
-        lambda x: not re.compile("#|dn:|cn:|objectClass:").match(x),
-        (line for line in data_file)))
-        
-    search_results.reverse()
-
+def get_ldap_group_members_dict():
     group_data_dict = dict()
-    index = 0
-    while index < len(search_results) - 1:
-        while not gidNumber_regex.match(search_results[index]):
-            index += 1
-        gid = search_results[index].replace(gidNumber_str, "")
-        members_list = []
-        while search_results[index] != "":
-            if member_regex.match(search_results[index]):
-                members_list.append(search_results[index].replace(member_str, ""))
-            index += 1
-        group_data_dict[gid] = members_list
-        index += 1
+    for group_gid in utils.get_ldap_groups(options.ldap_server, options.ldap_user, options.ldap_authtok):
+        group_members = utils.get_ldap_group_members(group_gid, options.ldap_server, options.ldap_user, options.ldap_authtok)
+        group_data_dict[group_gid] = group_members
 
     return group_data_dict
 
 
-def get_ldap_active_users(filter_group_name):
-    auth_str = subprocess.run(
-            LDAP_AUTH_COMMAND,
-            stdout=subprocess.PIPE
-            ).stdout.decode('utf-8').strip()
-
-    filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None 
-                  else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))")
-    
-    ldap_active_users_command = LDAP_ACTIVE_USERS_COMMAND
-    ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{auth}")] = auth_str
-    ldap_active_users_command[LDAP_ACTIVE_USERS_COMMAND.index("{filter}")] = filter_str
-
-    active_users = subprocess.run(ldap_active_users_command, stdout=subprocess.PIPE).stdout.decode('utf-8').split('\n')
-    users = set(line.replace("voPersonApplicationUID: ", "") if re.compile("dn: voPerson*") 
-                else "" for line in active_users)
-    return users
-
-
 def create_user_to_projects_map(project_to_user_map, active_users, osggids_to_names):
     users_to_projects_map = dict()
     for osggid in project_to_user_map:
@@ -256,26 +183,25 @@ def get_co_api_data():
             for entry in entries:
                 osggid_name_pair = entry.split(":")
                 if len(osggid_name_pair) == 2:
-                    project_osggids_to_name[osggid_name_pair[0]] = osggid_name_pair[1]
+                    project_osggids_to_name[int(osggid_name_pair[0])] = osggid_name_pair[1].strip()
+            r.close()
         else:
+            r.close()
             raise OSError
     except OSError:
         with open(CACHE_FILENAME, "w") as w:
             project_osggids_to_name = get_groups_data_from_api()
             print(time.time(), file=w)
             for osggid, name in project_osggids_to_name.items():
                 print(f"{osggid}:{name}", file=w)
-    finally:
-        if r:
-            r.close()
 
     return project_osggids_to_name
 
 
 def get_osguser_groups(filter_group_name=None):
     project_osggids_to_name = get_co_api_data()
-    ldap_groups_members = get_ldap_group_members_data()
-    ldap_users = get_ldap_active_users(filter_group_name)
+    ldap_groups_members = get_ldap_group_members_dict()
+    ldap_users = utils.get_ldap_active_users(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name)
 
     active_project_osggids = set(ldap_groups_members.keys()).intersection(set(project_osggids_to_name.keys()))
     project_to_user_map = {