From 73538ff878ba599340f025a552acd754f2c604b9 Mon Sep 17 00:00:00 2001 From: Elango Senthilnathan Date: Thu, 25 Jan 2024 19:06:27 -0800 Subject: [PATCH 1/3] Added a feature to list all projects and team names in report --- shiftleft-utils/common.py | 14 ++++++ shiftleft-utils/stats.py | 98 +++++++++++++++++++++++++++------------ 2 files changed, 83 insertions(+), 29 deletions(-) diff --git a/shiftleft-utils/common.py b/shiftleft-utils/common.py index 806a7b2..3090adf 100644 --- a/shiftleft-utils/common.py +++ b/shiftleft-utils/common.py @@ -76,6 +76,20 @@ def get_all_apps(org_id): ) return None +def get_all_teams(org_id): + """Return all the teams for the given organization""" + list_teams_url = f"https://{config.SHIFTLEFT_API_HOST}/api/v4/orgs/{org_id}/rbac/teams" + r = requests.get(list_teams_url, headers=headers) + if r.ok: + raw_response = r.json() + if raw_response and raw_response.get("response"): + teams_list = raw_response.get("response") + return teams_list + else: + print( + f"Unable to retrieve teams list for the organization {org_id} due to {r.status_code} error" + ) + return None def get_all_findings(org_id, app_name, version): """Method to retrieve all findings""" diff --git a/shiftleft-utils/stats.py b/shiftleft-utils/stats.py index 3b584ae..d34881c 100644 --- a/shiftleft-utils/stats.py +++ b/shiftleft-utils/stats.py @@ -23,6 +23,7 @@ get_findings_counts_url, get_findings_url, get_scan_run, + get_all_teams, headers, ) @@ -37,11 +38,12 @@ def to_arr(counts_dict): def process_app( - progress, task, org_id, report_file, app, detailed, branch, include_run_info + progress, task, org_id, report_file, app, detailed, branch, include_run_info, include_app_apps, teams_list ): start = time.time() app_id = app.get("id") app_name = app.get("name") + isActive = True # Stats only considers the first page for performance so the detailed report is based only on the latest 250 findings # The various counts, however, are based on the full list of findings so are correct findings_url = ( @@ -64,10 +66,12 @@ def process_app( scan = response.get("scan") # Scan will be None if there are any issues/errors if not scan: + isActive = False console.print( f"""\nINFO: No scans found for {app_name} {branch if branch else ""}""" ) - return [] + if not include_app_apps: + return [] run_info = {} token_name = "" if include_run_info: @@ -81,15 +85,18 @@ def process_app( ) tags = app.get("tags") app_group = "" - app_branch = scan.get("tags", {}).get("branch", "") + app_branch = "" + if isActive: + app_branch = scan.get("tags", {}).get("branch", "") if tags: for tag in tags: if tag.get("key") == "group": app_group = tag.get("value") break - # Other unused properties such as findings or counts - spid = scan.get("internal_id") - projectSpId = f'sl/{org_id}/{scan.get("app")}' + if isActive: + # Other unused properties such as findings or counts + spid = scan.get("internal_id") + projectSpId = f'sl/{org_id}/{scan.get("app")}' counts = response.get("counts", []) findings = response.get("findings", []) vuln_counts = [ @@ -256,35 +263,56 @@ def process_app( ml_assisted_count += vc["count"] # Convert date time to BigQuery friendly format completed_at = "" - try: - ctime = scan.get("completed_at", "") - completed_at_dt = datetime.strptime( - ctime, - "%Y-%m-%dT%H:%M:%S.%fZ %Z" - if "UTC" in ctime - else "%Y-%m-%dT%H:%M:%S.%fZ", - ) - completed_at = completed_at_dt.strftime("%Y-%m-%d %H:%M:%S.%f") - except Exception as e: - completed_at = ( - scan.get("completed_at", "") - .replace(" UTC", "") - .replace("Z", "") - .replace("T", " ") - ) + if isActive: + try: + ctime = scan.get("completed_at", "") + completed_at_dt = datetime.strptime( + ctime, + "%Y-%m-%dT%H:%M:%S.%fZ %Z" + if "UTC" in ctime + else "%Y-%m-%dT%H:%M:%S.%fZ", + ) + completed_at = completed_at_dt.strftime("%Y-%m-%d %H:%M:%S.%f") + except Exception as e: + completed_at = ( + scan.get("completed_at", "") + .replace(" UTC", "") + .replace("Z", "") + .replace("T", " ") + ) progress.update( task, description=f"""Processed [bold]{app.get("name")}[/bold] in {math.ceil(time.time() - start)} seconds""", ) + if isActive: + appName = scan.get("app") + appVersion = scan.get("version") + scanID = scan.get("id") + scanLang = scan.get("language") + scanExp = scan.get("number_of_expressions") + else: + appName = app_name + appVersion = "" + scanID = "" + scanLang = "" + scanExp = "" + appTeam = "" + for eachTeam in teams_list: + if eachTeam.get("projects"): + if appName in eachTeam.get("projects"): + appTeam = eachTeam.get("team_name") + return [ - scan.get("app"), + appName, app_group, + appTeam, + isActive, app_branch, - scan.get("version"), + appVersion, completed_at, - scan.get("id"), - scan.get("language"), - scan.get("number_of_expressions"), + scanID, + scanLang, + scanExp, ml_assisted_count, critical_count, high_count, @@ -325,6 +353,8 @@ def write_to_csv(report_file, row): csv_cols = [ "App", "App Group", + "Team Name", + "ActiveApp", "Branch", "Version", "Last Scan", @@ -369,9 +399,10 @@ def write_to_csv(report_file, row): reportwriter.writerow(row) -def collect_stats_parallel(org_id, report_file, detailed, branch, include_run_info): +def collect_stats_parallel(org_id, report_file, detailed, branch, include_run_info, include_all_apps): """Method to collect stats for all apps to a csv""" apps_list = get_all_apps(org_id) + teams_list = get_all_teams(org_id) if not apps_list: console.print("No apps were found in this organization") return @@ -405,6 +436,8 @@ def collect_stats_parallel(org_id, report_file, detailed, branch, include_run_in detailed, branch, include_run_info, + include_all_apps, + teams_list, ) rows.append(row) rows = dask.compute(*rows) @@ -446,6 +479,13 @@ def build_args(): help="Run info includes runtime information, tokens and scan statistics", default=False, ) + parser.add_argument( + "--include-all-apps", + action="store_true", + dest="include_app_apps", + help="Run info includes data for all apps including app placeholders", + default=False, + ) return parser.parse_args() @@ -466,7 +506,7 @@ def main(): args = build_args() report_file = args.report_file collect_stats_parallel( - org_id, report_file, args.detailed, args.branch, args.include_run_info + org_id, report_file, args.detailed, args.branch, args.include_run_info, args.include_app_apps ) From c1e596545d6984e3128e845415b8bd7e296bca25 Mon Sep 17 00:00:00 2001 From: Elango Senthilnathan Date: Fri, 9 Feb 2024 13:48:39 -0800 Subject: [PATCH 2/3] Included support for multiple apps export in export.py and added team admin to stats.py --- shiftleft-utils/common.py | 30 ++++++++++++++++++++++++++++++ shiftleft-utils/export.py | 5 ++++- shiftleft-utils/stats.py | 19 ++++++++++++++++++- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/shiftleft-utils/common.py b/shiftleft-utils/common.py index 3090adf..666a1ac 100644 --- a/shiftleft-utils/common.py +++ b/shiftleft-utils/common.py @@ -76,6 +76,21 @@ def get_all_apps(org_id): ) return None +def get_all_users(org_id): + """Return all the teams for the given organization""" + list_users_url = f"https://{config.SHIFTLEFT_API_HOST}/api/v4/orgs/{org_id}/rbac/users" + r = requests.get(list_users_url, headers=headers) + if r.ok: + raw_response = r.json() + if raw_response and raw_response.get("response"): + teams_list = raw_response.get("response") + return teams_list + else: + print( + f"Unable to retrieve users list for the organization {org_id} due to {r.status_code} error" + ) + return None + def get_all_teams(org_id): """Return all the teams for the given organization""" list_teams_url = f"https://{config.SHIFTLEFT_API_HOST}/api/v4/orgs/{org_id}/rbac/teams" @@ -91,6 +106,21 @@ def get_all_teams(org_id): ) return None +def get_team_members(org_id, team_id): + """Return all the teams for the given organization""" + list_team_members_url = f"https://{config.SHIFTLEFT_API_HOST}/api/v4/orgs/{org_id}/rbac/teams/{team_id}" + r = requests.get(list_team_members_url, headers=headers) + if r.ok: + raw_response = r.json() + if raw_response and raw_response.get("response"): + teams_list = raw_response.get("response") + return teams_list + else: + print( + f"Unable to retrieve team members list for the organization {team_id} due to {r.status_code} error" + ) + return None + def get_all_findings(org_id, app_name, version): """Method to retrieve all findings""" with Progress( diff --git a/shiftleft-utils/export.py b/shiftleft-utils/export.py index 5bd8b10..18d532a 100644 --- a/shiftleft-utils/export.py +++ b/shiftleft-utils/export.py @@ -399,6 +399,8 @@ def build_args(): dest="app_name", help="App name", default=config.SHIFTLEFT_APP, + action='append', + nargs='+', ) parser.add_argument( "-o", @@ -438,7 +440,8 @@ def build_args(): args = build_args() app_list = [] if args.app_name: - app_list.append({"id": args.app_name, "name": args.app_name}) + for eachApp in args.app_name: + app_list.append({"id": eachApp[0], "name": eachApp[0]}) report_file = args.report_file reports_dir = args.reports_dir format = args.format diff --git a/shiftleft-utils/stats.py b/shiftleft-utils/stats.py index d34881c..9235577 100644 --- a/shiftleft-utils/stats.py +++ b/shiftleft-utils/stats.py @@ -24,6 +24,8 @@ get_findings_url, get_scan_run, get_all_teams, + get_team_members, + get_all_users, headers, ) @@ -38,7 +40,7 @@ def to_arr(counts_dict): def process_app( - progress, task, org_id, report_file, app, detailed, branch, include_run_info, include_app_apps, teams_list + progress, task, org_id, report_file, app, detailed, branch, include_run_info, include_app_apps, teams_list, user_dict, ): start = time.time() app_id = app.get("id") @@ -297,15 +299,22 @@ def process_app( scanLang = "" scanExp = "" appTeam = "" + teamAdmins = "" for eachTeam in teams_list: if eachTeam.get("projects"): if appName in eachTeam.get("projects"): appTeam = eachTeam.get("team_name") + teamMembers = get_team_members(org_id, eachTeam.get("team_id")).get("members") + for eachMember in teamMembers: + memberRoleinTeam = eachMember.get("team_role_aliases") + if "TEAM_MANAGER" in memberRoleinTeam or "TEAM_ADMIN" in memberRoleinTeam: + teamAdmins = str(user_dict[eachMember.get('user_id_v2')]) + ", " + teamAdmins return [ appName, app_group, appTeam, + teamAdmins.rstrip(', '), isActive, app_branch, appVersion, @@ -354,6 +363,7 @@ def write_to_csv(report_file, row): "App", "App Group", "Team Name", + "Team Admins", "ActiveApp", "Branch", "Version", @@ -403,6 +413,12 @@ def collect_stats_parallel(org_id, report_file, detailed, branch, include_run_in """Method to collect stats for all apps to a csv""" apps_list = get_all_apps(org_id) teams_list = get_all_teams(org_id) + users_list = get_all_users(org_id) + + user_dict = {} + for eachUser in users_list: + user_dict[eachUser.get("id_v2")] = eachUser.get("email") + if not apps_list: console.print("No apps were found in this organization") return @@ -438,6 +454,7 @@ def collect_stats_parallel(org_id, report_file, detailed, branch, include_run_in include_run_info, include_all_apps, teams_list, + user_dict, ) rows.append(row) rows = dask.compute(*rows) From da6f228c527522fdf83aa45ee7696781b7530e9d Mon Sep 17 00:00:00 2001 From: Elango Senthilnathan Date: Mon, 12 Feb 2024 12:04:06 -0800 Subject: [PATCH 3/3] Fixed a big related to Team Admin data pull --- shiftleft-utils/stats.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shiftleft-utils/stats.py b/shiftleft-utils/stats.py index 9235577..09c90b8 100644 --- a/shiftleft-utils/stats.py +++ b/shiftleft-utils/stats.py @@ -305,10 +305,11 @@ def process_app( if appName in eachTeam.get("projects"): appTeam = eachTeam.get("team_name") teamMembers = get_team_members(org_id, eachTeam.get("team_id")).get("members") - for eachMember in teamMembers: - memberRoleinTeam = eachMember.get("team_role_aliases") - if "TEAM_MANAGER" in memberRoleinTeam or "TEAM_ADMIN" in memberRoleinTeam: - teamAdmins = str(user_dict[eachMember.get('user_id_v2')]) + ", " + teamAdmins + if teamMembers: + for eachMember in teamMembers: + memberRoleinTeam = eachMember.get("team_role_aliases") + if "TEAM_MANAGER" in memberRoleinTeam or "TEAM_ADMIN" in memberRoleinTeam: + teamAdmins = str(user_dict[eachMember.get('user_id_v2')]) + ", " + teamAdmins return [ appName,