From 4e2e47fa876c770ace3759e40008d560c4d20310 Mon Sep 17 00:00:00 2001 From: Dragana Grbic Date: Thu, 12 Sep 2024 20:27:41 +0000 Subject: [PATCH 1/2] feature: HPCToolkit integration --- thicket/thicket.py | 120 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/thicket/thicket.py b/thicket/thicket.py index 32f2c2d2..0e4cd55e 100644 --- a/thicket/thicket.py +++ b/thicket/thicket.py @@ -276,6 +276,126 @@ def from_hpctoolkit( dirname, ) + @staticmethod + def from_hpctoolkit_latest( + dirname, + intersection=False, + disable_tqdm=False, + directory_mapping=None, + parallel_profiles_mode=False, + max_depth=None, + min_percentage_of_application_time=None, + exclude_mpi_function_details=False, + exclude_openmp_function_details=False, + exclude_cuda_function_details=False, + exclude_system_libraries_source_code=False, + exclude_function_call_lines=False, + exclude_no_source_code_instructions=False, + exclude_instructions=False, + exclude_non_function_nodes=False, + label_function_nodes=True, + metric_names=None, + metric_scopes=None, + summary_metrics=None, + profile_ranks=None, + sort_method="nodes", + ): + """Create a GraphFrame using hatchet's HPCToolkit reader and use its attributes + to make a new thicket. + + Arguments: + dirname (str): directory containing HPCToolkit performance data + intersection (bool): whether to perform intersection or union (default) + disable_tqdm (bool): whether to display tqdm progress bar + + Returns: + (thicket): new thicket containing HPCToolkit profile data + """ + + if parallel_profiles_mode: + graphframe = GraphFrame.from_hpctoolkit_latest( + dirname, + directory_mapping=directory_mapping, + parallel_profiles_mode=True, + max_depth=max_depth, + min_percentage_of_application_time=min_percentage_of_application_time, + exclude_mpi_function_details=exclude_mpi_function_details, + exclude_openmp_function_details=exclude_openmp_function_details, + exclude_cuda_function_details=exclude_cuda_function_details, + exclude_system_libraries_source_code=exclude_system_libraries_source_code, + exclude_function_call_lines=exclude_function_call_lines, + exclude_no_source_code_instructions=exclude_no_source_code_instructions, + exclude_instructions=exclude_instructions, + exclude_non_function_nodes=exclude_non_function_nodes, + label_function_nodes=label_function_nodes, + metric_names=metric_names, + metric_scopes=metric_scopes, + summary_metrics=summary_metrics, + profile_ranks=profile_ranks, + ) + + profile_ids = ( + graphframe.dataframe.index.get_level_values(1).unique().to_numpy() + ) + diff_dataframe = [] + + def visit_node(node) -> None: + temp = graphframe.dataframe.loc[node, :].index.unique().to_numpy() + + for profile in np.setdiff1d(profile_ids, temp): + diff_dataframe.append({"node": node, "profile": profile}) + + for child in node.children: + visit_node(child) + + root = graphframe.graph.roots[0] + visit_node(root) + + if len(diff_dataframe): + diff_dataframe = pd.DataFrame(diff_dataframe).set_index( + ["node", "profile"] + ) + graphframe.dataframe = pd.concat([graphframe.dataframe, diff_dataframe]) + + return Thicket.thicketize_graphframe(graphframe, None) + + else: + + thicket = Thicket.reader_dispatch( + GraphFrame.from_hpctoolkit_latest, + intersection, + disable_tqdm, + dirname, + directory_mapping=directory_mapping, + max_depth=max_depth, + min_percentage_of_application_time=min_percentage_of_application_time, + exclude_mpi_function_details=exclude_mpi_function_details, + exclude_openmp_function_details=exclude_openmp_function_details, + exclude_cuda_function_details=exclude_cuda_function_details, + exclude_system_libraries_source_code=exclude_system_libraries_source_code, + exclude_function_call_lines=exclude_function_call_lines, + exclude_no_source_code_instructions=exclude_no_source_code_instructions, + exclude_instructions=exclude_instructions, + metric_names=metric_names, + metric_scopes=metric_scopes, + summary_metrics=summary_metrics, + profile_ranks=profile_ranks, + ) + thicket.dataframe = ( + thicket.dataframe.reset_index() + .merge( + thicket.metadata[["nodes" if sort_method == "nodes" else "date"]], + how="left", + left_on="profile", + right_on="profile", + ) + .set_index(["node", "profile"]) + .sort_values("nodes" if sort_method == "nodes" else "date") + .drop("nodes" if sort_method == "nodes" else "date", axis=1) + ) + + return thicket + @staticmethod def from_caliperreader( filename_or_caliperreader, From 7be0d49bbb4c91bd552bf7cbb41a4359a574b9d5 Mon Sep 17 00:00:00 2001 From: Dragana GRBIC Date: Thu, 23 Jan 2025 19:44:55 -0600 Subject: [PATCH 2/2] minnor fix --- thicket/thicket.py | 1 + 1 file changed, 1 insertion(+) diff --git a/thicket/thicket.py b/thicket/thicket.py index 0e4cd55e..801cea30 100644 --- a/thicket/thicket.py +++ b/thicket/thicket.py @@ -364,6 +364,7 @@ def visit_node(node) -> None: thicket = Thicket.reader_dispatch( GraphFrame.from_hpctoolkit_latest, intersection, + None, disable_tqdm, dirname, directory_mapping=directory_mapping,