Support relocatable and standalone ParEval (#54)

* add build config flag to run all

* update test-serial-outputs to match newest prompts

* run drivers in driver root for relocatable runs

---------

Co-authored-by: Daniel Nichols <[email protected]>

Author: Dando18

drivers/build-configs.json

@@ -0,0 +1,9 @@
+{
+    "serial": {"CXX": "g++", "CXXFLAGS": "-std=c++17 -O3"},
+    "omp": {"CXX": "g++", "CXXFLAGS": "-std=c++17 -O3 -fopenmp"},
+    "mpi": {"CXX": "mpicxx", "CXXFLAGS": "-std=c++17 -O3"},
+    "mpi+omp": {"CXX": "mpicxx", "CXXFLAGS": "-std=c++17 -O3 -fopenmp"},
+    "kokkos": {"CXX": "g++", "CXXFLAGS": "-std=c++17 -O3 -fopenmp -I../tpl/kokkos/build/include ../tpl/kokkos/build/lib64/libkokkoscore.a ../tpl/kokkos/build/lib64/libkokkoscontainers.a ../tpl/kokkos/build/lib64/libkokkossimd.a"},
+    "cuda": {"CXX": "nvcc", "CXXFLAGS": "-std=c++17 --generate-code arch=compute_80,code=sm_80 -O3 -Xcompiler \"-std=c++17 -O3\""},
+    "hip": {"CXX": "hipcc", "CXXFLAGS": "-std=c++17 -O3 -Xcompiler \"-std=c++17\" -Xcompiler \"-O3\" -Wno-unused-result"}
+}

drivers/cpp/cpp_driver_wrapper.py

@@ -41,7 +41,7 @@
 def build_kokkos(driver_src: PathLike, output_root: PathLike, problem_size: str = "(1<<20)"):
     """ Custom steps for the Kokkos programs, since they require cmake """
     # cp cmake file into the output directory
-    cmake_path = "cpp/KokkosCMakeLists.txt"
+    cmake_path = os.path.join("cpp", "KokkosCMakeLists.txt")
     cmake_dest = os.path.join(output_root, "CMakeLists.txt")
     run_command(f"cp {cmake_path} {cmake_dest}", dry=False)
 
@@ -57,6 +57,8 @@ class CppDriverWrapper(DriverWrapper):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+
+        self.build_configs = self.build_configs or COMPILER_SETTINGS
         self.model_driver_file = os.path.join("cpp", "models", DRIVER_MAP[self.parallelism_model])
 
     def write_source(self, content: str, fpath: PathLike) -> bool:
@@ -125,7 +127,7 @@ def test_single_output(self, prompt: str, output: str, test_driver_file: PathLik
 
             # compile and run the output
             exec_path = os.path.join(tmpdir, "a.out")
-            compiler_kwargs = copy.deepcopy(COMPILER_SETTINGS[self.parallelism_model])
+            compiler_kwargs = copy.deepcopy(self.build_configs[self.parallelism_model])
             compiler_kwargs["problem_size"] = problem_size  # for kokkos
             compiler_kwargs["CXXFLAGS"] += f" -I{tmpdir} -DDRIVER_PROBLEM_SIZE=\"{problem_size}\""
             build_result = self.compile(self.model_driver_file, test_driver_file, output_path=exec_path, **compiler_kwargs)

drivers/driver_wrapper.py

@@ -167,6 +167,7 @@ def __init__(
         self, 
         parallelism_model: str = "serial", 
         launch_configs: dict = {"format": "{exec_path} {args}", "params": [{}]},
+        build_configs: Optional[dict] = None,
         problem_sizes: dict = {},
         scratch_dir: Optional[PathLike] = None,
         build_timeout: int = 20,
@@ -180,6 +181,7 @@ def __init__(
         self.validator = VALIDATORS[parallelism_model]
         self.scratch_dir = scratch_dir
         self.launch_configs = launch_configs[parallelism_model]
+        self.build_configs = build_configs
         self.problem_sizes = problem_sizes
         self.build_timeout = build_timeout
         self.run_timeout = run_timeout
@@ -213,15 +215,15 @@ def test_single_output(self, prompt: str, output: str, test_driver_file: PathLik
 
     def test_all_outputs_in_prompt(self, prompt: dict) -> dict:
         """ Run all the generated outputs in the given prompt. """
-        root = prompt["language"]
+        lang = prompt["language"]
         type = prompt["problem_type"]
         name = prompt["name"]
         ext = LANGUAGE_EXTENSIONS[prompt["language"]]
-        if root == "cpp" and self.parallelism_model in ["cuda", "hip"]:
+        if lang == "cpp" and self.parallelism_model in ["cuda", "hip"]:
             ext = ".cu"
-        driver_root = f"{name}"
+        driver_dirname = f"{name}"
         driver_base = DRIVER_MAP[self.parallelism_model]
-        test_driver_file = os.path.join(root, "benchmarks", type, driver_root, driver_base + ext)
+        test_driver_file = os.path.join(lang, "benchmarks", type, driver_dirname, driver_base + ext)
         problem_size = self.problem_sizes.get(name, {}).get(self.parallelism_model, "(1<<18)")
 
         outputs = []

drivers/run-all.py

@@ -5,10 +5,10 @@
 """
 # std imports
 from argparse import ArgumentParser
+import contextlib
 import json
 import logging
 import os
-import tempfile
 from typing import Optional
 
 # tpl imports
@@ -30,8 +30,11 @@ def get_args():
     parser.add_argument("input_json", type=str, help="Input JSON file containing the test cases.")
     parser.add_argument("-o", "--output", type=str, help="Output JSON file containing the results.")
     parser.add_argument("--scratch-dir", type=str, help="If provided, put scratch files here.")
+    parser.add_argument("--driver-root", type=str, help="Where to look for the driver files, if not in cwd.")
     parser.add_argument("--launch-configs", type=str, default="launch-configs.json", 
         help="config for how to run samples.")
+    parser.add_argument("--build-configs", type=str, default="build-configs.json",
+        help="config for how to build samples. If not provided, will use the default build settings for each model.")
     parser.add_argument("--problem-sizes", type=str, default="problem-sizes.json", 
         help="config for how to run samples.")
     parser.add_argument("--yes-to-all", action="store_true", help="If provided, automatically answer yes to all prompts.")
@@ -56,11 +59,19 @@ def get_args():
     parser.add_argument("--log-runs", action="store_true", help="Display the stderr and stdout of runs.")
     return parser.parse_args()
 
-def get_driver(prompt: dict, scratch_dir: Optional[os.PathLike], launch_configs: dict, problem_sizes: dict, dry: bool, **kwargs) -> DriverWrapper:
+def get_driver(
+    prompt: dict, 
+    scratch_dir: Optional[os.PathLike], 
+    launch_configs: dict, 
+    build_configs: dict, 
+    problem_sizes: dict, 
+    dry: bool, 
+    **kwargs
+) -> DriverWrapper:
     """ Get the language drive wrapper for this prompt """
     driver_cls = LANGUAGE_DRIVERS[prompt["language"]]
     return driver_cls(parallelism_model=prompt["parallelism_model"], launch_configs=launch_configs, 
-        problem_sizes=problem_sizes, scratch_dir=scratch_dir, dry=dry, **kwargs)
+        build_configs=build_configs, problem_sizes=problem_sizes, scratch_dir=scratch_dir, dry=dry, **kwargs)
 
 def already_has_results(prompt: dict) -> bool:
     """ Check if a prompt already has results stored in it. """
@@ -102,10 +113,25 @@ def main():
     launch_configs = load_json(args.launch_configs)
     logging.info(f"Loaded launch configs from {args.launch_configs}.")
 
+    # load build configs
+    build_configs = load_json(args.build_configs)
+    logging.info(f"Loaded build configs from {args.build_configs}.")
+
     # load problem sizes
     problem_sizes = load_json(args.problem_sizes)
     logging.info(f"Loaded problem sizes from {args.problem_sizes}.")
 
+    # set driver root; If provided, use user argument. If it's not provided, then check if the PAREVAL_ROOT environment
+    # variable is set, then use "${PAREVAL_ROOT}/drivers" as the root. If neither is set, then use the location of 
+    # this script as the root.
+    if args.driver_root:
+        DRIVER_ROOT = args.driver_root
+    elif "PAREVAL_ROOT" in os.environ:
+        DRIVER_ROOT = os.path.join(os.environ["PAREVAL_ROOT"], "drivers")
+    else:
+        DRIVER_ROOT = os.path.dirname(os.path.abspath(__file__))
+    logging.info(f"Using driver root: {DRIVER_ROOT}")
+
     # gather the list of parallelism models to test
     models_to_test = args.include_models if args.include_models else ["serial", "omp", "mpi", "mpi+omp", "kokkos", "cuda", "hip"]
     if args.exclude_models:
@@ -139,15 +165,18 @@ def main():
             prompt, 
             args.scratch_dir, 
             launch_configs, 
+            build_configs,
             problem_sizes,
             args.dry, 
             display_build_errors=args.log_build_errors,
             display_runs=args.log_runs,
             early_exit_runs=args.early_exit_runs,
             build_timeout=args.build_timeout,
-            run_timeout=args.run_timeout
+            run_timeout=args.run_timeout,
         )
-        driver.test_all_outputs_in_prompt(prompt)
+
+        with contextlib.chdir(DRIVER_ROOT):
+            driver.test_all_outputs_in_prompt(prompt)
 
         # go ahead and write out outputs now
         if args.output and args.output != '-':