Custom json output path with Make variable substitution support

README.md

@@ -146,6 +146,9 @@ refresh_compile_commands(
     # Wildcard patterns, like //... for everything, *are* allowed here, just like a build.
       # As are additional targets (+) and subtractions (-), like in bazel query https://docs.bazel.build/versions/main/query.html#expressions
     # And if you're working on a header-only library, specify a test or binary target that compiles it.
+
+    # Optionally, you can change the output path to the compile_commands.json file. Make Variable Substitutions are also supported: https://bazel.build/reference/be/make-variables
+    json_output_path = "myFolder/NewCompileCommands.json",
 )
 ```
 
@@ -163,6 +166,23 @@ Adding `exclude_external_sources = True` and `exclude_headers = "external"` can
 
 For now, we'd suggest continuing on to set up `clangd` (below). Thereafter, if you your project proves to be large enough that it stretches the capacity of `clangd` and/or this tool to index quickly, take a look at the docs at the top of [`refresh_compile_commands.bzl`](./refresh_compile_commands.bzl) for instructions on how to tune those flags and others.
 
+### ⚠️ EXPERIMENTAL FEATURE: --symlink-prefix support
+
+Bazel allows use of the --symlink-prefix argument, commonly in `.bazelrc`, ie `build --symlink_prefix=build/bazel-`. Using this can help keep your workspace tidy by changing the names of the generated symlinks or even putting them in a subdirectory. Experimental support for this feature can be used by adding `experimental_symlink_prefix = <your_prefix>` to your rule. Make sure the prefix you add here matches what you use with bazel commands/.bazelrc! For example:
+
+```Starlark
+refresh_compile_commands(
+    name = "refresh_compile_commands",
+    experimental_symlink_prefix = "build/bazel-",
+)
+```
+
+This will tell the tool to expect symlinks with the prefix `build/bazel-` instead of `bazel-`. It will also place the `external` directory symlink in the subdirectory associated with the prefix (if any), ie `build/` to keep things tidy and the subdirectory matching bazel's build workspace.
+
+**IMPORTANT ADDITIONAL REQUIREMENT:**
+
+If you use a symlink prefix with a subdirectory, the `external` folder will no longer be in the project root. Bazel will no longer ignore it by default, and will try to look for targets inside it too, which will cause many commands to fail. To avoid this, you'll need to add a `.bazelignore` file to the root of your project, and add `build` or `build/external` to it.
+
 ## Editor Setup — for autocomplete based on `compile_commands.json`
 
 

refresh.template.py

@@ -36,6 +36,9 @@
 import types
 import typing  # MIN_PY=3.9: Switch e.g. typing.List[str] -> List[str]
 
+symlink_prefix = {experimental_symlink_prefix} or "bazel-"
+external_symlink_path = ({experimental_symlink_prefix}.rsplit("/", 1)[0] + "/external" if {experimental_symlink_prefix} and "/" in {experimental_symlink_prefix} else "external") # External should be moved to the same directory as the symlink_prefix files if they have been moved to a subdirectory
+
 
 @enum.unique
 class SGR(enum.Enum):
@@ -507,7 +510,7 @@ def _file_is_in_main_workspace_and_not_external(file_str: str):
 
     # some/file.h, but not external/some/file.h
     # also allows for things like bazel-out/generated/file.h
-    if _is_relative_to(file_path, pathlib.PurePath("external")):
+    if _is_relative_to(file_path, pathlib.PurePath(external_symlink_path)):
         return False
 
     # ... but, ignore files in e.g. bazel-out/<configuration>/bin/external/
@@ -1107,6 +1110,39 @@ def _get_cpp_command_for_files(compile_action):
     if 'PATH' not in compile_action.environmentVariables: # Bazel only adds if --incompatible_strict_action_env is passed--and otherwise inherits.
         compile_action.environmentVariables['PATH'] = os.environ['PATH']
 
+    if {experimental_symlink_prefix}:
+        # Regex matching for experimentally aliased symlinks
+        top_level_symlinks = r"(?P<prefix>bazel-out|external)"  # The symlinks we can expect to be used as the start of paths in compile_commands.json
+        flag = (
+            r"[-/][a-zA-Z0-9\-_]+"  # A flag like "-I" or "-frandom-seed" or "-something_weird"
+        )
+        full_regex = re.compile(
+            r"""
+            ^                           # Start at the beginning of the line
+            (?P<content>                # Capture content before prefix
+            (?:{flag})?                 # Optionally match a single flag, ie for "-Ibazel-out/..."
+            =?                          # Optionally match a single equals sign, ie for "-frandom-seed=bazel-out/..."
+            )                           # End capture content before prefix
+            {top_level_symlinks}        # Match one of the top level symlinks
+            /                           # End with the OS path separator
+            """.format(
+                flag=flag,
+                top_level_symlinks=top_level_symlinks,
+            ),
+            re.VERBOSE,
+        )
+
+        def replace_prefix(match):
+            content = match.group('content')
+            prefix = match.group('prefix')
+            if prefix == "bazel-out":
+                return f"{content}{symlink_prefix}out/"
+            elif prefix == "external":
+                return f"{content}{external_symlink_path}/"
+
+        for idx, arg in enumerate(compile_action.arguments):
+            compile_action.arguments[idx] = re.sub(full_regex, replace_prefix, arg)
+
     # Patch command by platform, revealing any hidden arguments.
     compile_action.arguments = _apple_platform_patch(compile_action.arguments)
     compile_action.arguments = _emscripten_platform_patch(compile_action)
@@ -1279,19 +1315,22 @@ def _get_commands(target: str, flags: str):
 def _ensure_external_workspaces_link_exists():
     """Postcondition: Either //external points into Bazel's fullest set of external workspaces in output_base, or we've exited with an error that'll help the user resolve the issue."""
     is_windows = os.name == 'nt'
-    source = pathlib.Path('external')
+    source = pathlib.Path(external_symlink_path)
 
-    if not os.path.lexists('bazel-out'):
-        log_error(">>> //bazel-out is missing. Please remove --symlink_prefix and --experimental_convenience_symlinks, so the workspace mirrors the compilation environment.")
+    if not os.path.lexists(f'{symlink_prefix}out'):
+        if {experimental_symlink_prefix}:
+            log_error(f">>> //{symlink_prefix}out is missing. Double check your --experimental_symlink_prefix location or disable this experimental feature.")
+        else:
+            log_error(">>> //bazel-out is missing. Please remove --symlink_prefix and --experimental_convenience_symlinks, so the workspace mirrors the compilation environment. Alternatively, you can try adding --experimental_symlink_prefix=<your_prefix> to the extractor command to experimentally support the --symlink_prefix bazel flag.")
         # Crossref: https://github.com/hedronvision/bazel-compile-commands-extractor/issues/14 https://github.com/hedronvision/bazel-compile-commands-extractor/pull/65
         # Note: experimental_no_product_name_out_symlink is now enabled by default. See https://github.com/bazelbuild/bazel/commit/06bd3e8c0cd390f077303be682e9dec7baf17af2
         sys.exit(1)
 
     # Traverse into output_base via bazel-out, keeping the workspace position-independent, so it can be moved without rerunning
-    dest = pathlib.Path('bazel-out/../../../external')
-    if is_windows:
+    dest = pathlib.Path(f'{symlink_prefix}out/../../../external')
+    if is_windows or {experimental_symlink_prefix}: # When using experimental prefix, resolution needs to be done in two steps as well
         # On Windows, unfortunately, bazel-out is a junction, and accessing .. of a junction brings you back out the way you came. So we have to resolve bazel-out first. Not position-independent, but I think the best we can do
-        dest = (pathlib.Path('bazel-out').resolve()/'../../../external').resolve()
+        dest = (pathlib.Path(f'{symlink_prefix}out').resolve()/'../../../external').resolve()
 
     # Handle problem cases where //external exists
     if os.path.lexists(source): # MIN_PY=3.12: use source.exists(follow_symlinks=False), here and elsewhere.
@@ -1354,9 +1393,9 @@ def _ensure_gitignore_entries_exist():
 
     # Each (pattern, explanation) will be added to the `.gitignore` file if the pattern isn't present.
     needed_entries = [
-        (f'/{pattern_prefix}external', "# Ignore the `external` link (that is added by `bazel-compile-commands-extractor`). The link differs between macOS/Linux and Windows, so it shouldn't be checked in. The pattern must not end with a trailing `/` because it's a symlink on macOS/Linux."),
-        (f'/{pattern_prefix}bazel-*', "# Ignore links to Bazel's output. The pattern needs the `*` because people can change the name of the directory into which your repository is cloned (changing the `bazel-<workspace_name>` symlink), and must not end with a trailing `/` because it's a symlink on macOS/Linux. This ignore pattern should almost certainly be checked into a .gitignore in your workspace root, too, for folks who don't use this tool."),
-        (f'/{pattern_prefix}compile_commands.json', "# Ignore generated output. Although valuable (after all, the primary purpose of `bazel-compile-commands-extractor` is to produce `compile_commands.json`!), it should not be checked in."),
+        (f'/{pattern_prefix}{external_symlink_path}', "# Ignore the `external` link (that is added by `bazel-compile-commands-extractor`). The link differs between macOS/Linux and Windows, so it shouldn't be checked in. The pattern must not end with a trailing `/` because it's a symlink on macOS/Linux."),
+        (f'/{pattern_prefix}{symlink_prefix}*', "# Ignore links to Bazel's output. The pattern needs the `*` because people can change the name of the directory into which your repository is cloned (changing the `bazel-<workspace_name>` symlink), and must not end with a trailing `/` because it's a symlink on macOS/Linux. This ignore pattern should almost certainly be checked into a .gitignore in your workspace root, too, for folks who don't use this tool."),
+        (f'/{pattern_prefix}{json_output_path}', "# Ignore generated output. Although valuable (after all, the primary purpose of `bazel-compile-commands-extractor` is to produce `compile_commands.json`!), it should not be checked in."),
         ('.cache/', "# Ignore the directory in which `clangd` stores its local index."),
     ]
 
@@ -1410,15 +1449,20 @@ def main():
         compile_command_entries.extend(_get_commands(target, flags))
 
     if not compile_command_entries:
-        log_error(""">>> Not (over)writing compile_commands.json, since no commands were extracted and an empty file is of no use.
+        log_error(f""">>> Not (over)writing {json_output_path}, since no commands were extracted and an empty file is of no use.
     There should be actionable warnings, above, that led to this.""")
         sys.exit(1)
 
+    # Create the containing directory if it doesn't exist.
+    json_output_dir = pathlib.Path({json_output_path}).parent
+    json_output_dir.mkdir(parents=True, exist_ok=True)
+
     # Chain output into compile_commands.json
-    with open('compile_commands.json', 'w') as output_file:
+    with open({json_output_path}, 'w') as output_file:
         json.dump(
             compile_command_entries,
             output_file,
             indent=2, # Yay, human readability!
             check_circular=False # For speed.
         )
+

refresh_compile_commands.bzl

@@ -49,6 +49,12 @@ refresh_compile_commands(
         # exclude_headers = "external",
     # Still not fast enough?
         # Make sure you're specifying just the targets you care about by setting `targets`, above.
+
+    # Want to change the filename or path for the output compile_commands.json file?
+        # json_output_path = "NewCompileCommands.json",
+    # Make Variable Substitutions are also supported: https://bazel.build/reference/be/make-variables
+        # json_output_path = "$(BINDIR)/compile_commands/bazel_compile_commands.json",
+        # This will result in bazel-out/compile_commands/bazel_compile_commands.json or a slightly different directory if you are using --symlink-prefix or similar to change where bazel's binary directory is.
 ```
 """
 
@@ -64,6 +70,8 @@ def refresh_compile_commands(
         targets = None,
         exclude_headers = None,
         exclude_external_sources = False,
+        experimental_symlink_prefix = None,
+        json_output_path = "compile_commands.json",
         **kwargs):  # For the other common attributes. Tags, compatible_with, etc. https://docs.bazel.build/versions/main/be/common-definitions.html#common-attributes.
     # Convert the various, acceptable target shorthands into the dictionary format
     # In Python, `type(x) == y` is an antipattern, but [Starlark doesn't support inheritance](https://bazel.build/rules/language), so `isinstance` doesn't exist, and this is the correct way to switch on type.
@@ -89,7 +97,7 @@ def refresh_compile_commands(
 
     # Generate the core, runnable python script from refresh.template.py
     script_name = name + ".py"
-    _expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
+    _expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, experimental_symlink_prefix = experimental_symlink_prefix, json_output_path = json_output_path, **kwargs)
 
     # Combine them so the wrapper calls the main script
     native.py_binary(
@@ -104,6 +112,13 @@ def refresh_compile_commands(
 def _expand_template_impl(ctx):
     """Inject targets of interest--and other settings--into refresh.template.py, and set it up to be run."""
     script = ctx.actions.declare_file(ctx.attr.name)
+
+    def _symlink_prefix_replacer(path):
+        """Replace the bazel- prefix with the experimental --symlink-prefix one if it exists."""
+        if path.startswith("bazel-") and ctx.attr.experimental_symlink_prefix:
+            return path.replace("bazel-", ctx.attr.experimental_symlink_prefix, 1)
+        return path
+
     ctx.actions.expand_template(
         output = script,
         is_executable = True,
@@ -114,7 +129,12 @@ def _expand_template_impl(ctx):
             "        {windows_default_include_paths}": "\n".join(["        %r," % path for path in find_cpp_toolchain(ctx).built_in_include_directories]),  # find_cpp_toolchain is from https://docs.bazel.build/versions/main/integrating-with-rules-cc.html
             "{exclude_headers}": repr(ctx.attr.exclude_headers),
             "{exclude_external_sources}": repr(ctx.attr.exclude_external_sources),
+            "{json_output_path}": repr(ctx.expand_make_variables("json_output_path_expansion", ctx.attr.json_output_path, {
+                "BINDIR": _symlink_prefix_replacer(ctx.bin_dir.path),
+                "GENDIR": _symlink_prefix_replacer(ctx.genfiles_dir.path),
+            })),  # Subject to make variable substitutions
             "{print_args_executable}": repr(ctx.executable._print_args_executable.path),
+            "{experimental_symlink_prefix}": repr(ctx.attr.experimental_symlink_prefix),
         },
     )
     return DefaultInfo(files = depset([script]))
@@ -124,6 +144,8 @@ _expand_template = rule(
         "labels_to_flags": attr.string_dict(mandatory = True),  # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
         "exclude_external_sources": attr.bool(default = False),
         "exclude_headers": attr.string(values = ["all", "external", ""]),  # "" needed only for compatibility with Bazel < 3.6.0
+        "experimental_symlink_prefix": attr.string(),
+        "json_output_path": attr.string(),
         "_script_template": attr.label(allow_single_file = True, default = "refresh.template.py"),
         "_print_args_executable": attr.label(executable = True, cfg = "target", default = "//:print_args"),
         # For Windows INCLUDE. If this were eliminated, for example by the resolution of https://github.com/clangd/clangd/issues/123, we'd be able to just use a macro and skylib's expand_template rule: https://github.com/bazelbuild/bazel-skylib/pull/330