Merge pull request #39 from sandialabs/tjfulle/cdash-subproject-labels

Add canary_cdash_subproject_label hook for adding cdash subproject labels

Author: tjfulle

src/_canary/config/__init__.py

@@ -8,6 +8,7 @@
 from typing import Generator
 
 from .config import Config
+from .config import ConfigScope  # noqa: F401
 from .config import get_scope_filename  # noqa: F401
 from .rpool import ResourcePool  # noqa: F401
 from .rpool import ResourceUnavailable  # noqa: F401
@@ -53,11 +54,11 @@ def __getattr__(name: str) -> Any:
 
 
 @contextmanager
-def override() -> Generator[None, None, None]:
+def override() -> Generator[Config, None, None]:
     global _config
     save_config = _config
     try:
         _config = Config()
-        yield
+        yield _config
     finally:
         _config = save_config

src/_canary/main.py

@@ -57,6 +57,7 @@ def main(argv: Sequence[str] | None = None) -> int:
             color.set_color_when(args.color)
 
         config.set_main_options(args)
+        config.plugin_manager.hook.canary_addhooks(pluginmanager=config.plugin_manager)
         config.plugin_manager.hook.canary_configure(config=config)
         command = parser.get_command(args.command)
         if command is None:
@@ -115,23 +116,25 @@ def __init__(self, command_name: str, debug: bool = False) -> None:
     def __call__(self, *args_in: str, fail_on_error: bool = True) -> int:
         try:
             global reraise
-            with config.temporary_scope() as scope:
+            with config.override() as cfg:
                 save_reraise: bool | None = None
                 if self.debug:
-                    scope["config:debug"] = True
+                    scope = config.ConfigScope("tmp", None, {"config": {"debug": True}})
+                    cfg.push_scope(scope)
                     save_reraise = reraise
                     reraise = True
                 argv = [self.command.name] + list(args_in)
                 parser = make_argument_parser()
-                for command in config.plugin_manager.get_subcommands():
+                for command in cfg.plugin_manager.get_subcommands():
                     parser.add_command(command)
                 with monkeypatch.context() as mp:
                     mp.setattr(parser, "add_argument", parser.add_plugin_argument)
                     mp.setattr(parser, "add_argument_group", parser.add_plugin_argument_group)
-                    config.plugin_manager.hook.canary_addoption(parser=parser)
+                    cfg.plugin_manager.hook.canary_addoption(parser=parser)
                 args = parser.parse_args(argv)
-                config.set_main_options(args)
-                config.plugin_manager.hook.canary_configure(config=config)
+                cfg.set_main_options(args)
+                cfg.plugin_manager.hook.canary_addhooks(pluginmanager=cfg.plugin_manager)
+                cfg.plugin_manager.hook.canary_configure(config=cfg)
                 rc = self.command.execute(args)
                 self.returncode = rc
         except Exception:

src/_canary/plugins/builtin/cdash/__init__.py

@@ -11,6 +11,7 @@
 
 from ....util.string import csvsplit
 from ...hookspec import hookimpl
+from ...hookspec import hookspec
 from ...types import CanaryReporter
 from .cdash_html_summary import cdash_summary
 from .gitlab_issue_generator import create_issues_from_failed_tests
@@ -19,13 +20,32 @@
 if TYPE_CHECKING:
     from ....config.argparsing import Parser
     from ....session import Session
+    from ....testcase import TestCase
+    from ...manager import CanaryPluginManager
+
+
+class CDashHooks:
+    @hookspec
+    def canary_cdash_labels_for_subproject(self) -> list[str] | None:
+        """Return a list of subproject labels to be added to Test.xml reports"""
+        ...
+
+    @hookspec(firstresult=True)
+    def canary_cdash_subproject_label(self, case: "TestCase") -> str | None:
+        """Return a subproject label for ``case`` that will be added in Test.xml reports"""
+        ...
 
 
 @hookimpl
 def canary_session_reporter() -> CanaryReporter:
     return CDashReporter()
 
 
+@hookimpl
+def canary_addhooks(pluginmanager: "CanaryPluginManager"):
+    pluginmanager.add_hookspecs(CDashHooks)
+
+
 class CDashReporter(CanaryReporter):
     type = "cdash"
     description = "CDash reporter"

src/_canary/plugins/builtin/cdash/xml_generator.py

@@ -98,6 +98,16 @@ def create(
         else:
             self.buildstamp = self.validate_buildstamp(buildstamp)
         mkdirp(self.xml_dir)
+
+        unique_subproject_labels: set[str] = set(subproject_labels or [])
+        if label_sets := config.plugin_manager.hook.canary_cdash_labels_for_subproject():
+            unique_subproject_labels.update([_ for label_set in label_sets for _ in label_set])
+        for case in self.data.cases:
+            if label := config.plugin_manager.hook.canary_cdash_subproject_label(case=case):
+                unique_subproject_labels.add(label)
+        if unique_subproject_labels:
+            subproject_labels = list(unique_subproject_labels)
+
         if chunk_size is None:
             chunk_size = 500
         if chunk_size > 0:  # type: ignore
@@ -219,7 +229,7 @@ def write_test_xml(
         doc = self.create_document()
         root = doc.firstChild
 
-        if subproject_labels is not None:
+        if subproject_labels:
             for label in subproject_labels:
                 subproject = doc.createElement("Subproject")
                 subproject.setAttribute("name", label)
@@ -342,9 +352,12 @@ def write_test_xml(
                     filename=os.path.basename(case.file),
                 )
 
-            if case.keywords:
+            keywords: set[str] = set(case.keywords)
+            if label := config.plugin_manager.hook.canary_cdash_subproject_label(case=case):
+                keywords.add(label)
+            if keywords:
                 labels = doc.createElement("Labels")
-                for keyword in case.keywords:
+                for keyword in keywords:
                     add_text_node(labels, "Label", keyword)
                 test_node.appendChild(labels)
 

src/_canary/plugins/hookspec.py

@@ -17,6 +17,7 @@
     from ..session import Session
     from ..testbatch import TestBatch
     from ..testcase import TestCase
+    from .manager import CanaryPluginManager
 
 project_name = "canary"
 hookspec = pluggy.HookspecMarker(project_name)
@@ -255,6 +256,10 @@ def canary_testbatch_finish(batch: "TestBatch") -> None:
     The default implementation runs ``batch.finish()``
 
     Args:
-        The test case.
+        The test case batch.
 
     """
+
+
+@hookspec
+def canary_addhooks(pluginmanager: "CanaryPluginManager") -> None: ...

src/_canary/plugins/manager.py

@@ -6,7 +6,6 @@
 import sys
 import warnings
 from typing import TYPE_CHECKING
-from typing import Any
 
 import pluggy
 
@@ -36,11 +35,6 @@ def factory(cls) -> "CanaryPluginManager":
         self.load_setuptools_entrypoints(hookspec.project_name)
         return self
 
-    def update(self, arg: dict[str, Any]) -> None:
-        if plugins := arg.get("plugins"):
-            for plugin in plugins:
-                self.consider_plugin(plugin)
-
     def get_subcommands(self) -> list[CanarySubcommand]:
         hook = self.hook.canary_subcommand
         return hook()

tests/cmake/subproject_labels.py

@@ -0,0 +1,125 @@
+import os
+import subprocess
+import sys
+import xml.dom.minidom as xml
+
+from _canary.util.filesystem import mkdirp
+from _canary.util.filesystem import working_dir
+
+
+def test_cdash_labels_for_subproject(tmpdir):
+    """Test the plugin 'canary_cdash_labels_for_subproject'
+
+    The canary_cdash_labels_for_subproject allows adding subproject labels for the entire test
+    session
+    """
+    with working_dir(tmpdir.strpath, create=True):
+        setup_cdash_labels_for_subproject()
+        run_the_thing_and_check()
+
+
+def test_cdash_subproject_label(tmpdir):
+    """Test the plugin 'canary_cdash_subroject_label'
+
+    This plugin allows tests to define their CDash subproject label.  If the subproject label is
+    not included in the test case's keywords, it is added
+
+    """
+    with working_dir(tmpdir.strpath, create=True):
+        setup_cdash_subproject_label()
+        run_the_thing_and_check()
+
+
+def run_the_thing_and_check():
+    env = dict(os.environ)
+    env["PYTHONPATH"] = os.getcwd()
+    subprocess.run([f"{sys.prefix}/bin/canary", "-p", "baz", "run", "."], env=env)
+    subprocess.run(
+        [f"{sys.prefix}/bin/canary", "-C", "TestResults", "report", "cdash", "create"], env=env
+    )
+    file = "TestResults/CDASH/Test-0.xml"
+    doc = xml.parse(open(file))
+    names = get_subproject_labels(doc)
+    assert sorted(names) == ["baz", "foo"]
+    names = get_test_labels(doc)
+    assert sorted(names) == ["baz", "foo"]
+
+
+def setup_cdash_labels_for_subproject():
+    mkdirp("baz")
+    with open("baz/__init__.py", "w") as fh:
+        fh.write("""\
+import canary
+@canary.hookimpl
+def canary_cdash_labels_for_subproject():
+    return ['foo', 'baz']
+""")
+    with open("baz.pyt", "w") as fh:
+        fh.write("""\
+import sys
+import canary
+canary.directives.keywords('baz')
+def test():
+    return 0
+if __name__ == '__main__':
+    sys.exit(test())
+""")
+    with open("foo.pyt", "w") as fh:
+        fh.write("""\
+import sys
+import canary
+canary.directives.keywords('foo')
+def test():
+    return 0
+if __name__ == '__main__':
+    sys.exit(test())
+""")
+
+
+def setup_cdash_subproject_label():
+    """Test the plugin 'canary_cdash_subroject_label'
+
+    This plugin allows tests to define their CDash subproject label.  If the subproject label is
+    not included in the test case's keywords, it is added
+
+    """
+    mkdirp("baz")
+    with open("baz/__init__.py", "w") as fh:
+        fh.write("""\
+import canary
+@canary.hookimpl
+def canary_cdash_subproject_label(case):
+    return case.family
+""")
+    with open("baz.pyt", "w") as fh:
+        fh.write("""\
+import sys
+import canary
+def test():
+    return 0
+if __name__ == '__main__':
+    sys.exit(test())
+""")
+    with open("foo.pyt", "w") as fh:
+        fh.write("""\
+import sys
+import canary
+def test():
+    return 0
+if __name__ == '__main__':
+    sys.exit(test())
+""")
+
+
+def get_test_labels(doc):
+    names = []
+    for el1 in doc.getElementsByTagName("Testing"):
+        for el2 in el1.getElementsByTagName("Test"):
+            for el3 in el2.getElementsByTagName("Labels"):
+                for el4 in el3.getElementsByTagName("Label"):
+                    names.append(el4.childNodes[0].nodeValue)
+    return names
+
+
+def get_subproject_labels(doc):
+    return [el.getAttribute("name") for el in doc.getElementsByTagName("Subproject")]