From cce91f3d6b6ae6992adaa1fd20fa35859109bd4b Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 9 Oct 2023 12:10:17 +0200 Subject: [PATCH 01/13] fixing python_requires in buildInfo --- extensions/commands/art/cmd_build_info.py | 30 +++++++++++++++- tests/test_artifactory_commands.py | 42 ++++++++++++++++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index 5205729b..c8f8e5e4 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -145,7 +145,7 @@ def _get_local_artifacts(): file_list = list(dl_folder.glob("*")) if len(file_list) >= 3: for file_path in dl_folder.glob("*"): - if file_path.is_file(): + if file_path.is_file(): # FIXME: Make it recursive for metadata folder file_name = file_path.name md5, sha1, sha256 = _get_hashes(file_path) artifact_info = {"type": os.path.splitext(file_name)[1].lstrip('.'), @@ -226,6 +226,33 @@ def _get_remote_artifacts(): return artifacts + def _get_python_requires(self, node, modules): + python_requires = node.get("python_requires") + if python_requires is None: + return + for pyref, pyreq in python_requires.items(): + pyrecipe = pyreq["recipe"] + artifacts_folder = pyreq["path"] + remote_path = _get_remote_path(pyref) + artifacts = [] + + dl_folder = Path(artifacts_folder).parents[0] / "d" + file_list = list(dl_folder.glob("*")) + for f in file_list: + if not f.is_file(): + continue # FIXME: This is discarding metadata folders + md5, sha1, sha256 = _get_hashes(f) + artifact_info = {"type": os.path.splitext(f.name)[1].lstrip('.'), + "sha256": sha256, + "sha1": sha1, + "md5": md5} + artifact_info.update({"name": f.name, "path": f'{self._repository}/{remote_path}/{f.name}'}) + artifacts.append(artifact_info) + pyreq_module = {"type": "conan", + "id": pyref, + "artifacts": artifacts} + modules.append(pyreq_module) + def get_modules(self): ret = [] try: @@ -237,6 +264,7 @@ def get_modules(self): ref = node.get("ref") if ref: transitive_dependencies = node.get("dependencies").keys() if node.get("dependencies").keys() else [] + self._get_python_requires(node, modules=ret) # only add the nodes that were marked as built if node.get("binary") == "Build": diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index 785f495e..298ce74d 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -1,7 +1,9 @@ +import json import os import tempfile +import textwrap -from tools import run +from tools import run, save import pytest @@ -198,6 +200,44 @@ def test_build_info_create_deps(): run('conan remove "*" -c -r extensions-stg') +@pytest.mark.requires_credentials +def test_build_info_create_python_requires(): + build_name = "mybuildinfo" + build_number = "1" + + run(f'conan art:server add artifactory {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + + # Create dependency packages and upload them + pytool = textwrap.dedent("""\ + from conan import ConanFile + class PyTool(ConanFile): + name = "pytool" + version = "0.1" + """) + save("conanfile.py", pytool) + run('conan create . ') + pkg = textwrap.dedent("""\ + from conan import ConanFile + class PyTool(ConanFile): + name = "pkg" + version = "0.1" + python_requires = "pytool/0.1" + """) + save("conanfile.py", pkg) + run('conan create . --format=json > create_release.json') + + run("conan upload * -c --dry-run -r=extensions-stg") + run(f'conan art:build-info create create_release.json {build_name}_release {build_number} extensions-stg --server artifactory --with-dependencies > {build_name}_release.json') + build_info = open("mybuildinfo_release.json").read() + + build_info = json.loads(build_info) + assert build_info["modules"][0]["id"] == "pytool/0.1#623dc6c0466e112d42f2b629d8abf49a" + artifacts = build_info["modules"][0]["artifacts"] + assert len(artifacts) == 2 + assert artifacts[0]["name"] == "conanfile.py" + assert artifacts[1]["name"] == "conanmanifest.txt" + + @pytest.mark.requires_credentials def test_fail_if_not_uploaded(): """ From 07a70c3454c0bb3b52f8ef13654bb6370a8b5e28 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Tue, 10 Oct 2023 16:25:13 +0200 Subject: [PATCH 02/13] Update tests/test_artifactory_commands.py --- tests/test_artifactory_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index 298ce74d..f23bf178 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -226,7 +226,7 @@ class PyTool(ConanFile): save("conanfile.py", pkg) run('conan create . --format=json > create_release.json') - run("conan upload * -c --dry-run -r=extensions-stg") + run("conan upload '*' -c --dry-run -r=extensions-stg") run(f'conan art:build-info create create_release.json {build_name}_release {build_number} extensions-stg --server artifactory --with-dependencies > {build_name}_release.json') build_info = open("mybuildinfo_release.json").read() From 0be00ae3ad3c0cdcc8f63a1230bdd7fd492f1b96 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 10 Oct 2023 17:55:07 +0200 Subject: [PATCH 03/13] consider conan versions --- extensions/commands/art/cmd_build_info.py | 7 +++++-- tests/test_artifactory_commands.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index c8f8e5e4..6a186787 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -231,8 +231,10 @@ def _get_python_requires(self, node, modules): if python_requires is None: return for pyref, pyreq in python_requires.items(): - pyrecipe = pyreq["recipe"] - artifacts_folder = pyreq["path"] + artifacts_folder = pyreq.get("path") + # this may happen for conan versions < 2.0.14, do not crash in that case + if artifacts_folder is None: + continue remote_path = _get_remote_path(pyref) artifacts = [] @@ -253,6 +255,7 @@ def _get_python_requires(self, node, modules): "artifacts": artifacts} modules.append(pyreq_module) + def get_modules(self): ret = [] try: diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index f23bf178..f2da90db 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -4,6 +4,7 @@ import textwrap from tools import run, save +from conan.tools.scm import Version import pytest @@ -201,6 +202,7 @@ def test_build_info_create_deps(): @pytest.mark.requires_credentials +@pytest.mark.skipif(conan_version <= Version("2.0.13"), reason="path key is only added to python requires in graph for conan >= 2.0.14") def test_build_info_create_python_requires(): build_name = "mybuildinfo" build_number = "1" From 87faa9a8140c59540f9b8ab904a9a1a5590214e7 Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 10 Oct 2023 17:57:05 +0200 Subject: [PATCH 04/13] fix missing import --- tests/test_artifactory_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index f2da90db..28c367ce 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -5,6 +5,7 @@ from tools import run, save from conan.tools.scm import Version +from conan import conan_version import pytest From dc44f9bed0ac9a42794382b30f9f48b349d9a1cc Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 10 Oct 2023 18:11:10 +0200 Subject: [PATCH 05/13] relax test --- tests/test_artifactory_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index 28c367ce..94767a4e 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -234,7 +234,7 @@ class PyTool(ConanFile): build_info = open("mybuildinfo_release.json").read() build_info = json.loads(build_info) - assert build_info["modules"][0]["id"] == "pytool/0.1#623dc6c0466e112d42f2b629d8abf49a" + assert "pytool/0.1#" in build_info["modules"][0]["id"] == artifacts = build_info["modules"][0]["artifacts"] assert len(artifacts) == 2 assert artifacts[0]["name"] == "conanfile.py" From 04d4686d9b05c40566c79c77345c9de8058e6c6e Mon Sep 17 00:00:00 2001 From: czoido Date: Tue, 10 Oct 2023 18:30:22 +0200 Subject: [PATCH 06/13] minor changes --- tests/test_artifactory_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index 94767a4e..6d6e81a3 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -234,7 +234,7 @@ class PyTool(ConanFile): build_info = open("mybuildinfo_release.json").read() build_info = json.loads(build_info) - assert "pytool/0.1#" in build_info["modules"][0]["id"] == + assert "pytool/0.1#" in build_info["modules"][0]["id"] artifacts = build_info["modules"][0]["artifacts"] assert len(artifacts) == 2 assert artifacts[0]["name"] == "conanfile.py" From 4f68eea1c16f5683c189a62b6c9deb25fa3eee60 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Oct 2023 08:07:19 +0200 Subject: [PATCH 07/13] make test order independent --- tests/test_artifactory_commands.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index 6d6e81a3..adc098e3 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -237,8 +237,11 @@ class PyTool(ConanFile): assert "pytool/0.1#" in build_info["modules"][0]["id"] artifacts = build_info["modules"][0]["artifacts"] assert len(artifacts) == 2 - assert artifacts[0]["name"] == "conanfile.py" - assert artifacts[1]["name"] == "conanmanifest.txt" + + artifact_names = [artifact["name"] for artifact in artifacts] + + assert "conanfile.py" in artifact_names + assert "conanmanifest.txt" in artifact_names @pytest.mark.requires_credentials From 5084ee3d7578776bb69ad83e16b05fbd398258bb Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Oct 2023 13:59:23 +0200 Subject: [PATCH 08/13] minor refactor --- extensions/commands/art/cmd_build_info.py | 76 +++++++++++------------ 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index 6a186787..72deb849 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -256,60 +256,56 @@ def _get_python_requires(self, node, modules): modules.append(pyreq_module) + def create_module(self, node, artifact_type, transitive_dependencies): + if artifact_type=="recipe" or (node.get("package_id") and node.get("prev")): + + ref = node.get("ref") + + module = { + "type": "conan", + "id": str(ref) if artifact_type=="recipe" else f'{str(ref)}:{node.get("package_id")}#{node.get("prev")}', + "artifacts": self.get_artifacts(node, artifact_type) + } + + if self._with_dependencies: + nodes = self._graph["graph"]["nodes"] + all_dependencies = [] + for require_id in transitive_dependencies: + deps_artifacts = self.get_artifacts(nodes.get(require_id), artifact_type, + is_dependency=True) + all_dependencies.extend(deps_artifacts) + + module.update({"dependencies": all_dependencies}) + return module + + def get_modules(self): - ret = [] + modules_list = [] try: nodes = self._graph["graph"]["nodes"] except KeyError: raise ConanException("JSON does not contain graph information") - for id, node in nodes.items(): - ref = node.get("ref") - if ref: - transitive_dependencies = node.get("dependencies").keys() if node.get("dependencies").keys() else [] - self._get_python_requires(node, modules=ret) + for _, node in nodes.items(): + if node.get("ref"): # only add the nodes that were marked as built if node.get("binary") == "Build": - # recipe module - module = { - "type": "conan", - "id": str(ref), - "artifacts": self.get_artifacts(node, "recipe") - } - - if self._with_dependencies: - all_dependencies = [] - for require_id in transitive_dependencies: - deps_artifacts = self.get_artifacts(nodes.get(require_id), "recipe", - is_dependency=True) - all_dependencies.extend(deps_artifacts) + transitive_dependencies = node.get("dependencies").keys() if node.get("dependencies").keys() else [] + self._get_python_requires(node, modules=modules_list) - module.update({"dependencies": all_dependencies}) + # For each package that was build we create a recipe module and a package module - ret.append(module) + # recipe module + recipe_module = self.create_module(node, "recipe", transitive_dependencies) + modules_list.append(recipe_module) # package module - if node.get("package_id") and node.get("prev"): - module = { - "type": "conan", - "id": f'{str(ref)}:{node.get("package_id")}#{node.get("prev")}', - "artifacts": self.get_artifacts(node, "package") - } - # get the dependencies and its artifacts - if self._with_dependencies: - all_dependencies = [] - for require_id in transitive_dependencies: - deps_artifacts = self.get_artifacts(nodes.get(require_id), "package", - is_dependency=True) - all_dependencies.extend(deps_artifacts) - - module.update({"dependencies": all_dependencies}) - - ret.append(module) - - return ret + package_module = self.create_module(node, "package", transitive_dependencies) + modules_list.append(package_module) + + return modules_list def header(self): return {"version": "1.0.1", From 675e896f88dd62d85b61aea1fe7eef8c263226fe Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Oct 2023 14:10:14 +0200 Subject: [PATCH 09/13] minor changes --- extensions/commands/art/cmd_build_info.py | 40 ++++++++++------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index 72deb849..6896ee5a 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -256,22 +256,22 @@ def _get_python_requires(self, node, modules): modules.append(pyreq_module) - def create_module(self, node, artifact_type, transitive_dependencies): - if artifact_type=="recipe" or (node.get("package_id") and node.get("prev")): + def create_module(self, node, module_type, transitive_dependencies): + if module_type=="recipe" or (node.get("package_id") and node.get("prev")): ref = node.get("ref") module = { "type": "conan", - "id": str(ref) if artifact_type=="recipe" else f'{str(ref)}:{node.get("package_id")}#{node.get("prev")}', - "artifacts": self.get_artifacts(node, artifact_type) + "id": str(ref) if module_type=="recipe" else f'{str(ref)}:{node.get("package_id")}#{node.get("prev")}', + "artifacts": self.get_artifacts(node, module_type) } if self._with_dependencies: nodes = self._graph["graph"]["nodes"] all_dependencies = [] for require_id in transitive_dependencies: - deps_artifacts = self.get_artifacts(nodes.get(require_id), artifact_type, + deps_artifacts = self.get_artifacts(nodes.get(require_id), module_type, is_dependency=True) all_dependencies.extend(deps_artifacts) @@ -283,27 +283,23 @@ def get_modules(self): modules_list = [] try: nodes = self._graph["graph"]["nodes"] - except KeyError: - raise ConanException("JSON does not contain graph information") + except KeyError as e: + raise ConanException(f"JSON does not contain graph information: {e}") - for _, node in nodes.items(): - if node.get("ref"): - - # only add the nodes that were marked as built - if node.get("binary") == "Build": - - transitive_dependencies = node.get("dependencies").keys() if node.get("dependencies").keys() else [] - self._get_python_requires(node, modules=modules_list) + for node in nodes.values(): + ref = node.get("ref") + binary = node.get("binary") + dependencies = node.get("dependencies", {}) - # For each package that was build we create a recipe module and a package module + if ref and binary == "Build": + transitive_dependencies = list(dependencies.keys()) - # recipe module - recipe_module = self.create_module(node, "recipe", transitive_dependencies) - modules_list.append(recipe_module) + self._get_python_requires(node, modules=modules_list) - # package module - package_module = self.create_module(node, "package", transitive_dependencies) - modules_list.append(package_module) + # For each package that was built we create a recipe module and a package module + for module_type in ["recipe", "package"]: + module = self.create_module(node, module_type, transitive_dependencies) + modules_list.append(module) return modules_list From 02dcae7628babcd0032781184062d5977bfd4d28 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Oct 2023 15:13:25 +0200 Subject: [PATCH 10/13] wip --- extensions/commands/art/cmd_build_info.py | 82 ++++++++++++----------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index 6896ee5a..49a8abd4 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -226,38 +226,9 @@ def _get_remote_artifacts(): return artifacts - def _get_python_requires(self, node, modules): - python_requires = node.get("python_requires") - if python_requires is None: - return - for pyref, pyreq in python_requires.items(): - artifacts_folder = pyreq.get("path") - # this may happen for conan versions < 2.0.14, do not crash in that case - if artifacts_folder is None: - continue - remote_path = _get_remote_path(pyref) - artifacts = [] - dl_folder = Path(artifacts_folder).parents[0] / "d" - file_list = list(dl_folder.glob("*")) - for f in file_list: - if not f.is_file(): - continue # FIXME: This is discarding metadata folders - md5, sha1, sha256 = _get_hashes(f) - artifact_info = {"type": os.path.splitext(f.name)[1].lstrip('.'), - "sha256": sha256, - "sha1": sha1, - "md5": md5} - artifact_info.update({"name": f.name, "path": f'{self._repository}/{remote_path}/{f.name}'}) - artifacts.append(artifact_info) - pyreq_module = {"type": "conan", - "id": pyref, - "artifacts": artifacts} - modules.append(pyreq_module) - - - def create_module(self, node, module_type, transitive_dependencies): - if module_type=="recipe" or (node.get("package_id") and node.get("prev")): + def create_module(self, node, module_type, transitive_dependencies=None, python_requires=None): + if module_type=="recipe" or (node.get("package_id") and node.get("prev") and module_type=="package"): ref = node.get("ref") @@ -267,7 +238,7 @@ def create_module(self, node, module_type, transitive_dependencies): "artifacts": self.get_artifacts(node, module_type) } - if self._with_dependencies: + if transitive_dependencies: nodes = self._graph["graph"]["nodes"] all_dependencies = [] for require_id in transitive_dependencies: @@ -277,7 +248,33 @@ def create_module(self, node, module_type, transitive_dependencies): module.update({"dependencies": all_dependencies}) return module - + elif (module_type=="python_requires"): + # FIXME: this deserves some refactoring + python_requires = node.get("python_requires") + for pyref, pyreq in python_requires.items(): + artifacts_folder = pyreq.get("path") + # this may happen for conan versions < 2.0.14, do not crash in that case + if artifacts_folder is None: + continue + remote_path = _get_remote_path(pyref) + artifacts = [] + + dl_folder = Path(artifacts_folder).parents[0] / "d" + file_list = list(dl_folder.glob("*")) + for f in file_list: + if not f.is_file(): + continue # FIXME: This is discarding metadata folders + md5, sha1, sha256 = _get_hashes(f) + artifact_info = {"type": os.path.splitext(f.name)[1].lstrip('.'), + "sha256": sha256, + "sha1": sha1, + "md5": md5} + artifact_info.update({"name": f.name, "path": f'{self._repository}/{remote_path}/{f.name}'}) + artifacts.append(artifact_info) + pyreq_module = {"type": "conan", + "id": pyref, + "artifacts": artifacts} + return pyreq_module def get_modules(self): modules_list = [] @@ -292,14 +289,23 @@ def get_modules(self): dependencies = node.get("dependencies", {}) if ref and binary == "Build": - transitive_dependencies = list(dependencies.keys()) + transitive_dependencies = list(dependencies.keys()) if self._with_dependencies else None - self._get_python_requires(node, modules=modules_list) + # If the package that was built had a python_requires then add it as a separate module + + python_requires = node.get("python_requires") + if python_requires: + module = self.create_module(node, "python_requires") + modules_list.append(module) # For each package that was built we create a recipe module and a package module - for module_type in ["recipe", "package"]: - module = self.create_module(node, module_type, transitive_dependencies) - modules_list.append(module) + recipe_module = self.create_module(node, "recipe", transitive_dependencies, python_requires) + modules_list.append(recipe_module) + + + package_module = self.create_module(node, "package", transitive_dependencies) + modules_list.append(package_module) + return modules_list From b57189493dee9ea13f5ce41209e1c4f52aacd3e3 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Oct 2023 15:53:30 +0200 Subject: [PATCH 11/13] add pyreqs as deps --- extensions/commands/art/cmd_build_info.py | 93 ++++++++++++++--------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index 49a8abd4..a7d5fade 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -226,55 +226,74 @@ def _get_remote_artifacts(): return artifacts + def get_pyreq_artifacts(self, pyreq, pyref, is_dependency=False): + artifacts_folder = pyreq.get("path") + # this may happen for conan versions < 2.0.14, do not crash in that case + if artifacts_folder is None: + return + remote_path = _get_remote_path(pyref) + artifacts = [] + + dl_folder = Path(artifacts_folder).parents[0] / "d" + file_list = list(dl_folder.glob("*")) + for f in file_list: + if not f.is_file(): + continue # FIXME: This is discarding metadata folders + md5, sha1, sha256 = _get_hashes(f) + + artifact_info = {"type": os.path.splitext(f.name)[1].lstrip('.'), + "sha256": sha256, + "sha1": sha1, + "md5": md5} + + if not is_dependency: + artifact_info.update({"name": f.name, "path": f'{self._repository}/{remote_path}/{f.name}'}) + else: + artifact_info.update({"id": f"{pyref} :: {f.name}"}) + + artifacts.append(artifact_info) + return artifacts + + def create_pyreq_modules(self, node): + python_requires = node.get("python_requires") + pyreq_modules = [] + for pyref, pyreq in python_requires.items(): + + artifacts = self.get_pyreq_artifacts(pyreq, pyref) + + module = {"type": "conan", + "id": pyref, + "artifacts": artifacts} + pyreq_modules.append(module) + return pyreq_modules def create_module(self, node, module_type, transitive_dependencies=None, python_requires=None): if module_type=="recipe" or (node.get("package_id") and node.get("prev") and module_type=="package"): - ref = node.get("ref") - module = { "type": "conan", "id": str(ref) if module_type=="recipe" else f'{str(ref)}:{node.get("package_id")}#{node.get("prev")}', "artifacts": self.get_artifacts(node, module_type) } + if transitive_dependencies or python_requires: + nodes = self._graph["graph"]["nodes"] + all_dependencies = [] + if transitive_dependencies: - nodes = self._graph["graph"]["nodes"] - all_dependencies = [] for require_id in transitive_dependencies: - deps_artifacts = self.get_artifacts(nodes.get(require_id), module_type, - is_dependency=True) + deps_artifacts = self.get_artifacts(nodes.get(require_id), module_type, is_dependency=True) all_dependencies.extend(deps_artifacts) - module.update({"dependencies": all_dependencies}) - return module - elif (module_type=="python_requires"): - # FIXME: this deserves some refactoring - python_requires = node.get("python_requires") - for pyref, pyreq in python_requires.items(): - artifacts_folder = pyreq.get("path") - # this may happen for conan versions < 2.0.14, do not crash in that case - if artifacts_folder is None: - continue - remote_path = _get_remote_path(pyref) - artifacts = [] - - dl_folder = Path(artifacts_folder).parents[0] / "d" - file_list = list(dl_folder.glob("*")) - for f in file_list: - if not f.is_file(): - continue # FIXME: This is discarding metadata folders - md5, sha1, sha256 = _get_hashes(f) - artifact_info = {"type": os.path.splitext(f.name)[1].lstrip('.'), - "sha256": sha256, - "sha1": sha1, - "md5": md5} - artifact_info.update({"name": f.name, "path": f'{self._repository}/{remote_path}/{f.name}'}) - artifacts.append(artifact_info) - pyreq_module = {"type": "conan", - "id": pyref, - "artifacts": artifacts} - return pyreq_module + if python_requires: + for pyref, pyreq in python_requires.items(): + pyreq_artifacts = self.get_pyreq_artifacts(pyreq, pyref, is_dependency=True) + all_dependencies.extend(pyreq_artifacts) + + module.update({"dependencies": all_dependencies}) + + return module + def get_modules(self): modules_list = [] @@ -295,8 +314,8 @@ def get_modules(self): python_requires = node.get("python_requires") if python_requires: - module = self.create_module(node, "python_requires") - modules_list.append(module) + modules = self.create_pyreq_modules(node) + modules_list.extend(modules) # For each package that was built we create a recipe module and a package module recipe_module = self.create_module(node, "recipe", transitive_dependencies, python_requires) From 3b41dc2f9f03ad74c413cac2205b1f71db0f605e Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Oct 2023 16:58:04 +0200 Subject: [PATCH 12/13] add add_cached_deps --- extensions/commands/art/cmd_build_info.py | 12 ++++++-- tests/test_artifactory_commands.py | 37 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/extensions/commands/art/cmd_build_info.py b/extensions/commands/art/cmd_build_info.py index a7d5fade..606f982d 100644 --- a/extensions/commands/art/cmd_build_info.py +++ b/extensions/commands/art/cmd_build_info.py @@ -110,7 +110,7 @@ def _get_requested_by(nodes, node_id, artifact_type): class _BuildInfo: def __init__(self, graph, name, number, repository, with_dependencies=False, - url=None, user=None, password=None): + add_cached_deps=False, url=None, user=None, password=None): self._graph = graph self._name = name self._number = number @@ -120,6 +120,7 @@ def __init__(self, graph, name, number, repository, with_dependencies=False, self._password = password self._cached_artifact_info = {} self._with_dependencies = with_dependencies + self._add_cached_deps = add_cached_deps def get_artifacts(self, node, artifact_type, is_dependency=False): """ @@ -307,7 +308,7 @@ def get_modules(self): binary = node.get("binary") dependencies = node.get("dependencies", {}) - if ref and binary == "Build": + if ref and (binary == "Build" or (binary == "Cache" and self._add_cached_deps)): transitive_dependencies = list(dependencies.keys()) if self._with_dependencies else None # If the package that was built had a python_requires then add it as a separate module @@ -408,6 +409,10 @@ def build_info_create(conan_api: ConanAPI, parser, subparser, *args): subparser.add_argument("--with-dependencies", help="Whether to add dependencies information or not. Default: false.", action='store_true', default=False) + subparser.add_argument("--add-cached-deps", help="It will add not only the Conan packages that are built " + "but also the ones that are used from the cache but not built. Default: false.", + action='store_true', default=False) + args = parser.parse_args(*args) url, user, password = get_url_user_password(args) @@ -418,7 +423,8 @@ def build_info_create(conan_api: ConanAPI, parser, subparser, *args): # remove the 'conanfile' node data["graph"]["nodes"].pop("0") bi = _BuildInfo(data, args.build_name, args.build_number, args.repository, - with_dependencies=args.with_dependencies, url=url, user=user, password=password) + with_dependencies=args.with_dependencies, + add_cached_deps=args.add_cached_deps, url=url, user=user, password=password) cli_out_write(bi.create()) diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index adc098e3..375616a4 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -202,6 +202,43 @@ def test_build_info_create_deps(): run('conan remove "*" -c -r extensions-stg') +@pytest.mark.requires_credentials +def test_build_info_create_from_cached_deps(): + # Make sure artifactory repos are empty before starting the test + run("conan remove mypkg* -c -r extensions-stg") + run("conan remove mypkg* -c -r extensions-prod") + + build_name = "mybuildinfo" + build_number = "1" + + run(f'conan art:server add artifactory {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') + + # Create dependency packages and upload them + run("conan new cmake_lib -d name=libc -d version=1.0 --force") + run("conan create . -tf=''") + run("conan new cmake_lib -d name=liba -d version=1.0 -d requires=libc/1.0 --force") + run("conan create . -tf=''") + + run("conan upload '*' --dry-run -c -r extensions-stg") + + # libc node in graph is cached + run("conan install . --format json > install_release.json") + + run(f'conan art:build-info create install_release.json bi_release 1 extensions-stg --server artifactory --with-dependencies > bi_release.json') + + with open("bi_release.json", "r") as file: + build_info = json.load(file) + + assert len(build_info.get("modules")) == 0 + + run(f'conan art:build-info create install_release.json bi_release 1 extensions-stg --server artifactory --with-dependencies --add-cached-deps > bi_release.json') + + with open("bi_release.json", "r") as file: + build_info = json.load(file) + + assert len(build_info.get("modules")) == 2 + + @pytest.mark.requires_credentials @pytest.mark.skipif(conan_version <= Version("2.0.13"), reason="path key is only added to python requires in graph for conan >= 2.0.14") def test_build_info_create_python_requires(): From 6808199a1757e72b4e6a2be9660be91a750d6783 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Oct 2023 17:02:58 +0200 Subject: [PATCH 13/13] unused --- tests/test_artifactory_commands.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_artifactory_commands.py b/tests/test_artifactory_commands.py index 375616a4..e9052271 100644 --- a/tests/test_artifactory_commands.py +++ b/tests/test_artifactory_commands.py @@ -208,9 +208,6 @@ def test_build_info_create_from_cached_deps(): run("conan remove mypkg* -c -r extensions-stg") run("conan remove mypkg* -c -r extensions-prod") - build_name = "mybuildinfo" - build_number = "1" - run(f'conan art:server add artifactory {os.getenv("ART_URL")} --user="{os.getenv("CONAN_LOGIN_USERNAME_EXTENSIONS_STG")}" --password="{os.getenv("CONAN_PASSWORD_EXTENSIONS_STG")}"') # Create dependency packages and upload them