diff --git a/grayskull/main.py b/grayskull/main.py index 7bd31a49..e8c6bce9 100644 --- a/grayskull/main.py +++ b/grayskull/main.py @@ -28,7 +28,10 @@ def init_parser(): # create parser for cran cran_parser = subparsers.add_parser("cran", help="Options to generate CRAN recipes") cran_parser.add_argument( - "cran_packages", nargs="+", help="Specify the CRAN packages name.", default=[] + "cran_packages", + nargs="+", + help="Specify the CRAN packages name. Grayskull can also accept a GitHub URL for R packages.", + default=[], ) cran_parser.add_argument( "--stdout", @@ -432,10 +435,15 @@ def create_python_recipe(pkg_name, sections_populate=None, **kwargs): def generate_r_recipes_from_list(list_pkgs, args): - cran_label = " (cran)" for pkg_name in list_pkgs: logging.debug(f"Starting grayskull for pkg: {pkg_name}") from_local_sdist = origin_is_local_sdist(pkg_name) + if origin_is_github(pkg_name): + cran_label = " (github)" + elif from_local_sdist: + cran_label = " (local)" + else: + cran_label = " (cran)" print_msg( f"{Fore.GREEN}\n\n" f"#### Initializing recipe for " diff --git a/grayskull/strategy/cran.py b/grayskull/strategy/cran.py index 5c1254d2..1f90d860 100644 --- a/grayskull/strategy/cran.py +++ b/grayskull/strategy/cran.py @@ -15,11 +15,12 @@ from bs4 import BeautifulSoup from souschef.jinja_expression import set_global_jinja_var +from grayskull.base.github import generate_git_archive_tarball_url, handle_gh_version from grayskull.cli.stdout import print_msg from grayskull.config import Configuration from grayskull.license.discovery import match_license from grayskull.strategy.abstract_strategy import AbstractStrategy -from grayskull.utils import sha256_checksum +from grayskull.utils import origin_is_github, sha256_checksum log = logging.getLogger(__name__) @@ -53,6 +54,12 @@ def fetch_data(recipe, config, sections=None): metadata_section = metadata.get(sec) if metadata_section: recipe[sec] = metadata_section + + # Set Jinja2 global variables for the recipe + set_global_jinja_var(recipe, "name", config.name) + if config.version: + set_global_jinja_var(recipe, "version", config.version) + if metadata.get("need_compiler", False): set_global_jinja_var(recipe, "posix", 'm2-" if win else "') set_global_jinja_var(recipe, "native", 'm2w64-" if win else "') @@ -260,9 +267,23 @@ def get_cran_metadata(config: Configuration, cran_url: str): :return: CRAN metadata""" if config.name.startswith("r-"): config.name = config.name[2:] + + # Check if this is a GitHub repository + # Use repo_github if it exists (set by Configuration when parsing GitHub URLs) + if ( + hasattr(config, "repo_github") + and config.repo_github + and origin_is_github(config.repo_github) + ): + return get_github_r_metadata(config) + pkg_name = config.name pkg_version = str(config.version) if config.version else None _, pkg_version, pkg_url = get_cran_index(cran_url, pkg_name, pkg_version) + + # Set version as global jinja variable for consistent recipe generation + config.version = pkg_version + print_msg(pkg_name) print_msg(pkg_version) download_file = download_cran_pkg(config, pkg_url) @@ -274,6 +295,31 @@ def get_cran_metadata(config: Configuration, cran_url: str): print_msg(r_recipe_end_comment) imports = [] + r_base_version = None + + # Extract dependencies from both 'Depends' and 'Imports' fields + # Process 'Depends' first to extract R version requirements + for s in metadata.get("Depends", "").split(","): + if not s.strip(): + continue + s = s.strip() + if s.startswith("R "): + # Extract R version constraint + r_parts = s.split("(") + if len(r_parts) > 1: + r_version_constraint = ( + r_parts[1].strip().replace(")", "").replace(" ", "") + ) + r_base_version = r_version_constraint + else: + # Regular package dependency + r = s.split("(") + if len(r) == 1: + imports.append(f"r-{r[0].strip()}") + else: + constrain = r[1].strip().replace(")", "").replace(" ", "") + imports.append(f"r-{r[0].strip()} {constrain.strip()}") + # Extract 'imports' from metadata. # Imports is equivalent to run and host dependencies. # Add 'r-' suffix to all packages listed in imports. @@ -290,7 +336,11 @@ def get_cran_metadata(config: Configuration, cran_url: str): # Every CRAN package will always depend on the R base package. # Hence, the 'r-base' package is always present # in the host and run requirements. - imports.append("r-base") + # Add version constraint if found in Depends field + if r_base_version: + imports.append(f"r-base {r_base_version}") + else: + imports.append("r-base") imports.sort() # this is not a requirement in conda but good for readability dict_metadata = { @@ -348,6 +398,242 @@ def get_cran_metadata(config: Configuration, cran_url: str): return dict_metadata, r_recipe_end_comment +def get_github_r_metadata(config: Configuration): + """Method responsible for getting R metadata from GitHub repositories. + + :param config: Configuration object containing package information + :return: R metadata dictionary and recipe comment + """ + print_msg("Fetching R package metadata from GitHub repository...") + + # Extract GitHub URL and package name + # Use repo_github if available (set by Configuration.__post_init__), otherwise use config.name + github_url = getattr(config, "repo_github", None) or config.name + if github_url.endswith("/"): + github_url = github_url.rstrip("/") + + # Extract package name from URL (last part) or use config.name if it's already parsed + if hasattr(config, "repo_github") and config.repo_github: + pkg_name = config.name # Already parsed by Configuration + else: + pkg_name = github_url.split("/")[-1] + + # Handle version and get the appropriate Git reference + version, version_tag = handle_gh_version( + name=pkg_name, version=config.version, url=github_url, tag=None + ) + + # Generate archive URL for the specific version/tag + archive_url = generate_git_archive_tarball_url( + git_url=github_url, git_ref=version_tag + ) + + print_msg(f"Package: {pkg_name}") + print_msg(f"Version: {version}") + print_msg(f"Archive URL: {archive_url}") + + # Download and extract the GitHub archive + download_file = download_github_r_pkg(config, archive_url, pkg_name, version) + + # Extract metadata from the DESCRIPTION file + metadata = get_github_archive_metadata(download_file) + + r_recipe_end_comment = "\n".join( + [f"# {line}" for line in metadata["orig_lines"] if line] + ) + + print_msg(r_recipe_end_comment) + + imports = [] + r_base_version = None + + # Extract dependencies from both 'Depends' and 'Imports' fields + # Process 'Depends' first to extract R version requirements + for s in metadata.get("Depends", "").split(","): + if not s.strip(): + continue + s = s.strip() + if s.startswith("R "): + # Extract R version constraint + r_parts = s.split("(") + if len(r_parts) > 1: + r_version_constraint = ( + r_parts[1].strip().replace(")", "").replace(" ", "") + ) + r_base_version = r_version_constraint + else: + # Regular package dependency + r = s.split("(") + if len(r) == 1: + imports.append(f"r-{r[0].strip()}") + else: + constrain = r[1].strip().replace(")", "").replace(" ", "") + imports.append(f"r-{r[0].strip()} {constrain.strip()}") + + # Extract 'imports' from metadata. + # Imports is equivalent to run and host dependencies. + # Add 'r-' suffix to all packages listed in imports. + for s in metadata.get("Imports", "").split(","): + if not s.strip(): + continue + r = s.split("(") + if len(r) == 1: + imports.append(f"r-{r[0].strip()}") + else: + constrain = r[1].strip().replace(")", "").replace(" ", "") + imports.append(f"r-{r[0].strip()} {constrain.strip()}") + + # Every CRAN package will always depend on the R base package. + # Add version constraint if found in Depends field + if r_base_version: + imports.append(f"r-base {r_base_version}") + else: + imports.append("r-base") + imports.sort() # this is not a requirement in conda but good for readability + + # Create source URL with placeholders for templating + source_url = archive_url.replace(version_tag, "{{ version }}") + if version_tag.startswith("v") and not version.startswith("v"): + source_url = archive_url.replace(version_tag, "v{{ version }}") + + dict_metadata = { + "package": { + "name": "r-{{ name }}", + "version": "{{ version }}", + }, + "source": { + "url": source_url, + "sha256": sha256_checksum(download_file), + }, + "build": { + "number": 0, + "merge_build_host": True, + "script": "R CMD INSTALL --build .", + "entry_points": metadata.get("entry_points"), + "rpaths": ["lib/R/lib/", "lib/"], + }, + "requirements": { + "build": [], + "host": deepcopy(imports), + "run": deepcopy(imports), + }, + "test": { + "imports": metadata.get("tests"), + "commands": [ + f"$R -e \"library('{pkg_name}')\" # [not win]", + f'"%R%" -e "library(\'{pkg_name}\')" # [win]', + ], + }, + "about": { + "home": github_url, + "summary": metadata.get("Description"), + "doc_url": metadata.get("URL"), + "dev_url": github_url, + "license": match_license(metadata.get("License", "")).get("licenseId") + or metadata.get("License", ""), + }, + } + + if metadata.get("NeedsCompilation", "no").lower() == "yes": + dict_metadata["need_compiler"] = True + dict_metadata["requirements"]["build"].extend( + [ + "cross-r-base {{ r_base }} # [build_platform != target_platform]", + "autoconf # [unix]", + "{{ compiler('c') }} # [unix]", + "{{ compiler('m2w64_c') }} # [win]", + "{{ compiler('cxx') }} # [unix]", + "{{ compiler('m2w64_cxx') }} # [win]", + "posix # [win]", + ] + ) + if not dict_metadata["requirements"]["build"]: + del dict_metadata["requirements"]["build"] + + # Set the package name and version in config for recipe generation + config.name = pkg_name + config.version = version + + return dict_metadata, r_recipe_end_comment + + +def download_github_r_pkg( + config: Configuration, archive_url: str, pkg_name: str, version: str +): + """Download R package archive from GitHub. + + :param config: Configuration object + :param archive_url: GitHub archive URL + :param pkg_name: Package name + :param version: Package version + :return: Path to downloaded file + """ + tarball_name = f"{pkg_name}-{version}.tar.gz" + print_msg(f"Downloading from: {archive_url}") + + response = requests.get(archive_url, timeout=30) + response.raise_for_status() + + download_file = os.path.join( + str(mkdtemp(f"grayskull-github-r-{pkg_name}-")), tarball_name + ) + + with open(download_file, "wb") as f: + f.write(response.content) + + return download_file + + +def get_github_archive_metadata(archive_path: str): + """Extract metadata from GitHub R package archive. + + :param archive_path: Path to the downloaded archive + :return: Metadata dictionary + """ + print_msg("Extracting metadata from GitHub R package...") + + # Create temporary directory for extraction + temp_dir = mkdtemp(prefix="grayskull-github-r-extract-") + + try: + # Extract the archive + with tarfile.open(archive_path, "r:gz") as tar: + tar.extractall(temp_dir, filter="data") + + # Find the DESCRIPTION file + # GitHub archives typically have a top-level directory + extracted_dirs = [ + d for d in os.listdir(temp_dir) if os.path.isdir(os.path.join(temp_dir, d)) + ] + + if not extracted_dirs: + raise ValueError("No directories found in the archive") + + # Use the first directory (there should be only one) + package_dir = os.path.join(temp_dir, extracted_dirs[0]) + description_path = os.path.join(package_dir, "DESCRIPTION") + + if not os.path.exists(description_path): + raise ValueError("DESCRIPTION file not found in the R package") + + # Read and parse the DESCRIPTION file + with open(description_path, encoding="utf-8") as f: + description_content = f.read() + + # Parse the DESCRIPTION file content + lines = description_content.strip().split("\n") + lines = remove_package_line_continuations(lines) + metadata = dict_from_cran_lines(lines) + + return metadata + + finally: + # Clean up temporary directory + import shutil + + shutil.rmtree(temp_dir, ignore_errors=True) + + def download_cran_pkg(config, pkg_url): tarball_name = pkg_url.rsplit("/", 1)[-1] print_msg(pkg_url) diff --git a/grayskull/utils.py b/grayskull/utils.py index dcd7f5d1..15e5091f 100644 --- a/grayskull/utils.py +++ b/grayskull/utils.py @@ -230,7 +230,7 @@ def generate_recipe( recipe_dir = Path(folder_path) / pkg_name logging.debug(f"Generating recipe on: {recipe_dir}") if not recipe_dir.is_dir(): - recipe_dir.mkdir() + recipe_dir.mkdir(parents=True, exist_ok=True) recipe_path = ( recipe_dir / "recipe.yaml" if use_v1_format else recipe_dir / "meta.yaml" ) diff --git a/tests/test_cran.py b/tests/test_cran.py index 1060a154..302479f5 100644 --- a/tests/test_cran.py +++ b/tests/test_cran.py @@ -5,6 +5,7 @@ from grayskull.config import Configuration from grayskull.strategy.cran import ( get_cran_metadata, + get_github_r_metadata, scrap_cran_archive_page_for_package_folder_url, scrap_cran_pkg_folder_page_for_full_url, scrap_main_page_cran_find_latest_package, @@ -103,3 +104,107 @@ def test_get_cran_metadata_need_compilation( "{{ compiler('m2w64_cxx') }} # [win]", "posix # [win]", ] + + +@patch("grayskull.strategy.cran.origin_is_github") +@patch("grayskull.strategy.cran.get_github_r_metadata") +def test_get_cran_metadata_github_url_detection(mock_github_metadata, mock_is_github): + """Test that get_cran_metadata detects GitHub URLs and uses GitHub strategy""" + # Setup + mock_is_github.return_value = True + mock_github_metadata.return_value = ({"package": "test"}, "# comment") + + config = Configuration(name="https://github.com/user/repo") + + # Call + result, comment = get_cran_metadata(config, "https://cran.r-project.org") + + # Verify GitHub path was taken + mock_is_github.assert_called_once_with("https://github.com/user/repo") + mock_github_metadata.assert_called_once_with(config) + + assert result == {"package": "test"} + assert comment == "# comment" + + +@patch("grayskull.strategy.cran.origin_is_github") +@patch("grayskull.strategy.cran.get_cran_index") +@patch("grayskull.strategy.cran.download_cran_pkg") +@patch("grayskull.strategy.cran.get_archive_metadata") +@patch("grayskull.strategy.cran.sha256_checksum") +def test_get_cran_metadata_regular_cran_package( + mock_sha256, mock_get_archive, mock_download, mock_get_index, mock_is_github +): + """Test that get_cran_metadata handles regular CRAN packages correctly""" + # Setup + mock_is_github.return_value = False + mock_get_index.return_value = ( + "testpkg", + "1.0.0", + "http://cran.../testpkg_1.0.0.tar.gz", + ) + mock_download.return_value = "/tmp/testpkg_1.0.0.tar.gz" + mock_sha256.return_value = "fake_sha256_hash" + mock_get_archive.return_value = { + "Package": "testpkg", + "Version": "1.0.0", + "orig_lines": ["Package: testpkg"], + "URL": "http://example.com", + } + + config = Configuration(name="testpkg") + + # Call + result, comment = get_cran_metadata(config, "https://cran.r-project.org") + + # Verify CRAN path was taken + # Note: origin_is_github is not called for regular CRAN packages anymore + # as the logic now checks for config.repo_github attribute first + mock_get_index.assert_called_once() + assert "package" in result + assert result["package"]["name"] == "r-{{ name }}" + + +@patch("grayskull.strategy.cran.handle_gh_version") +@patch("grayskull.strategy.cran.generate_git_archive_tarball_url") +@patch("grayskull.strategy.cran.download_github_r_pkg") +@patch("grayskull.strategy.cran.get_github_archive_metadata") +@patch("grayskull.strategy.cran.sha256_checksum") +def test_get_github_r_metadata_basic_flow( + mock_sha256, mock_get_metadata, mock_download, mock_gen_url, mock_handle_version +): + """Test basic flow of get_github_r_metadata""" + # Setup + config = Configuration(name="https://github.com/user/testpkg", version="1.0.0") + + mock_handle_version.return_value = ("1.0.0", "v1.0.0") + mock_gen_url.return_value = "https://github.com/user/testpkg/archive/v1.0.0.tar.gz" + mock_download.return_value = "/tmp/testpkg-1.0.0.tar.gz" + mock_sha256.return_value = "fake_sha256_hash" + mock_get_metadata.return_value = { + "Package": "testpkg", + "Version": "1.0.0", + "Description": "Test package", + "License": "MIT", + "Imports": "dplyr", + "NeedsCompilation": "no", + "orig_lines": ["Package: testpkg", "Version: 1.0.0"], + } + + # Call + result, comment = get_github_r_metadata(config) + + # Verify structure + assert "package" in result + assert "source" in result + assert "requirements" in result + assert "about" in result + + # Verify GitHub-specific fields + assert result["about"]["home"] == "https://github.com/user/testpkg" + assert result["about"]["dev_url"] == "https://github.com/user/testpkg" + assert "v{{ version }}" in result["source"]["url"] + + # Verify dependencies + assert "r-dplyr" in result["requirements"]["host"] + assert "r-base" in result["requirements"]["host"] diff --git a/tests/test_github_r_integration.py b/tests/test_github_r_integration.py new file mode 100644 index 00000000..fb650258 --- /dev/null +++ b/tests/test_github_r_integration.py @@ -0,0 +1,130 @@ +""" +Integration tests for GitHub R packages. +These tests require internet access and use real GitHub repositories. +""" + +import pytest +from souschef.jinja_expression import get_global_jinja_var + +from grayskull.main import create_r_recipe + + +@pytest.mark.github +def test_github_r_package_integration(): + """Test creating a recipe from a real GitHub R package""" + # Use a small, stable R package for testing + # Using the 'praise' package which is simple and stable + github_url = "https://github.com/rladies/praise" + + # Create recipe + recipe, config = create_r_recipe(github_url, version="1.0.0") + + # Basic structure checks + assert "package" in recipe + assert "source" in recipe + assert "build" in recipe + assert "requirements" in recipe + assert "test" in recipe + assert "about" in recipe + + # Package info + assert recipe["package"]["name"] == "r-{{ name }}" + assert recipe["package"]["version"] == "{{ version }}" + assert get_global_jinja_var(recipe, "name") == "praise" + assert get_global_jinja_var(recipe, "version") == "1.0.0" + + # Source info - should use GitHub archive with version placeholder + assert "github.com" in recipe["source"]["url"] + assert ( + "{{ version }}" in recipe["source"]["url"] + or "v{{ version }}" in recipe["source"]["url"] + ) + assert "sha256" in recipe["source"] + + # Requirements - should have r-base at minimum + assert "r-base" in recipe["requirements"]["host"] + assert "r-base" in recipe["requirements"]["run"] + + # About section - should have GitHub info + assert recipe["about"]["home"] == "https://github.com/rladies/praise" + assert recipe["about"]["dev_url"] == "https://github.com/rladies/praise" + + # Test commands should be present + assert any("library('praise')" in cmd for cmd in recipe["test"]["commands"]) + + +@pytest.mark.github +def test_github_r_package_with_dependencies(): + """Test GitHub R package with dependencies""" + # Using a package that has some dependencies + github_url = "https://github.com/hadley/stringr" + + # Create recipe without specifying version (should get latest) + recipe, config = create_r_recipe(github_url) + + # Should have dependencies + host_deps = recipe["requirements"]["host"] + run_deps = recipe["requirements"]["run"] + + # Should have r-base + assert "r-base" in host_deps + assert "r-base" in run_deps + + # Should have additional R dependencies + assert len(host_deps) > 1 # More than just r-base + assert len(run_deps) > 1 # More than just r-base + + # All dependencies should be prefixed with 'r-' + for dep in host_deps: + assert dep.startswith("r-") or dep.startswith("cross-r-base") + for dep in run_deps: + assert dep.startswith("r-") + + +@pytest.mark.github +def test_github_r_package_version_placeholder(): + """Test that GitHub R package URLs get proper version placeholders""" + github_url = "https://github.com/rladies/praise" + + recipe, config = create_r_recipe(github_url, version="1.0.0") + + # The source URL should have version placeholder + source_url = recipe["source"]["url"] + assert "{{ version }}" in source_url or "v{{ version }}" in source_url + + # Should not contain the actual version string + assert "1.0.0" not in source_url + + +@pytest.mark.github +def test_github_r_package_needs_compilation(): + """Test GitHub R package that needs compilation""" + # This would test a package that has NeedsCompilation: yes + # For now, we'll skip this as it requires finding a suitable test package + pytest.skip("Need to identify a suitable R package that requires compilation") + + +@pytest.mark.github +def test_github_r_package_url_variations(): + """Test different GitHub URL formats""" + test_urls = [ + "https://github.com/rladies/praise", + "https://github.com/rladies/praise/", # with trailing slash + "http://github.com/rladies/praise", # http instead of https + ] + + for url in test_urls: + recipe, config = create_r_recipe(url, version="1.0.0") + + # Should all produce valid recipes + assert "package" in recipe + assert recipe["package"]["name"] == "r-{{ name }}" + assert get_global_jinja_var(recipe, "name") == "praise" + + # About section should normalize to https + assert recipe["about"]["home"] == "https://github.com/rladies/praise" + assert recipe["about"]["dev_url"] == "https://github.com/rladies/praise" + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-m", "github"]) diff --git a/tests/test_github_r_packages.py b/tests/test_github_r_packages.py new file mode 100644 index 00000000..81401d6e --- /dev/null +++ b/tests/test_github_r_packages.py @@ -0,0 +1,341 @@ +import os +import tarfile +import tempfile +from unittest.mock import Mock, patch + +import pytest + +from grayskull.config import Configuration +from grayskull.strategy.cran import ( + download_github_r_pkg, + get_cran_metadata, + get_github_archive_metadata, + get_github_r_metadata, +) + + +@pytest.fixture +def mock_github_r_description(): + """Mock DESCRIPTION file content for a GitHub R package""" + return """Package: testpkg +Version: 1.0.0 +Title: Test Package +Description: This is a test R package from GitHub +Authors@R: person("Test", "Author", email = "test@example.com", role = c("aut", "cre")) +License: MIT + file LICENSE +Encoding: UTF-8 +Imports: + dplyr (>= 1.0.0), + ggplot2 +Suggests: + testthat +URL: https://github.com/testuser/testpkg +BugReports: https://github.com/testuser/testpkg/issues +NeedsCompilation: no +""" + + +@pytest.fixture +def mock_config(): + """Mock configuration for testing""" + return Configuration(name="https://github.com/testuser/testpkg", version="1.0.0") + + +@pytest.fixture +def mock_archive_file(mock_github_r_description): + """Create a mock tarball with DESCRIPTION file""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create package directory structure + pkg_dir = os.path.join(temp_dir, "testpkg-1.0.0") + os.makedirs(pkg_dir) + + # Create DESCRIPTION file + with open(os.path.join(pkg_dir, "DESCRIPTION"), "w") as f: + f.write(mock_github_r_description) + + # Create tarball + tarball_path = os.path.join(temp_dir, "testpkg-1.0.0.tar.gz") + with tarfile.open(tarball_path, "w:gz") as tar: + tar.add(pkg_dir, arcname="testpkg-1.0.0") + + yield tarball_path + + +class TestGitHubRPackages: + """Test suite for GitHub R package functionality""" + + @patch("grayskull.strategy.cran.sha256_checksum") + @patch("grayskull.strategy.cran.handle_gh_version") + @patch("grayskull.strategy.cran.generate_git_archive_tarball_url") + @patch("grayskull.strategy.cran.download_github_r_pkg") + @patch("grayskull.strategy.cran.get_github_archive_metadata") + def test_get_github_r_metadata_basic( + self, + mock_get_metadata, + mock_download, + mock_gen_url, + mock_handle_version, + mock_sha256, + mock_config, + ): + """Test basic GitHub R metadata extraction""" + + # Setup mocks + mock_handle_version.return_value = ("1.0.0", "v1.0.0") + mock_gen_url.return_value = ( + "https://github.com/testuser/testpkg/archive/v1.0.0.tar.gz" + ) + mock_download.return_value = "/tmp/testpkg-1.0.0.tar.gz" + mock_sha256.return_value = "abcd1234567890" + mock_get_metadata.return_value = { + "Package": "testpkg", + "Version": "1.0.0", + "Description": "Test package", + "License": "MIT + file LICENSE", + "Imports": "dplyr (>= 1.0.0), ggplot2", + "NeedsCompilation": "no", + "orig_lines": ["Package: testpkg", "Version: 1.0.0"], + } + + # Test the function + metadata, comment = get_github_r_metadata(mock_config) + + # Verify the structure + assert "package" in metadata + assert "source" in metadata + assert "build" in metadata + assert "requirements" in metadata + assert "test" in metadata + assert "about" in metadata + + # Verify package information + assert metadata["package"]["name"] == "r-{{ name }}" + assert metadata["package"]["version"] == "{{ version }}" + + # Verify source information + assert "github.com" in metadata["source"]["url"] + assert "{{ version }}" in metadata["source"]["url"] + + # Verify requirements + host_requirements = metadata["requirements"]["host"] + assert any("r-dplyr" in req for req in host_requirements) + assert "r-ggplot2" in host_requirements + assert "r-base" in host_requirements + + # Verify about section + assert metadata["about"]["home"] == "https://github.com/testuser/testpkg" + assert metadata["about"]["dev_url"] == "https://github.com/testuser/testpkg" + + @patch("requests.get") + def test_download_github_r_pkg(self, mock_get, mock_config): + """Test downloading GitHub R package archive""" + + # Mock the response + mock_response = Mock() + mock_response.content = b"fake tarball content" + mock_response.raise_for_status = Mock() + mock_get.return_value = mock_response + + # Test download + result = download_github_r_pkg( + mock_config, + "https://github.com/testuser/testpkg/archive/v1.0.0.tar.gz", + "testpkg", + "1.0.0", + ) + + # Verify the file was created + assert os.path.exists(result) + assert result.endswith("testpkg-1.0.0.tar.gz") + + # Clean up + os.unlink(result) + + def test_get_github_archive_metadata(self, mock_archive_file): + """Test extracting metadata from GitHub archive""" + + metadata = get_github_archive_metadata(mock_archive_file) + + # Verify extracted metadata + assert metadata["Package"] == "testpkg" + assert metadata["Version"] == "1.0.0" + assert metadata["License"] == "MIT + file LICENSE" + assert "dplyr" in metadata["Imports"] + assert "ggplot2" in metadata["Imports"] + assert metadata["NeedsCompilation"] == "no" + + @patch("grayskull.strategy.cran.origin_is_github") + @patch("grayskull.strategy.cran.get_github_r_metadata") + @patch("grayskull.strategy.cran.get_cran_index") + def test_get_cran_metadata_github_detection( + self, mock_cran_index, mock_github_metadata, mock_is_github, mock_config + ): + """Test that get_cran_metadata properly detects and handles GitHub URLs""" + + # Setup mocks + mock_is_github.return_value = True + mock_github_metadata.return_value = ({}, "# comment") + + # Call with GitHub URL + config = Configuration(name="https://github.com/testuser/testpkg") + get_cran_metadata(config, "https://cran.r-project.org") + + # Verify GitHub path was taken + mock_github_metadata.assert_called_once() + mock_cran_index.assert_not_called() + + @patch("grayskull.strategy.cran.origin_is_github") + @patch("grayskull.strategy.cran.get_github_r_metadata") + @patch("grayskull.strategy.cran.get_cran_index") + def test_get_cran_metadata_cran_path( + self, mock_cran_index, mock_github_metadata, mock_is_github + ): + """Test that get_cran_metadata properly handles CRAN packages""" + + # Setup mocks + mock_is_github.return_value = False + mock_cran_index.return_value = ( + "testpkg", + "1.0.0", + "http://cran.r-project.org/...", + ) + + # Mock other required functions + with ( + patch("grayskull.strategy.cran.download_cran_pkg") as mock_download, + patch("grayskull.strategy.cran.get_archive_metadata") as mock_metadata, + patch("grayskull.strategy.cran.sha256_checksum") as mock_sha256, + ): + mock_download.return_value = "/tmp/file.tar.gz" + mock_sha256.return_value = "abcd1234567890" + mock_metadata.return_value = { + "Package": "testpkg", + "Version": "1.0.0", + "orig_lines": [], + "URL": "http://example.com", + } + + # Call with regular package name + config = Configuration(name="testpkg") + get_cran_metadata(config, "https://cran.r-project.org") + + # Verify CRAN path was taken + mock_cran_index.assert_called_once() + mock_github_metadata.assert_not_called() + + @pytest.mark.github + def test_github_url_version_placeholder_r_package(self): + """Test that GitHub R package URLs get proper version placeholders""" + + # This would be an integration test that requires actual GitHub access + # For now, we'll skip it but it's here for future implementation + pytest.skip("Integration test - requires actual GitHub access") + + def test_version_tag_handling(self): + """Test proper handling of version tags (v1.0.0 vs 1.0.0)""" + + config = Configuration( + name="https://github.com/testuser/testpkg", version="1.0.0" + ) + + with ( + patch("grayskull.strategy.cran.handle_gh_version") as mock_handle, + patch( + "grayskull.strategy.cran.generate_git_archive_tarball_url" + ) as mock_gen_url, + patch("grayskull.strategy.cran.download_github_r_pkg") as mock_download, + patch( + "grayskull.strategy.cran.get_github_archive_metadata" + ) as mock_metadata, + patch("grayskull.strategy.cran.sha256_checksum") as mock_sha256, + ): + # Test with v-prefixed tag + mock_handle.return_value = ("1.0.0", "v1.0.0") + mock_gen_url.return_value = ( + "https://github.com/testuser/testpkg/archive/v1.0.0.tar.gz" + ) + mock_download.return_value = "/tmp/test.tar.gz" + mock_sha256.return_value = "abcd1234567890" + mock_metadata.return_value = {"Package": "testpkg", "orig_lines": []} + + metadata, _ = get_github_r_metadata(config) + + # Should use v{{ version }} in URL for v-prefixed tags + assert "v{{ version }}" in metadata["source"]["url"] + + def test_needs_compilation_handling(self): + """Test that NeedsCompilation: yes adds proper build requirements""" + + config = Configuration(name="https://github.com/testuser/testpkg") + + with ( + patch("grayskull.strategy.cran.handle_gh_version") as mock_handle, + patch( + "grayskull.strategy.cran.generate_git_archive_tarball_url" + ) as mock_gen_url, + patch("grayskull.strategy.cran.download_github_r_pkg") as mock_download, + patch( + "grayskull.strategy.cran.get_github_archive_metadata" + ) as mock_metadata, + patch("grayskull.strategy.cran.sha256_checksum") as mock_sha256, + ): + mock_handle.return_value = ("1.0.0", "1.0.0") + mock_gen_url.return_value = ( + "https://github.com/testuser/testpkg/archive/1.0.0.tar.gz" + ) + mock_download.return_value = "/tmp/test.tar.gz" + mock_sha256.return_value = "abcd1234567890" + mock_metadata.return_value = { + "Package": "testpkg", + "NeedsCompilation": "yes", + "orig_lines": [], + } + + metadata, _ = get_github_r_metadata(config) + + # Should have compilation requirements + assert metadata.get("need_compiler") is True + build_requirements = metadata["requirements"]["build"] + assert any("{{ compiler('c') }}" in req for req in build_requirements) + assert any("autoconf" in req for req in build_requirements) + + def test_imports_parsing(self): + """Test proper parsing of Imports field with version constraints""" + + config = Configuration(name="https://github.com/testuser/testpkg") + + with ( + patch("grayskull.strategy.cran.handle_gh_version") as mock_handle, + patch( + "grayskull.strategy.cran.generate_git_archive_tarball_url" + ) as mock_gen_url, + patch("grayskull.strategy.cran.download_github_r_pkg") as mock_download, + patch( + "grayskull.strategy.cran.get_github_archive_metadata" + ) as mock_metadata, + patch("grayskull.strategy.cran.sha256_checksum") as mock_sha256, + ): + mock_handle.return_value = ("1.0.0", "1.0.0") + mock_gen_url.return_value = ( + "https://github.com/testuser/testpkg/archive/1.0.0.tar.gz" + ) + mock_download.return_value = "/tmp/test.tar.gz" + mock_sha256.return_value = "abcd1234567890" + mock_metadata.return_value = { + "Package": "testpkg", + "Imports": "dplyr (>= 1.0.0), ggplot2, stringr (>= 1.4.0)", + "orig_lines": [], + } + + metadata, _ = get_github_r_metadata(config) + + # Should parse imports with version constraints + requirements = metadata["requirements"]["host"] + assert any("r-dplyr" in req and ">=1.0.0" in req for req in requirements) + assert "r-ggplot2" in requirements + assert any("r-stringr" in req and ">=1.4.0" in req for req in requirements) + assert "r-base" in requirements + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 00000000..77858174 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +Integration test for GitHub R package functionality. +This script tests the complete workflow of generating a recipe for an R package from GitHub. +""" + +import sys + +# Add the grayskull module to path +sys.path.insert(0, "/data02/work/guozhonghao/grayskull") + +from grayskull.main import create_r_recipe + + +def test_github_r_package_integration(): + """Test creating a recipe for a real GitHub R package""" + + # Use a simple, small R package from GitHub for testing + # This package should have a proper DESCRIPTION file + github_url = "https://github.com/tidyverse/stringr" + + print(f"Testing recipe generation for: {github_url}") + + try: + # Create recipe + recipe, config = create_r_recipe(github_url) + + # Verify basic structure + assert "package" in recipe + assert "source" in recipe + assert "build" in recipe + assert "requirements" in recipe + assert "test" in recipe + assert "about" in recipe + + # Verify package information + assert recipe["package"]["name"] == "r-{{ name }}" + assert recipe["package"]["version"] == "{{ version }}" + + # Verify source points to GitHub + assert "github.com" in recipe["source"]["url"] + assert "archive" in recipe["source"]["url"] + assert "{{ version }}" in recipe["source"]["url"] + + # Verify requirements include r-base + assert "r-base" in recipe["requirements"]["host"] + assert "r-base" in recipe["requirements"]["run"] + + # Verify about section + assert "github.com" in recipe["about"]["home"] + assert recipe["about"]["dev_url"] == recipe["about"]["home"] + + print("āœ… Recipe structure looks good!") + print(f"Package name: {config.name}") + print(f"Package version: {config.version}") + print(f"Source URL: {recipe['source']['url']}") + print(f"Home URL: {recipe['about']['home']}") + + return True + + except Exception as e: + print(f"āŒ Test failed with error: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_github_r_package_detection(): + """Test that GitHub R packages are properly detected""" + from grayskull.config import Configuration + from grayskull.utils import origin_is_github + + # Test URL detection + github_url = "https://github.com/tidyverse/stringr" + assert origin_is_github(github_url), "GitHub URL should be detected" + + # Test configuration parsing + config = Configuration(name=github_url) + assert hasattr(config, "repo_github"), "repo_github should be set" + assert config.repo_github == github_url, "repo_github should match input URL" + assert config.name == "stringr", "name should be extracted from URL" + + print("āœ… GitHub R package detection works correctly!") + return True + + +if __name__ == "__main__": + print("Running GitHub R package integration tests...") + print("=" * 50) + + # Test 1: Basic detection + print("Test 1: GitHub package detection") + if not test_github_r_package_detection(): + sys.exit(1) + + print("\n" + "=" * 50) + + # Test 2: Full integration (requires network access) + print("Test 2: Full recipe generation (requires network)") + response = input("Run integration test with real GitHub package? (y/N): ") + + if response.lower() in ["y", "yes"]: + if not test_github_r_package_integration(): + sys.exit(1) + else: + print("Skipping integration test") + + print("\nāœ… All tests completed successfully!")