Add support for generating recipes from R packages on GitHub

grayskull/main.py

@@ -28,7 +28,10 @@ def init_parser():
     # create parser for cran
     cran_parser = subparsers.add_parser("cran", help="Options to generate CRAN recipes")
     cran_parser.add_argument(
-        "cran_packages", nargs="+", help="Specify the CRAN packages name.", default=[]
+        "cran_packages",
+        nargs="+",
+        help="Specify the CRAN packages name. Grayskull can also accept a GitHub URL for R packages.",
+        default=[],
     )
     cran_parser.add_argument(
         "--stdout",
@@ -432,10 +435,15 @@ def create_python_recipe(pkg_name, sections_populate=None, **kwargs):
 
 
 def generate_r_recipes_from_list(list_pkgs, args):
-    cran_label = " (cran)"
     for pkg_name in list_pkgs:
         logging.debug(f"Starting grayskull for pkg: {pkg_name}")
         from_local_sdist = origin_is_local_sdist(pkg_name)
+        if origin_is_github(pkg_name):
+            cran_label = " (github)"
+        elif from_local_sdist:
+            cran_label = " (local)"
+        else:
+            cran_label = " (cran)"
         print_msg(
             f"{Fore.GREEN}\n\n"
             f"#### Initializing recipe for "

grayskull/strategy/cran.py

@@ -15,11 +15,12 @@
 from bs4 import BeautifulSoup
 from souschef.jinja_expression import set_global_jinja_var
 
+from grayskull.base.github import generate_git_archive_tarball_url, handle_gh_version
 from grayskull.cli.stdout import print_msg
 from grayskull.config import Configuration
 from grayskull.license.discovery import match_license
 from grayskull.strategy.abstract_strategy import AbstractStrategy
-from grayskull.utils import sha256_checksum
+from grayskull.utils import origin_is_github, sha256_checksum
 
 log = logging.getLogger(__name__)
 
@@ -53,6 +54,12 @@ def fetch_data(recipe, config, sections=None):
             metadata_section = metadata.get(sec)
             if metadata_section:
                 recipe[sec] = metadata_section
+
+        # Set Jinja2 global variables for the recipe
+        set_global_jinja_var(recipe, "name", config.name)
+        if config.version:
+            set_global_jinja_var(recipe, "version", config.version)
+
         if metadata.get("need_compiler", False):
             set_global_jinja_var(recipe, "posix", 'm2-" if win else "')
             set_global_jinja_var(recipe, "native", 'm2w64-" if win else "')
@@ -260,9 +267,23 @@ def get_cran_metadata(config: Configuration, cran_url: str):
     :return: CRAN metadata"""
     if config.name.startswith("r-"):
         config.name = config.name[2:]
+
+    # Check if this is a GitHub repository
+    # Use repo_github if it exists (set by Configuration when parsing GitHub URLs)
+    if (
+        hasattr(config, "repo_github")
+        and config.repo_github
+        and origin_is_github(config.repo_github)
+    ):
+        return get_github_r_metadata(config)
+
     pkg_name = config.name
     pkg_version = str(config.version) if config.version else None
     _, pkg_version, pkg_url = get_cran_index(cran_url, pkg_name, pkg_version)
+
+    # Set version as global jinja variable for consistent recipe generation
+    config.version = pkg_version
+
     print_msg(pkg_name)
     print_msg(pkg_version)
     download_file = download_cran_pkg(config, pkg_url)
@@ -274,6 +295,31 @@ def get_cran_metadata(config: Configuration, cran_url: str):
     print_msg(r_recipe_end_comment)
 
     imports = []
+    r_base_version = None
+
+    # Extract dependencies from both 'Depends' and 'Imports' fields
+    # Process 'Depends' first to extract R version requirements
+    for s in metadata.get("Depends", "").split(","):
+        if not s.strip():
+            continue
+        s = s.strip()
+        if s.startswith("R "):
+            # Extract R version constraint
+            r_parts = s.split("(")
+            if len(r_parts) > 1:
+                r_version_constraint = (
+                    r_parts[1].strip().replace(")", "").replace(" ", "")
+                )
+                r_base_version = r_version_constraint
+        else:
+            # Regular package dependency
+            r = s.split("(")
+            if len(r) == 1:
+                imports.append(f"r-{r[0].strip()}")
+            else:
+                constrain = r[1].strip().replace(")", "").replace(" ", "")
+                imports.append(f"r-{r[0].strip()} {constrain.strip()}")
+
     # Extract 'imports' from metadata.
     # Imports is equivalent to run and host dependencies.
     # Add 'r-' suffix to all packages listed in imports.
@@ -290,7 +336,11 @@ def get_cran_metadata(config: Configuration, cran_url: str):
     # Every CRAN package will always depend on the R base package.
     # Hence, the 'r-base' package is always present
     # in the host and run requirements.
-    imports.append("r-base")
+    # Add version constraint if found in Depends field
+    if r_base_version:
+        imports.append(f"r-base {r_base_version}")
+    else:
+        imports.append("r-base")
     imports.sort()  # this is not a requirement in conda but good for readability
 
     dict_metadata = {
@@ -348,6 +398,242 @@ def get_cran_metadata(config: Configuration, cran_url: str):
     return dict_metadata, r_recipe_end_comment
 
 
+def get_github_r_metadata(config: Configuration):
+    """Method responsible for getting R metadata from GitHub repositories.
+
+    :param config: Configuration object containing package information
+    :return: R metadata dictionary and recipe comment
+    """
+    print_msg("Fetching R package metadata from GitHub repository...")
+
+    # Extract GitHub URL and package name
+    # Use repo_github if available (set by Configuration.__post_init__), otherwise use config.name
+    github_url = getattr(config, "repo_github", None) or config.name
+    if github_url.endswith("/"):
+        github_url = github_url.rstrip("/")
+
+    # Extract package name from URL (last part) or use config.name if it's already parsed
+    if hasattr(config, "repo_github") and config.repo_github:
+        pkg_name = config.name  # Already parsed by Configuration
+    else:
+        pkg_name = github_url.split("/")[-1]
+
+    # Handle version and get the appropriate Git reference
+    version, version_tag = handle_gh_version(
+        name=pkg_name, version=config.version, url=github_url, tag=None
+    )
+
+    # Generate archive URL for the specific version/tag
+    archive_url = generate_git_archive_tarball_url(
+        git_url=github_url, git_ref=version_tag
+    )
+
+    print_msg(f"Package: {pkg_name}")
+    print_msg(f"Version: {version}")
+    print_msg(f"Archive URL: {archive_url}")
+
+    # Download and extract the GitHub archive
+    download_file = download_github_r_pkg(config, archive_url, pkg_name, version)
+
+    # Extract metadata from the DESCRIPTION file
+    metadata = get_github_archive_metadata(download_file)
+
+    r_recipe_end_comment = "\n".join(
+        [f"# {line}" for line in metadata["orig_lines"] if line]
+    )
+
+    print_msg(r_recipe_end_comment)
+
+    imports = []
+    r_base_version = None
+
+    # Extract dependencies from both 'Depends' and 'Imports' fields
+    # Process 'Depends' first to extract R version requirements
+    for s in metadata.get("Depends", "").split(","):
+        if not s.strip():
+            continue
+        s = s.strip()
+        if s.startswith("R "):
+            # Extract R version constraint
+            r_parts = s.split("(")
+            if len(r_parts) > 1:
+                r_version_constraint = (
+                    r_parts[1].strip().replace(")", "").replace(" ", "")
+                )
+                r_base_version = r_version_constraint
+        else:
+            # Regular package dependency
+            r = s.split("(")
+            if len(r) == 1:
+                imports.append(f"r-{r[0].strip()}")
+            else:
+                constrain = r[1].strip().replace(")", "").replace(" ", "")
+                imports.append(f"r-{r[0].strip()} {constrain.strip()}")
+
+    # Extract 'imports' from metadata.
+    # Imports is equivalent to run and host dependencies.
+    # Add 'r-' suffix to all packages listed in imports.
+    for s in metadata.get("Imports", "").split(","):
+        if not s.strip():
+            continue
+        r = s.split("(")
+        if len(r) == 1:
+            imports.append(f"r-{r[0].strip()}")
+        else:
+            constrain = r[1].strip().replace(")", "").replace(" ", "")
+            imports.append(f"r-{r[0].strip()} {constrain.strip()}")
+
+    # Every CRAN package will always depend on the R base package.
+    # Add version constraint if found in Depends field
+    if r_base_version:
+        imports.append(f"r-base {r_base_version}")
+    else:
+        imports.append("r-base")
+    imports.sort()  # this is not a requirement in conda but good for readability
+
+    # Create source URL with placeholders for templating
+    source_url = archive_url.replace(version_tag, "{{ version }}")
+    if version_tag.startswith("v") and not version.startswith("v"):
+        source_url = archive_url.replace(version_tag, "v{{ version }}")
+
+    dict_metadata = {
+        "package": {
+            "name": "r-{{ name }}",
+            "version": "{{ version }}",
+        },
+        "source": {
+            "url": source_url,
+            "sha256": sha256_checksum(download_file),
+        },
+        "build": {
+            "number": 0,
+            "merge_build_host": True,
+            "script": "R CMD INSTALL --build .",
+            "entry_points": metadata.get("entry_points"),
+            "rpaths": ["lib/R/lib/", "lib/"],
+        },
+        "requirements": {
+            "build": [],
+            "host": deepcopy(imports),
+            "run": deepcopy(imports),
+        },
+        "test": {
+            "imports": metadata.get("tests"),
+            "commands": [
+                f"$R -e \"library('{pkg_name}')\"  # [not win]",
+                f'"%R%" -e "library(\'{pkg_name}\')"  # [win]',
+            ],
+        },
+        "about": {
+            "home": github_url,
+            "summary": metadata.get("Description"),
+            "doc_url": metadata.get("URL"),
+            "dev_url": github_url,
+            "license": match_license(metadata.get("License", "")).get("licenseId")
+            or metadata.get("License", ""),
+        },
+    }
+
+    if metadata.get("NeedsCompilation", "no").lower() == "yes":
+        dict_metadata["need_compiler"] = True
+        dict_metadata["requirements"]["build"].extend(
+            [
+                "cross-r-base {{ r_base }}  # [build_platform != target_platform]",
+                "autoconf  # [unix]",
+                "{{ compiler('c') }}  # [unix]",
+                "{{ compiler('m2w64_c') }}  # [win]",
+                "{{ compiler('cxx') }}  # [unix]",
+                "{{ compiler('m2w64_cxx') }}  # [win]",
+                "posix  # [win]",
+            ]
+        )
+    if not dict_metadata["requirements"]["build"]:
+        del dict_metadata["requirements"]["build"]
+
+    # Set the package name and version in config for recipe generation
+    config.name = pkg_name
+    config.version = version
+
+    return dict_metadata, r_recipe_end_comment
+
+
+def download_github_r_pkg(
+    config: Configuration, archive_url: str, pkg_name: str, version: str
+):
+    """Download R package archive from GitHub.
+
+    :param config: Configuration object
+    :param archive_url: GitHub archive URL
+    :param pkg_name: Package name
+    :param version: Package version
+    :return: Path to downloaded file
+    """
+    tarball_name = f"{pkg_name}-{version}.tar.gz"
+    print_msg(f"Downloading from: {archive_url}")
+
+    response = requests.get(archive_url, timeout=30)
+    response.raise_for_status()
+
+    download_file = os.path.join(
+        str(mkdtemp(f"grayskull-github-r-{pkg_name}-")), tarball_name
+    )
+
+    with open(download_file, "wb") as f:
+        f.write(response.content)
+
+    return download_file
+
+
+def get_github_archive_metadata(archive_path: str):
+    """Extract metadata from GitHub R package archive.
+
+    :param archive_path: Path to the downloaded archive
+    :return: Metadata dictionary
+    """
+    print_msg("Extracting metadata from GitHub R package...")
+
+    # Create temporary directory for extraction
+    temp_dir = mkdtemp(prefix="grayskull-github-r-extract-")
+
+    try:
+        # Extract the archive
+        with tarfile.open(archive_path, "r:gz") as tar:
+            tar.extractall(temp_dir, filter="data")
+
+        # Find the DESCRIPTION file
+        # GitHub archives typically have a top-level directory
+        extracted_dirs = [
+            d for d in os.listdir(temp_dir) if os.path.isdir(os.path.join(temp_dir, d))
+        ]
+
+        if not extracted_dirs:
+            raise ValueError("No directories found in the archive")
+
+        # Use the first directory (there should be only one)
+        package_dir = os.path.join(temp_dir, extracted_dirs[0])
+        description_path = os.path.join(package_dir, "DESCRIPTION")
+
+        if not os.path.exists(description_path):
+            raise ValueError("DESCRIPTION file not found in the R package")
+
+        # Read and parse the DESCRIPTION file
+        with open(description_path, encoding="utf-8") as f:
+            description_content = f.read()
+
+        # Parse the DESCRIPTION file content
+        lines = description_content.strip().split("\n")
+        lines = remove_package_line_continuations(lines)
+        metadata = dict_from_cran_lines(lines)
+
+        return metadata
+
+    finally:
+        # Clean up temporary directory
+        import shutil
+
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+
 def download_cran_pkg(config, pkg_url):
     tarball_name = pkg_url.rsplit("/", 1)[-1]
     print_msg(pkg_url)

grayskull/utils.py

@@ -230,7 +230,7 @@ def generate_recipe(
         recipe_dir = Path(folder_path) / pkg_name
         logging.debug(f"Generating recipe on: {recipe_dir}")
         if not recipe_dir.is_dir():
-            recipe_dir.mkdir()
+            recipe_dir.mkdir(parents=True, exist_ok=True)
         recipe_path = (
             recipe_dir / "recipe.yaml" if use_v1_format else recipe_dir / "meta.yaml"
         )