Add an inject command (#115)

Author: kurtmckee

changelog.d/20250924_215958_kurtmckee_support_injection.rst

@@ -0,0 +1,9 @@
+Added
+-----
+
+*   Add an ``inject`` command.
+
+    This allows a database of dependencies to be injected into a target file.
+    This can be used to make a Python file a standalone executable.
+
+    This feature requires Python 3.11+ due to Python 3.11+ sqlite3 API usage.

pyproject.toml

@@ -83,7 +83,7 @@ source = [
 
 [tool.coverage.report]
 skip_covered = true
-fail_under = 60
+fail_under = 54
 
 
 # flake8

src/sqliteimport/cli.py

@@ -10,6 +10,7 @@
 
 from . import bundler
 from . import compiler
+from . import injector
 from .accessor import Accessor
 from .util import get_magic_number
 
@@ -156,3 +157,106 @@ def describe(database: pathlib.Path) -> None:
             print(textwrap.indent(str(table), "    "))
         else:
             print("No installed packages were found.")
+
+
+DEFAULT_MARKER = "sqliteimport-inject-here"
+
+
+@group.command(name="inject", no_args_is_help=True)
+@click.option(
+    "--database",
+    type=click.Path(
+        exists=True, dir_okay=False, file_okay=True, path_type=pathlib.Path
+    ),
+    help="The database of packages to inject.",
+)
+@click.option(
+    "--target-file",
+    type=click.Path(
+        exists=True, dir_okay=False, file_okay=True, path_type=pathlib.Path
+    ),
+    help=(
+        """
+        The Python code file to inject sqliteimport
+        and the `--database` of packages into.
+
+        The target file WILL NOT be overwritten by default;
+        it can only be overwritten if it is specified again as the `--output-file`.
+        """
+    ),
+)
+@click.option(
+    "--marker",
+    default=DEFAULT_MARKER,
+    help=(
+        f"""
+        The marker to search for in the `--target-file`.
+
+        The marker must exist in the `--target-file`,
+        and must be a standalone Python comment.
+        For example, by default the marker is "{DEFAULT_MARKER}",
+        so the `--target-file` should contain code like this somewhere:
+
+        \b
+            # {DEFAULT_MARKER}
+        \b
+        """
+    ),
+)
+@click.option(
+    "--output-file",
+    type=click.Path(dir_okay=False, file_okay=True, path_type=pathlib.Path),
+    help=(
+        """
+        The output file to write, containing the code of the `--target-file`
+        combined with the sqliteimport source code and database of dependencies.
+        """
+    ),
+)
+@click.option(
+    "--overwrite",
+    is_flag=True,
+    help=(
+        """
+        If set, the `--output-file` will be overwritten if it exists.
+
+        By default, the `--output-file` will never be overwritten.
+        """
+    ),
+)
+def inject(
+    database: pathlib.Path,
+    target_file: pathlib.Path,
+    output_file: pathlib.Path,
+    marker: str,
+    overwrite: bool,
+) -> None:
+    """Inject sqliteimport and a database of dependencies into a target code file.
+
+    After injection, the code will have the ability to load its own dependencies.
+    This is accomplished by encoding and adding the sqliteimport source code
+    -- along with the database of packages -- directly into the code file.
+
+    The `--marker` option can be used to configure where the code is injected.
+    For example, using the default marker, the target code file might look like this:
+
+    \b
+        def main() -> None:
+            # sqliteimport-inject-here
+    \b
+            import requests
+            requests.get(...)
+    \b
+        if __name__ == "__main__":
+            main()
+
+    Note: this relies on APIs that were first introduced in Python 3.11.
+    Injection will fail on older version of Python.
+    """
+
+    if output_file.exists() and not overwrite:
+        click.echo("The output file already exists.")
+        sys.exit(1)
+    prologue = injector.generate_prologue(database)
+    code = target_file.read_text()
+    injector.inject_prologue(prologue, code, marker)

src/sqliteimport/injector-template.py

@@ -0,0 +1,102 @@
+# This file is a part of sqliteimport <https://github.com/kurtmckee/sqliteimport>
+# Copyright 2024-2025 Kurt McKee <[email protected]>
+# SPDX-License-Identifier: MIT
+
+import importlib.machinery
+import importlib.metadata
+import sqlite3
+import sys
+import types
+import typing
+
+# IGNORE: START
+# -------------
+# The lines here allow coherent type-checking of this file.
+# However, the actual lines are removed and replaced when this template is rendered.
+database: bytes = b""
+sqliteimport_modules: dict[str, str] = {}  # Inject: sqliteimport_modules
+# -------------
+# IGNORE: END
+
+
+if sys.version_info < (3, 11):
+    msg = """
+        Python 3.11 or higher is required to run this program.
+        (Python {sys.version} detected.)
+        ----
+        If you are a user:
+        This program may support Python 3.10 and lower in other circumstances.
+        However, it is packaged in a way that requires Python 3.11 and higher.
+        If you have access to Python 3.11 or higher,
+        you may be able to re-run the program.
+        ----
+        If you are developer:
+        This program's Python dependencies were injected using sqliteimport.
+        Only Python 3.11 and higher have compatible sqlite3 APIs to support this.
+    """
+    raise RuntimeError(msg)
+
+
+class DictFinder(importlib.metadata.DistributionFinder):
+    def __init__(self, modules: dict[str, str]) -> None:
+        self.modules = modules
+
+    def find_spec(
+        self,
+        fullname: str,
+        path: typing.Sequence[str] | None,
+        target: types.ModuleType | None = None,
+    ) -> importlib.machinery.ModuleSpec | None:
+        if not (fullname == "sqliteimport" or fullname.startswith("sqliteimport.")):
+            return None
+        if fullname not in self.modules:
+            return None
+
+        if fullname == "sqliteimport":
+            path = "sqliteimport/__init__.py"
+            is_package = True
+        else:
+            path = fullname.replace(".", "/") + ".py"
+            is_package = False
+        source = self.modules[fullname]
+        code = compile(source, filename=path, mode="exec", dont_inherit=True)
+        spec = importlib.machinery.ModuleSpec(
+            name=fullname,
+            loader=DictLoader(code, source),
+            origin=f"dict://{path}",
+            is_package=is_package,
+        )
+        spec.has_location = False
+        spec.cached = None
+
+        return spec
+
+    def find_distributions(
+        self,
+        context: importlib.metadata.DistributionFinder.Context | None = None,
+    ) -> typing.Iterable[importlib.metadata.Distribution]:
+        yield from ()
+
+
+class DictLoader(importlib.abc.InspectLoader):
+    def __init__(self, code: types.CodeType, source: str) -> None:
+        self.code = code
+        self.source = source
+
+    def exec_module(self, module: types.ModuleType) -> None:
+        exec(self.code, module.__dict__)
+
+    def get_source(self, fullname: str) -> str:
+        return self.source
+
+
+# Import sqliteimport.
+sys.meta_path.insert(0, DictFinder(sqliteimport_modules))
+import sqliteimport  # noqa: E402
+
+del sys.meta_path[0]
+
+# Load the database in-memory.
+connection = sqlite3.connect(":memory:")
+connection.deserialize(database)
+sqliteimport.load(connection)

src/sqliteimport/injector.py

@@ -0,0 +1,85 @@
+# This file is a part of sqliteimport <https://github.com/kurtmckee/sqliteimport>
+# Copyright 2024-2025 Kurt McKee <[email protected]>
+# SPDX-License-Identifier: MIT
+
+import importlib.resources
+import pathlib
+import textwrap
+
+
+def generate_prologue(database_path: pathlib.Path) -> str:
+    header = textwrap.dedent(
+        """\
+        # The code in this block was generated by sqliteimport.
+        # https://github.com/kurtmckee/sqliteimport
+        # Copyright Kurt McKee <[email protected]>
+        # SPDX-License-Identifier: MIT
+        """
+    ).rstrip()
+    lines: list[str] = [header]
+
+    # Add code variables.
+    sqliteimport_modules = get_sqliteimport_modules()
+    lines.append(f"sqliteimport_modules = {sqliteimport_modules!r}")
+
+    template = importlib.resources.read_text("sqliteimport", "injector-template.py")
+    drop_lines = False
+    for line in template.splitlines():
+        if not line.strip():
+            continue
+        if line.endswith("# IGNORE: START"):
+            drop_lines = True
+            continue
+        if line.endswith("# IGNORE: END"):
+            drop_lines = False
+            continue
+        if drop_lines:
+            continue
+        if line.lstrip().startswith("#"):
+            continue
+        lines.append(line)
+
+    wrapper = textwrap.dedent(
+        """
+        # BEGIN GENERATED CODE BLOCK. DO NOT EDIT!
+        def __sqliteimport_setup(database: bytes) -> None:
+        {function_block}
+        __sqliteimport_database = {database!r}
+        __sqliteimport_setup(database=__sqliteimport_database)
+        del __sqliteimport_database
+        del __sqliteimport_setup
+        # END GENERATED CODE BLOCK.
+        """
+    )
+    function_block = textwrap.indent("\n".join(lines), "    ")
+    database = database_path.read_bytes()
+    return wrapper.format(function_block=function_block, database=database)
+
+
+def inject_prologue(prologue: str, code: str, marker: str) -> str:
+    """Inject the *prologue* into *code*."""
+
+    lines: list[str] = []
+    code_iter = iter(code.splitlines())
+    for line in code_iter:
+        if line.strip() == f"# {marker}":
+            prefix = line[: line.find("#")]
+            lines.append(textwrap.indent(prologue, prefix))
+            break
+        lines.append(line)
+    lines.extend(code_iter)
+    return "\n".join(lines)
+
+
+def get_sqliteimport_modules() -> dict[str, str]:
+    files: dict[str, str] = {}
+    for file in importlib.resources.files("sqliteimport").iterdir():
+        if not file.name.endswith(".py"):
+            continue
+        stem = file.name[:-3]
+        if stem == "__init__":
+            fullname = "sqliteimport"
+        else:
+            fullname = f"sqliteimport.{stem}"
+        files[fullname] = file.read_text()
+    return files