refactor: Replace the gRPC-backed RepositoryDB with a JSON-file backed one. (#48)

This is an effort to reduce the impact of our services on starting and
using FIREWHEEL. Specifically, the RepositoryDB has been part of the
gRPC service that FIREWHEEL provides. This inter-dependency is
unnecessary for the following reasons:

1. It does not need to be accessed except from the "control node".
2. The gRPC service stored the repositories (e.g., a path) in a JSON
file.
3. As repositories become pip packages, the necessity of this feature
wains entirely.

In this PR, we strip away the unnecessary complexity of gRPC and simply
rely on the single JSON file for locally-installed model components. We
have updated all necessary documentation and tests accordingly. Finally,
we needed to provide a minimum version for the `grpcio` package as the
newly generated code checks version information.

Note that the `pb2*` files are auto-generated via `grpc_tools.protoc`.

---------

Signed-off-by: Steven Elliott <[email protected]>
Co-authored-by: Mitch Negus <[email protected]>

Author: sdelliot

docs/source/_static/proto_doc.html

@@ -77,7 +77,7 @@ FIREWHEEL gRPC Protocol
 
 This documentation was compiled using `protoc <https://github.com/protocolbuffers/protobuf>`_ and `protoc-gen-doc <https://github.com/pseudomuto/protoc-gen-doc>`_ with::
 
-    protoc --doc_out=./doc/_static/ --doc_opt=html,proto_doc.html ./grpc/firewheel_grpc.proto
+    python -m grpc_tools.protoc --plugin=protoc-gen-doc=./protoc-gen-doc --doc_out=./docs/source/_static/ --doc_opt=html,proto_doc.html --proto_path ./src/firewheel/lib/grpc firewheel_grpc.proto
 
 minimega/api.py
 ---------------

docs/source/code/lib.rst

@@ -58,8 +58,8 @@ dependencies = [
     "ClusterShell<=1.9.2",
     "colorama<=0.4.6",
     "coverage<=7.6.10",
-    "grpcio>=1.49.0,<=1.67.0",
-    "grpcio-tools>=1.49.0,<=1.69.0",
+    "grpcio>=1.63.0,<=1.67.0",
+    "grpcio-tools>=1.63.0,<=1.67.0",
     "importlib_metadata>=3.6,<=8.5.0",
     "Jinja2>=3.1.2,<=3.1.5",
     "netaddr<=1.3.0,>=0.7.0",
@@ -217,6 +217,7 @@ target-version = "py39"
 extend-exclude = [
     "test_*.py",
     "firewheel_grpc_pb2*.py",
+    "firewheel_grpc_pb2.pyi"
 ]
 
 [tool.ruff.lint]

pyproject.toml

@@ -1,19 +1,22 @@
 import os
+import json
+from pathlib import Path
 
 from importlib_metadata import entry_points
 
 from firewheel.config import config
 from firewheel.lib.log import Log
-from firewheel.lib.grpc.firewheel_grpc_client import FirewheelGrpcClient
 
 
 class RepositoryDb:
     """
-    Provides an interface to the Repository database on the gRPC service.
+    Provides an interface to the Repository database.
+    This database is functionally a JSON file which stores a list of locally installed
+    model component repositories which are uniquely identified by their path.
 
-    Repositories in the database are uniquely identified by their path.
+    A repository, as stored in the database, is expected to have the form:
 
-    A repository, as stored in the database, is expected to have the form::
+    .. code-block:: json
 
         {
             "path": ""
@@ -23,28 +26,22 @@ class RepositoryDb:
 
     def __init__(
         self,
-        host=config["grpc"]["hostname"],
-        port=config["grpc"]["port"],
-        db=config["grpc"]["db"],
+        db_basepath=config["system"]["default_output_dir"],
+        db_filename="repositories.json",
     ):
         """
-        Set up the connection to the gRPC server.
+        Set up the instance variables and path to the RepositoryDB file.
 
         Args:
-            host (str): The GRPC server IP/hostname.
-            port (int): The GRPC server port.
-            db (str): The GRPC database.
+            db_basepath (str): The base path where the RepositoryDB file is stored.
+            db_filename (str): The name of the RepositoryDB file. Defaults to "repositories.json".
         """
         self.log = Log(name="RepositoryDb").log
-        self.grpc_client = None
-        if host:
-            self.grpc_client = FirewheelGrpcClient(hostname=host, port=port, db=db)
-
-    def close(self):
-        """
-        Close the database connection.
-        """
-        self.grpc_client.close()
+        self.db_file = Path(db_basepath) / Path(db_filename)
+        self.db_file.parent.mkdir(parents=True, exist_ok=True)
+        if not self.db_file.exists():
+            with self.db_file.open("w") as db:
+                json.dump([], db)
 
     def list_repositories(self):
         """
@@ -60,9 +57,14 @@ def list_repositories(self):
         """
         entries = []
 
-        # Add all model component repositories identified by the gRPC client
-        if self.grpc_client is not None:
-            entries.extend(list(self.grpc_client.list_repositories()))
+        # Add all local model component repositories
+        if self.db_file.exists():
+            with self.db_file.open("r") as db:
+                try:
+                    local_entries = json.load(db)
+                    entries.extend(local_entries)
+                except json.decoder.JSONDecodeError:
+                    self.log.warning("Repository DB unable to be read.")
 
         # Add all model components that have been added via entry points
         for entry in entry_points(group="firewheel.mc_repo"):
@@ -78,7 +80,24 @@ def add_repository(self, repository):
             repository (dict): A repository dictionary to add. See format for the database.
         """
         self._validate_repository(repository)
-        self.grpc_client.set_repository(repository)
+
+        # Get all local model component repositories
+        if self.db_file.exists():
+            with self.db_file.open("r") as db:
+                try:
+                    entries = json.load(db)
+                except json.decoder.JSONDecodeError:
+                    self.log.warning("Repository DB unable to be read.")
+                    entries = []
+        else:
+            # No database file exists yet.
+            entries = []
+
+        entries.append(repository)
+        with self.db_file.open("w") as db:
+            json.dump(entries, db)
+
+        self.log.debug("Added repository: %s", repository)
 
     def delete_repository(self, repository):
         """
@@ -91,9 +110,36 @@ def delete_repository(self, repository):
             int: Number of entries removed (expect 0 or 1), or None if an
             error occurred.
         """
-        self._validate_repository(repository)
-        result = self.grpc_client.remove_repository(repository)["removed_count"]
-        return result
+        # Get all local model component repositories
+        if self.db_file.exists():
+            with self.db_file.open("r") as db:
+                try:
+                    entries = json.load(db)
+                except json.decoder.JSONDecodeError:
+                    self.log.warning("Repository DB unable to be read.")
+                    entries = []
+        else:
+            # No database file exists yet.
+            entries = []
+
+        try:
+            entries.remove(repository)
+            self._validate_repository(repository)
+            self.log.debug("Removed repository: %s", repository)
+        except ValueError:
+            self.log.debug(
+                "%s repository did not exist and could not be removed.", repository
+            )
+            return 0
+        except FileNotFoundError:
+            self.log.debug(
+                "%s repository path does not exist, but was removed anyways.", repository
+            )
+
+        with self.db_file.open("w") as db:
+            json.dump(entries, db)
+
+        return 1
 
     def _validate_repository(self, repository):
         """

src/firewheel/control/repository_db.py

@@ -10,15 +10,6 @@ package firewheel_grpc;
 
 
 service Firewheel {
-  // Set a repository.
-  rpc SetRepository(Repository) returns (SetRepositoryResponse) {}
-  // Remove a repository.
-  rpc RemoveRepository(Repository) returns (RemoveRepositoryResponse) {}
-  // List all repositories.
-  rpc ListRepositories(ListRepositoriesRequest) returns (stream Repository) {}
-  // Remove all repositories.
-  rpc RemoveAllRepositories(RemoveAllRepositoriesRequest) returns (RemoveAllRepositoriesResponse) {}
-
   // Gets the gRPC server info.
   rpc GetInfo(GetInfoRequest) returns (GetInfoResponse) {}
 
@@ -52,28 +43,6 @@ service Firewheel {
 
 }
 
-message Repository {
-    string db = 1;
-    string path = 2;
-}
-
-message SetRepositoryResponse {
-}
-
-message RemoveRepositoryResponse {
-    int32 removed_count = 1;
-}
-
-message ListRepositoriesRequest {
-    string db = 1;
-}
-message RemoveAllRepositoriesRequest {
-    string db = 1;
-}
-message RemoveAllRepositoriesResponse {
-    int32 removed_count = 1;
-}
-
 message GetInfoRequest {
 }
 

src/firewheel/lib/grpc/firewheel_grpc.proto

@@ -91,74 +91,6 @@ def get_info(self):
             return None
         return response_dict
 
-    def set_repository(self, repository):
-        """
-        Requests that the given repository is set.
-
-        Args:
-            repository (dict): The repository as a dictionary.
-
-        Returns:
-            dict: Dictionary representation of firewheel_grpc_pb2.SetRepositoryResponse.
-        """
-        req = firewheel_grpc_pb2.Repository(db=self.db, path=repository["path"])
-        response = self.stub.SetRepository(req)
-        return msg_to_dict(response)
-
-    def remove_repository(self, repository):
-        """
-        Requests that the given repository is removed.
-
-        Args:
-            repository (dict): The repository as a dictionary.
-
-        Returns:
-            dict: Dictionary representation of firewheel_grpc_pb2.RemoveRepositoryResponse.
-        """
-        req = firewheel_grpc_pb2.Repository(db=self.db, path=repository["path"])
-        response = self.stub.RemoveRepository(req)
-        return msg_to_dict(response)
-
-    def remove_all_repositories(self):
-        """
-        Requests to remove all repositories.
-
-        Returns:
-            dict: Dictionary representation of `firewheel_grpc_pb2.RemoveAllRepositoriesResponse`
-        """
-        req = firewheel_grpc_pb2.RemoveAllRepositoriesRequest(db=self.db)
-        response = self.stub.RemoveAllRepositories(req)
-        return msg_to_dict(response)
-
-    def get_repositories_as_dict(self):
-        """
-        Requests all of the repositories.
-
-        Returns:
-            (iterable) dict: Dictionary representations of `firewheel_grpc_pb2.Repository`.
-        """
-        req = firewheel_grpc_pb2.ListRepositoriesRequest(db=self.db)
-
-        repositories = self.stub.ListRepositories(req)
-        ret = {}
-        for repository in repositories:
-            repo_dict = msg_to_dict(repository)
-            ret[repo_dict["path"]] = repo_dict
-        return ret
-
-    def list_repositories(self):
-        """
-        Requests all of the repositories.
-
-        Yields:
-            dict: Dictionary representations of `firewheel_grpc_pb2.Repository`.
-        """
-        req = firewheel_grpc_pb2.ListRepositoriesRequest(db=self.db)
-
-        repositories = self.stub.ListRepositories(req)
-        for repository in repositories:
-            yield msg_to_dict(repository)
-
     def get_experiment_launch_time(self):
         """
         Requests the `experiment_launch_time`.