chore: migrate agent_morpheus_common_lib to NAT

pyproject.toml

@@ -37,6 +37,10 @@ classifiers = ["Programming Language :: Python"]
 [project.entry-points.'aiq.components']
 vuln_analysis = "vuln_analysis.register"
 
+[tool.setuptools.packages.find]
+where = ["src"]
+namespaces = false
+
 [dependency-groups]
 # Dependency groups are only for developers to aid in managing dependencies local to a dev machine.
 dev = [

src/vuln_analysis/data_models/info.py → src/exploit_iq_commons/data_models/info.py

@@ -15,8 +15,8 @@
 
 from pydantic import BaseModel
 
-from .cve_intel import CveIntel
-from .dependencies import VulnerableDependencies
+from exploit_iq_commons.data_models.cve_intel import CveIntel
+from exploit_iq_commons.data_models.dependencies import VulnerableDependencies
 
 
 class SBOMPackage(BaseModel):

src/vuln_analysis/data_models/input.py → src/exploit_iq_commons/data_models/input.py

@@ -26,12 +26,12 @@
 from pydantic import Tag
 from pydantic import field_validator
 
-from ..utils.string_utils import is_valid_cve_id
-from ..utils.string_utils import is_valid_ghsa_id
-from .common import HashableModel
-from .common import TypedBaseModel
-from .info import AgentMorpheusInfo
-from .info import SBOMPackage
+from exploit_iq_commons.utils.string_utils import is_valid_cve_id
+from exploit_iq_commons.utils.string_utils import is_valid_ghsa_id
+from exploit_iq_commons.data_models.common import HashableModel
+from exploit_iq_commons.data_models.common import TypedBaseModel
+from exploit_iq_commons.data_models.info import AgentMorpheusInfo
+from exploit_iq_commons.data_models.info import SBOMPackage
 
 
 class SourceDocumentsInfo(HashableModel):

src/vuln_analysis/utils/document_embedding.py → src/exploit_iq_commons/embedding/document_embedding.py

@@ -36,12 +36,13 @@
 from langchain_community.vectorstores import FAISS
 from langchain_core.document_loaders.blob_loaders import Blob
 
-from vuln_analysis.data_models.input import SourceDocumentsInfo
-from vuln_analysis.utils.go_segmenters_with_methods import GoSegmenterWithMethods
-from vuln_analysis.utils.js_extended_parser import ExtendedJavaScriptSegmenter
-from vuln_analysis.utils.source_code_git_loader import SourceCodeGitLoader
-from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher
-from vuln_analysis.logging.loggers_factory import LoggingFactory
+from exploit_iq_commons.data_models.input import SourceDocumentsInfo
+from exploit_iq_commons.embedding.go_segmenters_with_methods import GoSegmenterWithMethods
+from exploit_iq_commons.embedding.js_extended_parser import ExtendedJavaScriptSegmenter
+from exploit_iq_commons.embedding.source_code_git_loader import SourceCodeGitLoader
+from vuln_analysis.utils.git_utils import sanitize_git_url_for_path
+from exploit_iq_commons.embedding.transitive_code_searcher_tool import TransitiveCodeSearcher
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory
 
 if typing.TYPE_CHECKING:
     from langchain_core.embeddings import Embeddings  # pragma: no cover
@@ -348,7 +349,11 @@ def get_repo_path(self, source_info: SourceDocumentsInfo):
         Path
             Returns the path to the git repository.
         """
-        return self._git_directory / PurePath(source_info.git_repo)
+        # Sanitize the git repo URL to create a valid filesystem path
+        # Remove protocol separators and path separators that could cause issues
+        # Example: 'https://github.com/RHEcosystemAppEng/vulnerability-analysis' -> 'https.github.com.RHEcosystemAppEng.vulnerability-analysis'
+        sanitized_repo_path = sanitize_git_url_for_path(source_info.git_repo)
+        return self._git_directory / PurePath(sanitized_repo_path)
 
     def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]:
         """

src/vuln_analysis/utils/js_extended_parser.py → src/exploit_iq_commons/embedding/js_extended_parser.py

@@ -21,7 +21,7 @@
 import esprima
 from langchain_community.document_loaders.parsers.language.javascript import JavaScriptSegmenter
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory
 logger = LoggingFactory.get_agent_logger(__name__)
 
 

src/vuln_analysis/utils/source_code_git_loader.py → src/exploit_iq_commons/embedding/source_code_git_loader.py

@@ -24,8 +24,8 @@
 from langchain_core.document_loaders.blob_loaders import Blob
 from tqdm import tqdm
 
-from vuln_analysis.utils.transitive_code_searcher_tool import TransitiveCodeSearcher
-from vuln_analysis.logging.loggers_factory import LoggingFactory
+from exploit_iq_commons.embedding.transitive_code_searcher_tool import TransitiveCodeSearcher
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory
 
 PathLike = typing.Union[str, os.PathLike]
 
@@ -179,18 +179,18 @@ def yield_blobs(self) -> typing.Iterator[Blob]:
         logger.info("Processing %d files in the Git repository at path: '%s'", len(final_files), self.repo_path)
 
         for f in tqdm(final_files):
-
-            file_path = Path(f)
-
-            abs_file_path = base_path / file_path
-
-            rel_file_path = str(file_path)
-
-            metadata = {
-                "source": rel_file_path,
-                "file_path": rel_file_path,
-                "file_name": file_path.name,
-                "file_type": file_path.suffix,
-            }
-
-            yield Blob.from_path(abs_file_path, metadata=metadata)
+            abs_file_path = base_path / f
+            if abs_file_path.is_file():
+                try:
+                    rel_file_path = str(f)
+                    metadata = {
+                        "source": rel_file_path,
+                        "file_path": rel_file_path,
+                        "file_name": abs_file_path.name,
+                        "file_type": abs_file_path.suffix,
+                    }
+                    yield Blob.from_path(abs_file_path, metadata=metadata)
+                except Exception as e:
+                    logger.warning("Failed to read blob for '%s'. Ignoring this file. Error: %s", abs_file_path, e)
+            else:
+                logger.debug("Skipping path as it is a directory, not a file: '%s'", abs_file_path)

src/vuln_analysis/utils/transitive_code_searcher_tool.py → src/exploit_iq_commons/embedding/transitive_code_searcher_tool.py

@@ -18,8 +18,8 @@
 
 from langchain.docstore.document import Document
 
-from .chain_of_calls_retriever import ChainOfCallsRetriever
-from .dep_tree import (
+from exploit_iq_commons.utils.chain_of_calls_retriever import ChainOfCallsRetriever
+from exploit_iq_commons.utils.dep_tree import (
     GOLANG_MANIFEST,
     JAVA_MANIFEST,
     JS_MANIFEST,
@@ -29,7 +29,7 @@
     get_dependency_tree_builder,
 )
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory, MULTI_LINE_MESSAGE_TRUE
 
 logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}")
 

src/exploit_iq_commons/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["setuptools >= 64", "setuptools-scm>=8"]
+
+[project]
+name = "exploit-iq-commons"
+version = "0.1.0"
+description = "Common library for ExploitIQ."
+requires-python = ">=3.11,<3.13"
+
+dependencies = [
+    "esprima==4.0.1",
+    "GitPython==3.1.44",
+    "langchain-community>=0.3,<0.4",
+    "tqdm==4.67.1",
+    "tree-sitter-languages==1.10.2",
+    "tree-sitter==0.21.3",
+]
+
+[tool.setuptools.packages.find]
+where = [".."]
+include = ["exploit_iq_commons*"]

src/vuln_analysis/utils/chain_of_calls_retriever.py → src/exploit_iq_commons/utils/chain_of_calls_retriever.py

@@ -7,16 +7,16 @@
 import logging
 
 from .dep_tree import ROOT_LEVEL_SENTINEL, DependencyTree, Ecosystem
-from .functions_parsers.lang_functions_parsers import LanguageFunctionsParser
-from .functions_parsers.lang_functions_parsers_factory import (
+from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser
+from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers_factory import (
     get_language_function_parser,
 )
 
 PARENTS_INDEX = 0
 
 EXCLUSIONS_INDEX = 1
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory
 logger = LoggingFactory.get_agent_logger(f"morpheus.{__name__}")
 
 

src/vuln_analysis/utils/dep_tree.py → src/exploit_iq_commons/utils/dep_tree.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 import logging
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory
 logger = LoggingFactory.get_agent_logger(__name__)
 
 

src/vuln_analysis/utils/functions_parsers/golang_functions_parsers.py → src/exploit_iq_commons/utils/functions_parsers/golang_functions_parsers.py

@@ -3,7 +3,7 @@
 
 from langchain_core.documents import Document
 
-from .lang_functions_parsers import LanguageFunctionsParser
+from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser
 
 EMBEDDED_TYPE = "embedded_type"
 

src/vuln_analysis/utils/functions_parsers/lang_functions_parsers_factory.py → src/exploit_iq_commons/utils/functions_parsers/lang_functions_parsers_factory.py

@@ -1,6 +1,6 @@
-from ..dep_tree import Ecosystem
-from .golang_functions_parsers import GoLanguageFunctionsParser
-from .lang_functions_parsers import LanguageFunctionsParser
+from exploit_iq_commons.utils.dep_tree import Ecosystem
+from exploit_iq_commons.utils.functions_parsers.golang_functions_parsers import GoLanguageFunctionsParser
+from exploit_iq_commons.utils.functions_parsers.lang_functions_parsers import LanguageFunctionsParser
 
 
 def get_language_function_parser(ecosystem: Ecosystem) -> LanguageFunctionsParser:

src/vuln_analysis/data_models/output.py

@@ -18,7 +18,7 @@
 from pydantic import BaseModel
 from pydantic import model_validator
 
-from .input import AgentMorpheusEngineInput
+from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput
 
 
 class AgentIntermediateStep(BaseModel):

src/vuln_analysis/data_models/plugin.py

@@ -20,8 +20,8 @@
 
 import aiohttp
 
-from .common import TypedBaseModel
-from .cve_intel import IntelPluginData
+from exploit_iq_commons.data_models.common import TypedBaseModel
+from exploit_iq_commons.data_models.cve_intel import IntelPluginData
 
 
 _T = typing.TypeVar('_T', bound='PluginSchema')

src/vuln_analysis/data_models/plugins/intel_plugin.py

@@ -15,7 +15,7 @@
 from ..cve_intel import IntelPluginData
 from ..plugin import IntelPluginSchema
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory
 logger = LoggingFactory.get_agent_logger(__name__)
 
 

src/vuln_analysis/data_models/state.py

@@ -17,8 +17,8 @@
 
 from pydantic import BaseModel
 
-from vuln_analysis.data_models.cve_intel import CveIntel
-from vuln_analysis.data_models.input import AgentMorpheusEngineInput
+from exploit_iq_commons.data_models.cve_intel import CveIntel
+from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput
 
 
 class AgentMorpheusEngineState(BaseModel):

src/vuln_analysis/functions/cve_agent.py

@@ -32,7 +32,7 @@
 from pydantic import Field
 from vuln_analysis.data_models.state import AgentMorpheusEngineState
 from vuln_analysis.utils.prompting import get_agent_prompt
-from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id
 
 logger = LoggingFactory.get_agent_logger(__name__)
 

src/vuln_analysis/functions/cve_calculate_intel_score.py

@@ -39,7 +39,7 @@ class CVECalculateIntelScoreConfig(FunctionBaseConfig, name="cve_calculate_intel
 @register_function(config_type=CVECalculateIntelScoreConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
 async def cve_calculate_intel_score(config: CVECalculateIntelScoreConfig, builder: Builder):  # pylint: disable=unused-argument
 
-    from vuln_analysis.data_models.input import AgentMorpheusEngineInput
+    from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput
     from vuln_analysis.utils.intel_source_score import IntelScorer
 
     async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput:

src/vuln_analysis/functions/cve_check_vuln_deps.py

@@ -24,7 +24,7 @@
 from aiq.data_models.function import FunctionBaseConfig
 from pydantic import Field
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id
 
 logger = LoggingFactory.get_agent_logger(__name__)
 
@@ -41,10 +41,10 @@ class CVEVulnerableDepsChecksConfig(FunctionBaseConfig, name="cve_check_vuln_dep
 @register_function(config_type=CVEVulnerableDepsChecksConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
 async def cve_check_vuln_deps(config: CVEVulnerableDepsChecksConfig, builder: Builder):  # pylint: disable=unused-argument
 
-    from vuln_analysis.data_models.cve_intel import CveIntel
-    from vuln_analysis.data_models.dependencies import VulnerableDependencies
-    from vuln_analysis.data_models.dependencies import VulnerableSBOMPackage
-    from vuln_analysis.data_models.input import AgentMorpheusEngineInput
+    from exploit_iq_commons.data_models.cve_intel import CveIntel
+    from exploit_iq_commons.data_models.dependencies import VulnerableDependencies
+    from exploit_iq_commons.data_models.dependencies import VulnerableSBOMPackage
+    from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput
     from vuln_analysis.utils.vulnerable_dependency_checker import VulnerableDependencyChecker
 
     async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput:

src/vuln_analysis/functions/cve_checklist.py

@@ -24,7 +24,7 @@
 from aiq.data_models.function import FunctionBaseConfig
 from pydantic import Field
 from vuln_analysis.utils import data_utils
-from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id
 
 logger = LoggingFactory.get_agent_logger(__name__)
 

src/vuln_analysis/functions/cve_fetch_intel.py

@@ -25,10 +25,9 @@
 from aiq.data_models.function import FunctionBaseConfig
 from pydantic import Field
 
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id
 from vuln_analysis.data_models.plugin import PluginConfig
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id
-
 logger = LoggingFactory.get_agent_logger(__name__)
 
 
@@ -43,7 +42,7 @@ class CVEFetchIntelConfig(FunctionBaseConfig, name="cve_fetch_intel"):
 @register_function(config_type=CVEFetchIntelConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
 async def cve_fetch_intel(config: CVEFetchIntelConfig, builder: Builder):  # pylint: disable=unused-argument
 
-    from vuln_analysis.data_models.input import AgentMorpheusEngineInput
+    from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput
     from vuln_analysis.utils.intel_retriever import IntelRetriever
 
     async def _arun(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput:

src/vuln_analysis/functions/cve_file_output.py

@@ -21,7 +21,7 @@
 from aiq.data_models.function import FunctionBaseConfig
 from pydantic import Field
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory
 logger = LoggingFactory.get_agent_logger(__name__)
 
 

src/vuln_analysis/functions/cve_generate_vdbs.py

@@ -26,7 +26,7 @@
 from aiq.data_models.function import FunctionBaseConfig
 from pydantic import Field
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id
 
 logger = LoggingFactory.get_agent_logger(__name__)
 
@@ -56,12 +56,12 @@ class CVEGenerateVDBsToolConfig(FunctionBaseConfig, name="cve_generate_vdbs"):
 
 @register_function(config_type=CVEGenerateVDBsToolConfig, framework_wrappers=[LLMFrameworkEnum.LANGCHAIN])
 async def generate_vdb(config: CVEGenerateVDBsToolConfig, builder: Builder):
-    from vuln_analysis.data_models.info import AgentMorpheusInfo
-    from vuln_analysis.data_models.input import AgentMorpheusEngineInput
-    from vuln_analysis.data_models.input import AgentMorpheusInput
-    from vuln_analysis.data_models.input import SourceDocumentsInfo
+    from exploit_iq_commons.data_models.info import AgentMorpheusInfo
+    from exploit_iq_commons.data_models.input import AgentMorpheusEngineInput
+    from exploit_iq_commons.data_models.input import AgentMorpheusInput
+    from exploit_iq_commons.data_models.input import SourceDocumentsInfo
     from vuln_analysis.functions.cve_agent import CVEAgentExecutorToolConfig
-    from vuln_analysis.utils.document_embedding import DocumentEmbedding
+    from exploit_iq_commons.embedding.document_embedding import DocumentEmbedding
     from vuln_analysis.utils.full_text_search import FullTextSearch
     from vuln_analysis.utils.git_utils import get_repo_from_path
 
@@ -202,10 +202,12 @@ async def _arun(message: AgentMorpheusInput) -> AgentMorpheusEngineInput:
             # Replace ref with specific commit hash for each source info
             for si in source_infos:
                 try:
-                    repo = get_repo_from_path(config.base_git_dir, si.git_repo)
+                    # Get the sanitized path from the embedder instance
+                    repo_path = embedder.get_repo_path(si)
+                    repo = get_repo_from_path(str(repo_path.parent), repo_path.name)
                     si.ref = repo.commit().hexsha
                 except ValueError as e:
-                    logger.warning("Failed to get commit hash for %s/%s: %s", config.base_git_dir, si.git_repo, e)
+                    logger.warning("Failed to get commit hash for repo defined in %s: %s", si, e)
                     continue
 
         except Exception as e:

src/vuln_analysis/functions/cve_http_output.py

@@ -22,7 +22,7 @@
 from aiq.data_models.function import FunctionBaseConfig
 from pydantic import Field
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id
 
 logger = LoggingFactory.get_agent_logger(__name__)
 

src/vuln_analysis/functions/cve_justify.py

@@ -23,7 +23,7 @@
 from aiq.data_models.function import FunctionBaseConfig
 from pydantic import Field
 
-from vuln_analysis.logging.loggers_factory import LoggingFactory, trace_id
+from exploit_iq_commons.logging.loggers_factory import LoggingFactory, trace_id
 
 logger = LoggingFactory.get_agent_logger(__name__)