refactor!: [V8] Split container classes into read/write versions (DM-3215)

.coveragerc

@@ -1,5 +1,7 @@
 [run]
-source=cognite
+source =
+    cognite
+    tests
 omit =
     cognite/client/_proto/*
 

.gemini/styleguide.md

@@ -73,7 +73,7 @@ import logging
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-from requests import Response
+from httpx import Response
 
 from cognite.client.data_classes import Asset
 

.github/workflows/build.yml

@@ -4,7 +4,6 @@ name: build
 on:
   merge_group:
   pull_request:
-    branches: [master]
 
 concurrency:
   group: pr-${{ github.workflow }}-${{ github.head_ref }}
@@ -70,9 +69,10 @@ jobs:
           COGNITE_BASE_URL: ${{ vars.CDF_BASE_URL_CI }}
           COGNITE_CLIENT_NAME: python-sdk-integration-tests
         # Testpaths are defined in the pytest.ini file:
-        run: pytest --durations=10 --cov --cov-report term --cov-report xml:coverage.xml -n8 --dist loadscope --reruns 2 --maxfail 20
+        # run: pytest --durations=10 --cov --cov-report term --cov-report xml:coverage.xml -n8 --dist loadscope --reruns 2 --maxfail 20
+        run: pytest tests/tests_integration/test_api/test_workflows.py --reruns 5
 
-      - uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5
-        if: matrix.os == 'windows-latest' && matrix.python-version == '3.10'
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
+      # - uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5
+      #   if: matrix.os == 'windows-latest' && matrix.python-version == '3.10'
+      #   with:
+      #     token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/verify-jupyter.yml

@@ -29,5 +29,5 @@ jobs:
           whl_file=$(find dist -name "*.whl" | sed 's|^dist/||') # Find the built wheel file, remove dist/ prefix
           echo "Found built wheel file: $whl_file"
           SDK_FILE_PATH=$whl_file \
-          PACKAGES="[\"pyodide-http\", \"http://localhost:3000/dist/$whl_file\"]" \
+          PACKAGES="[\"anyio\", \"http://localhost:3000/dist/$whl_file\"]" \
           node scripts/test-pyodide.js

.github/workflows/verify-streamlit.yml

@@ -29,5 +29,5 @@ jobs:
           whl_file=$(find dist -name "*.whl" | sed 's|^dist/||') # Find the built wheel file, remove dist/ prefix
           echo "Found built wheel file: $whl_file"
           SDK_FILE_PATH=$whl_file \
-          PACKAGES="[\"pyodide-http\", \"http://localhost:3000/dist/$whl_file\"]" \
+          PACKAGES="[\"ssl\", \"anyio\", \"http://localhost:3000/dist/$whl_file\"]" \
           node scripts/test-pyodide.js

.pre-commit-config.yaml

@@ -32,16 +32,7 @@ repos:
     hooks:
       - id: mypy
         name: mypy
-      # Eventually we want to run mypy on the whole test code base. But for now we only enable it for data modeling, simulators and datapoints
-        entry: |
-          poetry run dmypy run -- cognite tests/tests_integration/test_api/test_data_modeling
-          tests/tests_unit/test_data_classes/test_data_models
-          tests/tests_unit/test_utils/test_time.py
-          tests/tests_integration/test_api/test_datapoints.py
-          tests/tests_integration/test_api/test_simulators
-          tests/tests_integration/test_api/test_principals.py
-          tests/tests_unit/test_api/test_simulators
-          tests/tests_unit/test_data_classes/test_simulators.py
+        entry: poetry run dmypy run -- cognite tests
         files: ^.*.(py|pyi)$
         language: system
         pass_filenames: false

MIGRATION_GUIDE.md

@@ -8,6 +8,64 @@ Changes are grouped as follows:
 - `Changed` for changes that do not fall into any other category
 - `Optional` for new, optional methods/features that you should be aware of - *and could take advantage of*
 
+
+## From v7 to v8
+
+### Dependency
+- **httpx**: The SDK now uses `httpx` instead of `requests` for HTTP operations. This is a required dependency.
+- **anyio**: Added as a required dependency (used by httpx, replaces pytest-asyncio).
+
+### Functionality
+- **Async Support**: The SDK now provides full async support. The main client is now `AsyncCogniteClient` (previously `CogniteClient`), and a new synchronous `CogniteClient` is available for backward compatibility.
+- **HTTP Client**: All HTTP operations now use `httpx.AsyncClient` instead of `requests`.
+- **DataFrames**: Datapoints DataFrames now use MultiIndex columns for better organization of aggregate data.
+- **File Handling**: FilesAPI now prefers `pathlib.Path` objects over strings (though string support is maintained).
+- **JSON Serialization**: The SDK now uses compact JSON encoding for better performance.
+
+### Removed
+- **Deprecated Aggregate Methods**: Removed deprecated `aggregate` methods from AssetsAPI, EventsAPI, SequencesAPI, and TimeSeriesAPI. Use `aggregate_count` instead.
+- **Deprecated Filter Methods**: Removed deprecated filter methods from various APIs.
+- **Deprecated Data Classes**:
+  - `CountAggregate` class removed
+  - `DataPointSubscriptionCreate` class removed
+  - `SequenceData` class replaced by `SequenceRows` (old class still available for backward compatibility)
+- **Deprecated Methods**:
+  - `DatapointsArray.__iter__` method removed
+  - `CogniteObject.load_yaml` method on Query removed
+  - `DataModelingInstancesList.get` method no longer accepts deprecated `id` argument
+  - All `__iter__` methods removed from API classes
+- **Legacy Attributes**:
+  - `legacy_name` attribute removed from time series data classes
+  - `expected_fields` parameter removed from `Datapoints._load`
+- **Pyodide Handling**: Removed special pyodide handling in FilesAPI upload operations.
+
+### Function Signature
+- **Client Rename**: `CogniteClient` is now `AsyncCogniteClient`. A new synchronous `CogniteClient` is available.
+- **Sequence Data**: `client.sequences.data.retrieve_latest` renamed to `client.sequences.data.retrieve_last_row`.
+- **Data Modeling**: `expand_properties` parameter in `to_pandas` methods now defaults to `True` for data modeling classes.
+- **File Paths**: FilesAPI methods now prefer `pathlib.Path` objects but still accept strings.
+
+### Changed
+- **API Organization**: Many APIs have been split into separate files (Java-style organization):
+  - Functions API split into `functions/` directory
+  - IAM API split into `iam/` directory
+  - Raw API split into `raw/` directory
+  - 3D API split into `three_d/` directory
+  - Workflows API split into `workflows/` directory
+- **Query Classes**: Query and sync classes have been split into separate dataclasses.
+- **Filter Classes**: Filter classes now have magic `__eq__` method for better usability.
+- **Data Classes**: Various data classes have been updated with new attributes and methods.
+- **Exception Handling**: New exception types added (e.g., `CogniteOrganizationError`, `CogniteModelFailedError`).
+
+### Optional
+- **New APIs**: Several new APIs have been added:
+  - Agents API for AI agent functionality
+  - Simulators API for simulation operations
+  - Document Preview API
+  - Space Statistics API
+- **New Data Classes**: Many new data classes added for new features like agents, simulators, and enhanced data modeling.
+- **Enhanced Capabilities**: New ACL capabilities added for Streams, Records, and AppConfig.
+
 ## From v6 to v7
 ### Functionality
 - `CogniteResource.to_pandas` and `CogniteResourceList.to_pandas` now converts known timestamps to `datetime` by

cognite/client/__init__.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
-from cognite.client._cognite_client import CogniteClient
+from cognite.client._cognite_client import AsyncCogniteClient
 from cognite.client._constants import _RUNNING_IN_BROWSER
+from cognite.client._sync_cognite_client import CogniteClient
 from cognite.client._version import __version__
 from cognite.client.config import ClientConfig, global_config
 from cognite.client.data_classes import data_modeling
 
-__all__ = ["ClientConfig", "CogniteClient", "__version__", "data_modeling", "global_config"]
+__all__ = ["AsyncCogniteClient", "ClientConfig", "CogniteClient", "__version__", "data_modeling", "global_config"]
 
 if _RUNNING_IN_BROWSER:
     from cognite.client.utils._pyodide_helpers import patch_sdk_for_pyodide

cognite/client/_api/agents/agents.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
-from typing import TYPE_CHECKING, Any, overload
+from typing import TYPE_CHECKING, overload
 
 from cognite.client._api_client import APIClient
 from cognite.client.data_classes.agents import Agent, AgentList, AgentUpsert
@@ -16,27 +16,27 @@
 from cognite.client.utils.useful_types import SequenceNotStr
 
 if TYPE_CHECKING:
-    from cognite.client import CogniteClient
+    from cognite.client import AsyncCogniteClient
     from cognite.client.config import ClientConfig
 
 
 class AgentsAPI(APIClient):
     _RESOURCE_PATH = "/ai/agents"
 
-    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: CogniteClient) -> None:
+    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None:
         super().__init__(config, api_version, cognite_client)
         self._warnings = FeaturePreviewWarning(api_maturity="beta", sdk_maturity="alpha", feature_name="Agents")
         self._api_subversion = "beta"
         self._CREATE_LIMIT = 1
         self._DELETE_LIMIT = 1
 
     @overload
-    def upsert(self, agents: AgentUpsert) -> Agent: ...
+    async def upsert(self, agents: AgentUpsert) -> Agent: ...
 
     @overload
-    def upsert(self, agents: Sequence[AgentUpsert]) -> AgentList: ...
+    async def upsert(self, agents: Sequence[AgentUpsert]) -> AgentList: ...
 
-    def upsert(self, agents: AgentUpsert | Sequence[AgentUpsert]) -> Agent | AgentList:
+    async def upsert(self, agents: AgentUpsert | Sequence[AgentUpsert]) -> Agent | AgentList:
         """`Create or update (upsert) one or more agents. <https://api-docs.cognite.com/20230101-beta/tag/Agents/operation/main_ai_agents_post/>`_
 
         Args:
@@ -159,20 +159,20 @@ def upsert(self, agents: AgentUpsert | Sequence[AgentUpsert]) -> Agent | AgentLi
 
         """
         self._warnings.warn()
-        return self._create_multiple(
+        return await self._create_multiple(
             list_cls=AgentList,
             resource_cls=Agent,
             items=agents,
             input_resource_cls=AgentUpsert,
         )
 
     @overload
-    def retrieve(self, external_ids: str, ignore_unknown_ids: bool = False) -> Agent | None: ...
+    async def retrieve(self, external_ids: str, ignore_unknown_ids: bool = False) -> Agent | None: ...
 
     @overload
-    def retrieve(self, external_ids: SequenceNotStr[str], ignore_unknown_ids: bool = False) -> AgentList: ...
+    async def retrieve(self, external_ids: SequenceNotStr[str], ignore_unknown_ids: bool = False) -> AgentList: ...
 
-    def retrieve(
+    async def retrieve(
         self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool = False
     ) -> Agent | AgentList | None:
         """`Retrieve one or more agents by external ID. <https://api-docs.cognite.com/20230101-beta/tag/Agents/operation/get_agents_by_ids_ai_agents_byids_post/>`_
@@ -188,8 +188,9 @@ def retrieve(
 
             Retrieve an agent by external id:
 
-                >>> from cognite.client import CogniteClient
+                >>> from cognite.client import CogniteClient, AsyncCogniteClient
                 >>> client = CogniteClient()
+                >>> # async_client = AsyncCogniteClient()  # another option
                 >>> res = client.agents.retrieve(external_ids="my_agent")
 
             Retrieve multiple agents:
@@ -198,14 +199,14 @@ def retrieve(
         """
         self._warnings.warn()
         identifiers = IdentifierSequence.load(external_ids=external_ids)
-        return self._retrieve_multiple(
+        return await self._retrieve_multiple(
             list_cls=AgentList,
             resource_cls=Agent,
             identifiers=identifiers,
             ignore_unknown_ids=ignore_unknown_ids,
         )
 
-    def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> None:
+    async def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> None:
         """`Delete one or more agents. <https://api-docs.cognite.com/20230101-beta/tag/Agents/operation/agent_delete_ai_agents_delete_post/>`_
 
         Args:
@@ -216,19 +217,20 @@ def delete(self, external_ids: str | SequenceNotStr[str], ignore_unknown_ids: bo
 
             Delete an agent by external id:
 
-                >>> from cognite.client import CogniteClient
+                >>> from cognite.client import CogniteClient, AsyncCogniteClient
                 >>> client = CogniteClient()
+                >>> # async_client = AsyncCogniteClient()  # another option
                 >>> client.agents.delete(external_ids="my_agent")
 
         """
         self._warnings.warn()
-        self._delete_multiple(
+        await self._delete_multiple(
             identifiers=IdentifierSequence.load(external_ids=external_ids),
             wrap_ids=True,
             extra_body_fields={"ignoreUnknownIds": ignore_unknown_ids},
         )
 
-    def list(self) -> AgentList:  # The API does not yet support limit or pagination
+    async def list(self) -> AgentList:  # The API does not yet support limit or pagination
         """`List agents. <https://api-docs.cognite.com/20230101-beta/tag/Agents/operation/agent_list_ai_agents_get/>`_
 
         Returns:
@@ -238,16 +240,17 @@ def list(self) -> AgentList:  # The API does not yet support limit or pagination
 
             List all agents:
 
-                >>> from cognite.client import CogniteClient
+                >>> from cognite.client import CogniteClient, AsyncCogniteClient
                 >>> client = CogniteClient()
+                >>> # async_client = AsyncCogniteClient()  # another option
                 >>> agent_list = client.agents.list()
 
         """
         self._warnings.warn()
-        res = self._get(url_path=self._RESOURCE_PATH)
+        res = await self._get(url_path=self._RESOURCE_PATH)
         return AgentList._load(res.json()["items"], cognite_client=self._cognite_client)
 
-    def chat(
+    async def chat(
         self,
         agent_external_id: str,
         messages: Message | ActionResult | Sequence[Message | ActionResult],
@@ -337,26 +340,18 @@ def chat(
         """
         self._warnings.warn()
 
-        # Convert single message to list
         if isinstance(messages, Message | ActionResult):
             messages = [messages]
 
-        # Build request body
-        body: dict[str, Any] = {
+        body = {
             "agentExternalId": agent_external_id,
             "messages": [msg.dump(camel_case=True) for msg in messages],
         }
-
         if cursor is not None:
             body["cursor"] = cursor
 
         if actions is not None:
             body["actions"] = [action.dump(camel_case=True) for action in actions]
 
-        # Make the API call
-        response = self._post(
-            url_path=self._RESOURCE_PATH + "/chat",
-            json=body,
-        )
-
+        response = await self._post(url_path=self._RESOURCE_PATH + "/chat", json=body)
         return AgentChatResponse._load(response.json(), cognite_client=self._cognite_client)

cognite/client/_api/ai/__init__.py

@@ -6,10 +6,10 @@
 from cognite.client._api_client import APIClient
 
 if TYPE_CHECKING:
-    from cognite.client import ClientConfig, CogniteClient
+    from cognite.client import AsyncCogniteClient, ClientConfig
 
 
 class AIAPI(APIClient):
-    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: CogniteClient) -> None:
+    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None:
         super().__init__(config, api_version, cognite_client)
         self.tools = AIToolsAPI(config, api_version, cognite_client)

cognite/client/_api/ai/tools/__init__.py

@@ -6,10 +6,10 @@
 from cognite.client._api_client import APIClient
 
 if TYPE_CHECKING:
-    from cognite.client import ClientConfig, CogniteClient
+    from cognite.client import AsyncCogniteClient, ClientConfig
 
 
 class AIToolsAPI(APIClient):
-    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: CogniteClient) -> None:
+    def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None:
         super().__init__(config, api_version, cognite_client)
         self.documents = AIDocumentsAPI(config, api_version, cognite_client)