BREAKING: remove old functions and refactor others to solely use S3

Author: paulmueller

CHANGELOG

@@ -1,4 +1,7 @@
 0.13.0
+ - BREAKING: remove `get_resource_path`, `get_dcor_depot_path`,
+   `get_dcor_users_depot_path`
+ - BREAKING: `wait_for_resource` and `get_dc_instance` now only check S3 data
  - feat: add `is_resource_private` method
  - enh: add "artifact" kwarg to `get_s3_dc_handle`
  - enh: partial resource ID in basin names for `get_s3_dc_handle_basin_based`

dcor_shared/__init__.py

@@ -1,9 +1,9 @@
 # flake8: noqa: F401
 from .ckan import (
-    get_ckan_config_option, get_resource_dc_config, get_resource_path,
+    get_ckan_config_option, get_resource_dc_config,
     get_resource_info, is_resource_private
 )
-from .data import DUMMY_BYTES, wait_for_resource
+from .data import wait_for_resource
 from .dcinst import get_dc_instance
 from .mime import DC_MIME_TYPES, VALID_FORMATS
 from . import paths

dcor_shared/ckan.py

@@ -1,6 +1,5 @@
 import os
 import pathlib
-import warnings
 
 from .parse import get_ini_config_option
 
@@ -73,30 +72,6 @@ def get_resource_info(resource_id):
     return ds_dict, res_dict
 
 
-def get_resource_path(resource_id, create_dirs=False):
-    """Return the expected local path for a resource identifier
-
-    If `create_dirs` is True, create the parent directory tree.
-    """
-    warnings.warn("`get_resource_path` should not be used since DCOR moved "
-                  "to storing data solely on S3",
-                  DeprecationWarning)
-    rid = resource_id
-    resources_path = get_ckan_storage_path() / "resources"
-    pdir = resources_path / rid[:3] / rid[3:6]
-    path = pdir / rid[6:]
-    if create_dirs:
-        try:
-            pdir.mkdir(parents=True, exist_ok=True)
-            os.makedirs(pdir)
-            os.chown(pdir,
-                     os.stat(resources_path).st_uid,
-                     os.stat(resources_path).st_gid)
-        except OSError:
-            pass
-    return pathlib.Path(path)
-
-
 def is_resource_private(resource_id):
     """Return true if a resource is private"""
     from ckan import model

dcor_shared/data.py

@@ -1,16 +1,10 @@
 import functools
-import pathlib
 import time
 
-from .ckan import get_resource_path
 from . import s3cc
 from .util import sha256sum  # noqa: F401
 
 
-#: Content of the dummy file created when importing data.
-DUMMY_BYTES = b"[Data import pending]"
-
-
 @functools.lru_cache(maxsize=100)
 def wait_for_resource(resource_id: str,
                       timeout: float = 10):
@@ -19,35 +13,21 @@ def wait_for_resource(resource_id: str,
     This function can be used by other plugins to ensure that
     a resource is available for processing.
 
-    There multiple ways for data to become available:
-
-    1. The ckanext-dcor_depot plugin imports data by touching
-       dummy files and then sym-linking to data on disk. Here
-       we just check that the file is not a dummy file anymore.
-    2. Legacy uploads via nginx/uwsgi directly into CKAN and onto
-       the local block storage worked the same way. We have to wait
-       for the dummy file to be replaced.
-    3. The new (2024) way of uploading data is via pre-signed URLs
-       to an S3 instance. Here, we have to make sure that the
-       upload is complete and the file exists. If this is the case,
-       then uploads should have already completed when this function
-       is called, so we only check for the existence of the resource
-       in ckan and whether the `s3_available` attribute is defined.
+    The only way (since 2024) of uploading data is via pre-signed URLs
+    to an S3 instance. Here, we have to make sure that the
+    upload is complete and the file exists. If this is the case,
+    then uploads should have already completed when this function
+    is called, so we only check for the existence of the resource
+    in ckan and whether the `s3_available` attribute is defined.
     """
-    from ckan.common import config
     from ckan import logic
 
     if len(resource_id) != 36:
         raise ValueError(f"Invalid resource id: {resource_id}")
 
     resource_show = logic.get_action("resource_show")
-    path = pathlib.Path(get_resource_path(resource_id))
 
-    dcor_depot_available = "dcor_depot" in config.get('ckan.plugins', "")
-    # Initially this was set to 10s, but if `/data` is mounted on a
-    # network share then this part here just takes too long.
     t0 = time.time()
-    ld = len(DUMMY_BYTES)
     while True:
         try:
             res_dict = resource_show(context={'ignore_auth': True,
@@ -74,21 +54,5 @@ def wait_for_resource(resource_id: str,
             break
 
         if time.time() - t0 > timeout:
-            raise OSError("Data import seems to take too long "
-                          "for '{}'!".format(path))
-        elif not path.exists():
-            time.sleep(5)
-            continue
-        elif dcor_depot_available and not path.is_symlink():
-            # Resource is only available when it is symlinked by
-            # the ckanext.dcor_depot `symlink_user_dataset` job
-            # (or by the ckanext.dcor_depot importers).
-            time.sleep(5)
-            continue
-        elif path.stat().st_size == ld and path.read_bytes() == DUMMY_BYTES:
-            # wait a bit
-            time.sleep(5)
-            continue
-        else:
-            # not a dummy file
-            break
+            raise OSError(f"Data import seems to take too long "
+                          f"for '{resource_id}'!")

dcor_shared/dcinst.py

@@ -1,20 +1,10 @@
-from dclab.rtdc_dataset import RTDC_HDF5
-
-from .ckan import get_resource_path
 from . import s3cc
 
 
 def get_dc_instance(rid):
     """Return an instance of dclab's `RTDCBase` for a resource identifier"""
-    # Try local file first
-    path = get_resource_path(rid)
-    if path.is_file():
-        # Disable basins, because they could point to files on the local
-        # file system (security).
-        return RTDC_HDF5(path, enable_basins=False)
+    # The resource must be on S3
+    if s3cc.artifact_exists(rid):
+        return s3cc.get_s3_dc_handle(rid)
     else:
-        # The resource must be on S3
-        if s3cc.artifact_exists(rid):
-            return s3cc.get_s3_dc_handle(rid)
-        else:
-            raise ValueError(f"Could not find resource {rid} anywhere")
+        raise ValueError(f"Could not find resource {rid} anywhere")

dcor_shared/paths.py

@@ -1,22 +1,7 @@
 import pathlib
 
-from .ckan import get_ckan_config_path, get_ckan_config_option
 from .ckan import get_ckan_storage_path, get_ckan_webassets_path  # noqa: F401
 
-from .parse import get_ini_config_option
-
-
-def get_dcor_depot_path():
-    return pathlib.Path(get_ckan_config_option(
-        "ckanext.dcor_depot.depots_path"))
-
-
-def get_dcor_users_depot_path():
-    depot = get_dcor_depot_path()
-    return depot / get_ini_config_option(
-        "ckanext.dcor_depot.users_depot_name",
-        get_ckan_config_path())
-
 
 def get_nginx_config_path():
     return pathlib.Path("/etc/nginx/sites-enabled/ckan")