Fix container tests in run_tests.sh use hard-coded DV_VERSION

README.md

@@ -21,7 +21,7 @@ In order to run the tests, you need to have a Dataverse instance running. We hav
 ./run_tests.sh -p 3.8
 ```
 
-Once finished, you can find the test results in the `dv/unit-tests.log` file and in the terminal.
+Once finished, you can find the test results in the `dv/unit-tests.log` file and in the terminal. The Dataverse logs are saved in the `dv/dataverse-logs.log` file and can be helpful in cases of internal server errors. If you happen to encounter such an error, please share the logs with us at the [Zulip](https://dataverse.zulipchat.com/#narrow/stream/377090-python) so we can investigate the issue.
 
 ## Manual setup
 
@@ -45,7 +45,16 @@ export $(grep "API_TOKEN" "dv/bootstrap.exposed.env")
 export API_TOKEN_SUPERUSER=$API_TOKEN
 ```
 
-**3. Run the test(s) with pytest**
+**3. Install test dependencies
+
+```
+python3 -m venv venv # or conda create -n pydataverse
+source venv/bin/activate
+pip install -e .
+python3 -m pip install pytest pytest-cov pytest-asyncio tox selenium
+```
+
+**4. Run the test(s) with pytest**
 
 ```bash
 python -m pytest -v

docker/docker-compose-base.yml

@@ -1,8 +1,7 @@
-version: "2.4"
 name: pydataverse
 services:
   dataverse:
-    container_name: "dataverse"
+    container_name: 'dataverse'
     hostname: dataverse
     image: ${DATAVERSE_IMAGE}
     restart: on-failure
@@ -18,7 +17,7 @@ services:
         -Ddataverse.pid.fake.authority=10.5072
         -Ddataverse.pid.fake.shoulder=FK2/
     ports:
-      - "8080:8080"
+      - '8080:8080'
     networks:
       - dataverse
     depends_on:
@@ -30,7 +29,6 @@ services:
         condition: service_completed_successfully
     volumes:
       - ${PWD}/dv/data:/dv
-      - ${PWD}:/secrets
     tmpfs:
       - /dumps:mode=770,size=2052M,uid=1000,gid=1000
       - /tmp:mode=770,size=2052M,uid=1000,gid=1000
@@ -45,68 +43,68 @@ services:
       timeout: 240s
 
   dv_initializer:
-    container_name: "dv_initializer"
+    container_name: 'dv_initializer'
     image: ${CONFIGBAKER_IMAGE}
-    restart: "no"
+    restart: 'no'
     command:
       - sh
       - -c
-      - "fix-fs-perms.sh dv"
+      - 'fix-fs-perms.sh dv'
     volumes:
       - ${PWD}/dv/data:/dv
 
   postgres:
-    container_name: "postgres"
+    container_name: 'postgres'
     hostname: postgres
     image: postgres:${POSTGRES_VERSION}
     restart: on-failure
     environment:
       - POSTGRES_USER=${DATAVERSE_DB_USER}
       - POSTGRES_PASSWORD=${DATAVERSE_DB_PASSWORD}
     ports:
-      - "5432:5432"
+      - '5432:5432'
     networks:
       - dataverse
 
   solr_initializer:
-    container_name: "solr_initializer"
+    container_name: 'solr_initializer'
     image: ${CONFIGBAKER_IMAGE}
-    restart: "no"
+    restart: 'no'
     command:
       - sh
       - -c
-      - "fix-fs-perms.sh solr && cp -a /template/* /solr-template"
+      - 'fix-fs-perms.sh solr && cp -a /template/* /solr-template'
     volumes:
       - ${PWD}/solr/data:/var/solr
       - ${PWD}/solr/conf:/solr-template
 
   solr:
-    container_name: "solr"
-    hostname: "solr"
+    container_name: 'solr'
+    hostname: 'solr'
     image: solr:${SOLR_VERSION}
     depends_on:
       solr_initializer:
         condition: service_completed_successfully
     restart: on-failure
     ports:
-      - "8983:8983"
+      - '8983:8983'
     networks:
       - dataverse
     command:
-      - "solr-precreate"
-      - "collection1"
-      - "/template"
+      - 'solr-precreate'
+      - 'collection1'
+      - '/template'
     volumes:
       - ${PWD}/solr/data:/var/solr
       - ${PWD}/solr/conf:/template
 
   smtp:
-    container_name: "smtp"
-    hostname: "smtp"
+    container_name: 'smtp'
+    hostname: 'smtp'
     image: maildev/maildev:2.0.5
     restart: on-failure
     expose:
-      - "25" # smtp server
+      - '25' # smtp server
     environment:
       - MAILDEV_SMTP_PORT=25
       - MAILDEV_MAIL_DIRECTORY=/mail
@@ -116,18 +114,18 @@ services:
       - /mail:mode=770,size=128M,uid=1000,gid=1000
 
   bootstrap:
-    container_name: "bootstrap"
-    hostname: "bootstrap"
+    container_name: 'bootstrap'
+    hostname: 'bootstrap'
     image: ${CONFIGBAKER_IMAGE}
-    restart: "no"
+    restart: 'no'
     networks:
       - dataverse
     volumes:
       - ${PWD}/dv/bootstrap.exposed.env:/.env
     command:
       - sh
       - -c
-      - "bootstrap.sh -e /.env dev"
+      - 'bootstrap.sh -e /.env dev'
     depends_on:
       dataverse:
         condition: service_healthy

docker/docker-compose-test-all.yml

@@ -1,30 +1,49 @@
-version: "2.4"
 services:
   unit-tests:
     container_name: unit-tests
     image: python:${PYTHON_VERSION}-slim
     environment:
       BASE_URL: http://dataverse:8080
-      DV_VERSION: 6.3
     networks:
       - dataverse
     volumes:
       - ${PWD}:/pydataverse
-      - ../dv:/dv
+      - ${PWD}/dv:/dv
     command:
       - sh
-      - -c
+      - -ceu
       - |
+        set -o pipefail
+
+        # Install curl
+        apt-get update && apt-get install -y curl
+
+        # Wait for Dataverse to be ready
+        echo "⏳ Waiting for Dataverse to be ready..."
+        until curl -f http://dataverse:8080/api/info/version >/dev/null 2>&1; do
+          echo "   Waiting for Dataverse API..."
+          sleep 5
+        done
+        echo "✅ Dataverse is ready!"
+
+        # Fetch Dataverse version dynamically
+        echo "🔍 Fetching Dataverse version..."
+        export DV_VERSION=$$(curl -s http://dataverse:8080/api/info/version | grep -o '"version":"[^"]*"' | cut -d'"' -f4)
+        echo "   Detected Dataverse version: $$DV_VERSION"
+
         # Fetch the API Token from the local file
-        export $(grep "API_TOKEN" "dv/bootstrap.exposed.env")
+        export $$(grep "API_TOKEN" "dv/bootstrap.exposed.env")
         export API_TOKEN_SUPERUSER=$$API_TOKEN
+
         cd /pydataverse
 
-        # Run the unit tests
+        # Install Poetry and dependencies
         python3 -m pip install --upgrade pip
-        python3 -m pip install pytest pytest-cov
-        python3 -m pip install -e .
-        python3 -m pytest > /dv/unit-tests.log
+        python3 -m pip install .
+        python3 -m pip install pytest pytest-cov pytest-asyncio tox selenium
+
+        echo "🧪 Starting pytest with DV_VERSION=${DV_VERSION}..."
+        python3 -m pytest -vv --tb=short 2>&1 | tee /dv/unit-tests.log
 
     depends_on:
       bootstrap:

local-test.env

@@ -6,4 +6,7 @@ CONFIGBAKER_IMAGE=docker.io/gdcc/configbaker:unstable
 
 # Services
 POSTGRES_VERSION=15
-SOLR_VERSION=9.3.0
+SOLR_VERSION=9.8.0
+
+# Default Dataverse version (will be dynamically detected in tests)
+DV_VERSION=6.7.1

run-tests.sh

@@ -44,27 +44,54 @@ export PYTHON_VERSION=${p}
 printf "\n🚀 Preparing containers\n"
 printf "   Using PYTHON_VERSION=${p}\n\n"
 
-# Run all containers
+# Start all containers (infrastructure + tests)
+printf "\n🚀 Starting all containers...\n"
+printf "   The test container will wait for Dataverse and fetch the version automatically\n\n"
+
 docker compose \
     -f docker/docker-compose-base.yml \
     -f ./docker/docker-compose-test-all.yml \
     --env-file local-test.env \
     up -d
 
 printf "\n🔎 Running pyDataverse tests\n"
-printf "   Logs will be printed once finished...\n\n"
+printf "   Test container will handle version detection and testing...\n\n"
 
 # Check if "unit-test" container has finished
+WAIT_COUNT=0
 while [ -n "$(docker ps -f "name=unit-tests" -f "status=running" -q)" ]; do
-    printf "   Waiting for unit-tests container to finish...\n"
+    WAIT_COUNT=$((WAIT_COUNT + 1))
+    printf "   ⏳ Waiting for tests to complete... (${WAIT_COUNT})\n"
     sleep 5
 done
 
 # Check if "unit-test" container has failed
-if [ "$(docker inspect -f '{{.State.ExitCode}}' unit-tests)" -ne 0 ]; then
-    printf "\n❌ Unit tests failed. Printing logs...\n"
-    docker logs unit-tests
-    printf "\n   Stopping containers\n"
+EXIT_CODE=$(docker inspect -f '{{.State.ExitCode}}' unit-tests 2>/dev/null)
+
+if [ -z "$EXIT_CODE" ]; then
+    printf "\n❌ Unit tests container not found or failed to start\n"
+    EXIT_CODE=1
+else
+    printf "\n📋 Unit tests completed with exit code: ${EXIT_CODE}\n"
+fi
+
+# Capture Dataverse logs after tests complete
+printf "\n📝 Capturing Dataverse logs...\n"
+docker logs dataverse > dv/dataverse-logs.log 2>&1
+
+if [ "${EXIT_CODE}" -ne 0 ]; then
+    printf "\n❌ Unit tests failed. Showing test results...\n\n"
+    printf "=== PYTEST OUTPUT ===\n"
+    if [ -f "dv/unit-tests.log" ]; then
+        cat dv/unit-tests.log
+    else
+        printf "⚠️  Test log file not found, showing container logs instead:\n"
+        docker logs unit-tests
+    fi
+    printf "\n=== END PYTEST OUTPUT ===\n"
+
+
+    printf "\n🧹 Stopping containers...\n"
     docker compose \
         -f docker/docker-compose-base.yml \
         -f ./docker/docker-compose-test-all.yml \
@@ -74,11 +101,18 @@ if [ "$(docker inspect -f '{{.State.ExitCode}}' unit-tests)" -ne 0 ]; then
 fi
 
 # Print test results
-printf "\n"
-cat dv/unit-tests.log
-printf "\n\n✅ Unit tests passed\n\n"
+printf "\n✅ Unit tests passed! Showing results...\n\n"
+printf "=== PYTEST RESULTS ===\n"
+if [ -f "dv/unit-tests.log" ]; then
+    cat dv/unit-tests.log
+else
+    printf "⚠️  Test log file not found, showing container logs:\n"
+    docker logs unit-tests
+fi
+printf "\n=== END PYTEST RESULTS ===\n"
 
 # Stop all containers
+printf "\n🧹 Stopping containers...\n"
 docker compose \
     -f docker/docker-compose-base.yml \
     -f ./docker/docker-compose-test-all.yml \

tests/api/test_api.py

@@ -174,14 +174,22 @@ def test_token_right_create_dataset_rights(self):
                 )
             )
             resp = api_su.create_dataset(":root", ds.json())
-            pid = resp.json()["data"]["persistentId"]
-            assert resp.json()["status"] == "OK"
+            resp.raise_for_status()
+
+            resp = resp.json()
+            pid = resp["data"]["persistentId"]
+
+            if resp["status"] != "OK":
+                raise Exception(resp["message"])
 
             # with pytest.raises(ApiAuthorizationError):
             #     resp = api_nru.get_dataset(pid)
 
             resp = api_su.delete_dataset(pid)
-            assert resp.json()["status"] == "OK"
+            resp.raise_for_status()
+            resp = resp.json()
+            if resp["status"] != "OK":
+                raise Exception(resp["message"])
 
         def test_token_should_not_be_exposed_on_error(self):
             BASE_URL = os.getenv("BASE_URL")

tests/api/test_upload.py

@@ -161,6 +161,8 @@ def test_file_replacement_wo_metadata(self):
             json_str=df.json(),
         )
 
+        response.raise_for_status()
+
         # Retrieve file ID
         file_id = response.json()["data"]["files"][0]["dataFile"]["id"]
 
@@ -182,6 +184,9 @@ def test_file_replacement_wo_metadata(self):
                 is_filepid=False,
             )
 
+            if not response.is_success:
+                raise Exception(response.json()["message"])
+
         # Assert
         file_id = response.json()["data"]["files"][0]["dataFile"]["id"]
         content = data_api.get_datafile(file_id, is_pid=False).text
@@ -218,6 +223,9 @@ def test_file_replacement_w_metadata(self):
             json_str=df.json(),
         )
 
+        if not response.is_success:
+            raise Exception(response.json()["message"])
+
         # Retrieve file ID
         file_id = response.json()["data"]["files"][0]["dataFile"]["id"]
 
@@ -244,6 +252,9 @@ def test_file_replacement_w_metadata(self):
                 is_filepid=False,
             )
 
+            if not response.is_success:
+                raise Exception(response.json()["message"])
+
         # Assert
         file_id = response.json()["data"]["files"][0]["dataFile"]["id"]
         data_file = api.get_dataset(pid).json()["data"]["latestVersion"]["files"][0]