diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1f162ec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,96 @@ +# mrf/Dockerfile + +# ===================================================================== +# Stage 1: Install all development tools, compile the C++ utilities, +# and create the Python virtual environment. +# ===================================================================== +FROM almalinux:9 AS builder + +# Build Arguments for the el9 GDAL RPM +ARG GDAL_VERSION=3.6.4 +ARG GIBS_GDAL_RELEASE=1 # +ARG ALMALINUX_VERSION=9 # + +# Install Build-time Dependencies +RUN dnf install -y epel-release dnf-plugins-core && \ + dnf config-manager --set-enabled crb && \ + dnf groupinstall -y "Development Tools" && \ + dnf install -y --allowerasing \ + cmake \ + git \ + python3-pip \ + python3-devel \ + libtiff-devel \ + sqlite-devel \ + wget \ + curl \ + geos \ + proj && \ + dnf clean all + +# Install Pre-compiled GIBS GDAL for el9 +RUN wget -P /tmp/ https://github.com/nasa-gibs/gibs-gdal/releases/download/v${GDAL_VERSION}/gibs-gdal-${GDAL_VERSION}-${GIBS_GDAL_RELEASE}.el${ALMALINUX_VERSION}.x86_64.rpm && \ + dnf install -y /tmp/gibs-gdal-${GDAL_VERSION}-${GIBS_GDAL_RELEASE}.el${ALMALINUX_VERSION}.x86_64.rpm && \ + rm -rf /tmp/* + +# Download the missing private marfa.h header +RUN curl -L "https://raw.githubusercontent.com/OSGeo/gdal/v${GDAL_VERSION}/frmts/mrf/marfa.h" -o /usr/local/include/marfa.h + +WORKDIR /app +COPY requirements.txt . +# Create the venv and install packages +RUN python3 -m venv /app/venv +ENV PATH="/app/venv/bin:$PATH" +RUN pip install --no-cache-dir -r requirements.txt +# Install the Python bindings for the installed GDAL version +RUN pip install GDAL==$(gdal-config --version) + +# Copy the rest of the project files +COPY . . + +# Build the C++ utilities +RUN cd mrf_apps && make + +# Install the project itself into the venv +RUN pip install -e . + +# ===================================================================== +# Stage 2: Minimal, distributable image. +# ===================================================================== +FROM almalinux:9 + +# Install only Runtime Dependencies +RUN dnf install -y epel-release dnf-plugins-core && \ + dnf config-manager --set-enabled crb && \ + dnf install -y --allowerasing python3 wget geos proj && \ + dnf clean all + +# Install the el9 GDAL RPM for its runtime libraries +ARG GDAL_VERSION=3.6.4 +ARG GIBS_GDAL_RELEASE=1 +ARG ALMALINUX_VERSION=9 +RUN wget -P /tmp/ https://github.com/nasa-gibs/gibs-gdal/releases/download/v${GDAL_VERSION}/gibs-gdal-${GDAL_VERSION}-${GIBS_GDAL_RELEASE}.el${ALMALINUX_VERSION}.x86_64.rpm && \ + dnf install -y /tmp/gibs-gdal-${GDAL_VERSION}-${GIBS_GDAL_RELEASE}.el${ALMALINUX_VERSION}.x86_64.rpm && \ + rm -rf /tmp/* + +# Tell the linker where to find the new libraries +# Create a new configuration file for the dynamic linker +RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/gdal-custom.conf + +# Update the shared library cache +RUN ldconfig + +WORKDIR /app + +# Copy Artifacts from the "builder" Stage +COPY --from=builder /app/mrf_apps/can /usr/local/bin/ +COPY --from=builder /app/mrf_apps/jxl /usr/local/bin/ +COPY --from=builder /app/mrf_apps/mrf_insert /usr/local/bin/ +COPY --from=builder /app/venv /app/venv +COPY mrf_apps/ ./mrf_apps/ +COPY pyproject.toml . +COPY README.md . + +# Set Final Environment Variables +ENV PATH="/app/venv/bin:$PATH" +ENV GDAL_DATA="/usr/local/share/gdal" diff --git a/MRF_Utilities_Test_Suite.md b/MRF_Utilities_Test_Suite.md new file mode 100644 index 0000000..7ec95c5 --- /dev/null +++ b/MRF_Utilities_Test_Suite.md @@ -0,0 +1,145 @@ +## **MRF Utilities Test Suite** + +This document outlines the unit tests for the Meta Raster Format (MRF) utilities. The tests are written in Python using the `unittest` framework and are designed to be run with a test runner like `pytest`. The suite is structured into separate files for each utility to ensure maintainability and clarity. + +A shared test helper, `tests/helpers.py`, provides a base class that handles the setup and teardown of a temporary testing directory and includes methods for creating mock MRF files (`.mrf`, `.idx`, `.dat`). This approach minimizes code duplication and standardizes test environments. + + +### Docker-Based Testing Environment + +Using Docker is the recommended method for running this test suite. It creates an environment with all the necessary C++, GDAL, and Python dependencies pre-installed, resolving any platform-specific issues and ensuring the tests run in this isolated environment. This workflow uses a two stage building approach: first creating a base application image, and then building a lightweight test runner image from it. + +#### Prerequisites + +Ensure Docker installed and running on your system. + +#### Building and Running the Tests + +**Step 1: Build the Base Application Image** +Navigate to the project's root directory and run the following command. This builds the main application image, compiling all C++ utilities and installing dependencies. It is tagged as `mrf-app:latest`. + +```bash +docker build --platform linux/amd64 -t mrf-app:latest -f Dockerfile . +``` + +> **Note**: The `--platform linux/amd64` flag is required if you are building on an ARM-based machine (like an Apple Silicon Mac) to ensure compatibility with the pre-compiled `x86_64` GDAL RPM used in the build. + +**Step 2: Build the Test Suite Image** +Next, build the dedicated test runner image. This build uses the `mrf-app` image from the previous step as its base. + +```bash +docker build --platform linux/amd64 -t mrf-test-suite -f tests/Dockerfile . +``` + +**Step 3: Run the Test Suite** +Finally, run the tests using the `mrf-test-suite` image. This command starts a container, executes `pytest`, and automatically removes the container (`--rm`) when finished. + +```bash +docker run --rm mrf-test-suite +``` + +You should see output from `pytest`, culminating in a summary showing tests passing or failing or skipping. + + +### `can` Utility Tests + +**File**: `tests/test_can.py` + +These tests validate the `can` C++ command-line utility, which is used for compressing and decompressing sparse MRF index files. + + * **`test_can_uncan_cycle`**: Verifies the round-trip integrity of the canning process. It creates a large, sparse mock index file (`.idx`), runs `can` to compress it to a canned index (`.ix`), and then runs it with the `-u` flag to decompress it back to an `.idx` file. The test passes if the final index file is identical to the original. + + +### `jxl` Utility Tests + +**File**: `tests/test_jxl.py` + +These tests validate the `jxl` C++ utility, which converts MRF data files and single images between JPEG (JFIF) and JPEG XL (Brunsli) formats. + + * **`test_jxl_mrf_round_trip`**: Verifies the primary MRF conversion. It converts a mock MRF data file (`.pjg`) and its index to JXL format and then back to JPEG, confirming the final files are identical to the originals and that the JXL file is smaller. + * **`test_jxl_single_file_round_trip`**: Validates the single-file mode (`-s`). It performs a round-trip conversion on a standalone JPEG file and confirms data integrity. + * **`test_jxl_bundle_mode` (Placeholder)**: A placeholder test for Esri bundle mode (`-b`) that is skipped, as creating a valid mock bundle file is non-trivial. + + +### `mrf_clean.py` Tests + +**File**: `tests/test_clean.py` + +These tests validate `mrf_clean.py`, a script used to optimize MRF storage by removing unused space. + + * **`test_mrf_clean_copy`**: Checks the default "copy" mode. It verifies that the script creates a new, smaller data file with slack space removed and that the new index file has correctly updated, contiguous tile offsets. + * **`test_mrf_clean_trim`**: Validates the in-place "trim" mode. It confirms that the original data file is truncated to the correct size and its index file is overwritten with updated offsets. + + +### `mrf_insert` Utility Tests + +**File**: `tests/test_mrf_insert.py` + +These tests validate the `mrf_insert` C++ utility, which is used to patch a smaller raster into a larger MRF. + + * **`test_mrf_insert_simple_patch`**: Validates the core functionality. It creates an empty target MRF and a smaller source raster, executes `mrf_insert`, and uses GDAL to verify the patched region was written correctly while unpatched regions remain unaffected. + * **`test_mrf_insert_with_overviews`**: Tests that inserting a patch with the `-r` flag correctly regenerates the affected overview tiles. + * **`test_mrf_insert_partial_tile_overlap`**: Confirms that inserting a source that only partially covers a target tile correctly merges the new data while preserving the uncovered portions of the original tile. + + +### `mrf_join.py` Tests + +**File**: `tests/test_join.py` + +These tests validate `mrf_join.py`, a script that merges or appends multiple MRF files. + + * **`test_mrf_join_simple_merge`**: Checks the script's ability to merge two sparse MRFs, verifying that the final data file is a concatenation of inputs and the final index correctly combines entries with updated offsets. + * **`test_mrf_join_overwrite`**: Confirms the "last-one-wins" logic by joining two MRFs that provide data for the same tile and verifying that the final index points to the data from the last-processed input. + * **`test_mrf_append_z_dimension`**: Validates the ability to stack 2D MRFs into a single 3D MRF, checking that the Z dimension is correctly set in the metadata and that the index layout is correct for multiple slices. + * **`test_mrf_append_with_overviews`**: Tests the scenario of appending MRFs that contain overviews, ensuring the final interleaved index structure is correctly assembled. + +### `mrf_read_data.py` Tests + +**File**: `tests/test_read_data.py` + +These tests validate `mrf_read_data.py`, which extracts a specific tile or data segment from an MRF data file. + + * **`test_read_with_offset_and_size`**: Validates the direct read mode by using `--offset` and `--size` to extract a specific data segment and confirming the output is correct. + * **`test_read_with_index_and_tile`**: Validates the index-based read mode by using `--index` and `--tile` to retrieve a specific tile and verifying its content. + * **`test_read_with_little_endian_index`**: Ensures the `--little-endian` flag functions correctly by reading from an index file with a different byte order. + + +### `mrf_read_idx.py` Tests + +**File**: `tests/test_read_idx.py` + +These tests validate `mrf_read_idx.py`, which converts a binary MRF index file into a CSV. + + * **`test_read_simple_index`**: Validates the script's core functionality with a standard, big-endian index file, verifying the output CSV has the correct headers and data. + * **`test_read_little_endian_index`**: Confirms that the `--little-endian` flag works by parsing an index with a different byte order and checking for correctly interpreted values. + * **`test_read_empty_index`**: Handles the edge case of an empty input file, ensuring the script produces a CSV with only the header row. + + +### `mrf_size.py` Tests + +**File**: `tests/test_mrf_size.py` + +These tests validate `mrf_size.py`, which generates a GDAL VRT to visualize the tile sizes from an MRF index. + + * **`test_vrt_creation_single_band`**: Checks VRT generation for a single-band MRF, verifying the VRT's dimensions, GeoTransform, and raw band parameters. + * **`test_vrt_creation_multi_band`**: Validates handling of multi-band MRFs, ensuring the VRT contains the correct number of bands with correctly calculated offsets. + * **`test_vrt_default_pagesize`**: Ensures the script correctly applies a default 512x512 page size when it's not specified in the MRF metadata. + + +### `tiles2mrf.py` Tests + +**File**: `tests/test_tiles2mrf.py` + +These tests validate `tiles2mrf.py`, which assembles an MRF from a directory of individual tiles. + + * **`test_simple_conversion`**: Validates basic functionality by assembling a 2x2 grid of tiles and verifying the concatenated data file and sequential index offsets. + * **`test_with_overviews_and_padding`**: Checks the creation of a multi-level pyramid, ensuring the script correctly processes all levels and adds necessary padding records to the index. + * **`test_blank_tile_handling`**: Validates the `--blank-tile` feature, confirming that blank tiles are omitted from the data file and are represented by a zero-record in the index. + + +### Conditional Test Skipping + +The test suite is designed to be run primarily within the provided Docker container, where all dependencies are guaranteed to be met. However, the tests include conditional skipping logic to fail gracefully if run in a local environment that is not fully configured. + + * **C++ Executable Tests**: The tests for **`can`**, **`jxl`**, and **`mrf_insert`** will be skipped if their respective compiled executables are not found in the system's PATH. + * **GDAL Python Dependency**: The test for `mrf_insert` requires the GDAL Python bindings to create test files. It will be skipped if the `osgeo.gdal` library cannot be imported. diff --git a/Makefile.lcl b/Makefile.lcl new file mode 100644 index 0000000..f332325 --- /dev/null +++ b/Makefile.lcl @@ -0,0 +1,12 @@ +# Makefile.lcl +# Local configuration file that provides the system-specific paths needed to compile the C++ utilities. + +# Set the base directory where the GDAL RPM installed its files. +PREFIX=/usr/local + +# Set the root for GDAL headers, as expected by the mrf_apps/Makefile. +# This points to the same location as PREFIX/include but satisfies the variable requirement. +GDAL_ROOT=/usr/local/include + +# Override the library directory to point to lib64 +LIBDIR = $(PREFIX)/lib64 diff --git a/mrf_apps/Makefile b/mrf_apps/Makefile index 5dccfda..a63cf60 100644 --- a/mrf_apps/Makefile +++ b/mrf_apps/Makefile @@ -4,7 +4,7 @@ # PREFIX=/home/ec2-user # GDAL_ROOT=$(PREFIX)/src/gdal/gdal # -include Makefile.lcl +include ../Makefile.lcl TARGETS = can mrf_insert jxl GDAL_INCLUDE = -I $(PREFIX)/include -I $(GDAL_ROOT) diff --git a/mrf_apps/mrf_join.py b/mrf_apps/mrf_join.py index 393a6de..da2bbb1 100755 --- a/mrf_apps/mrf_join.py +++ b/mrf_apps/mrf_join.py @@ -205,7 +205,8 @@ def mrf_append(inputs, output, outsize, startidx = 0): assert os.path.splitext(f)[1] == ext,\ "All input files should have the same extension as the output" # Get the template mrf information from the first input - mrfinfo, tree = getmrfinfo(os.path.splitext(inputs[1])[0] + ".mrf", ofname + ".mrf") + # Use the first input file (inputs[0]) as template for the output MRF + mrfinfo, tree = getmrfinfo(os.path.splitext(inputs[0])[0] + ".mrf") # Create the output .mrf if it doesn't exist if not os.path.isfile(ofname + ".mrf"): diff --git a/mrf_apps/mrf_size.py b/mrf_apps/mrf_size.py index 4786d55..ca41919 100755 --- a/mrf_apps/mrf_size.py +++ b/mrf_apps/mrf_size.py @@ -121,7 +121,7 @@ def VRT_Size(mrf): gt[1] *= mrf.pagesize.x gt[5] *= mrf.pagesize.y XML.SubElement(root,'GeoTransform').text = ",".join((str(x) for x in gt)) - bands = int(mrf.size.c / mrf.pagesize.c) + bands = int(mrf.size.c) for band in range(bands): xband = XML.SubElement(root, 'VRTRasterBand', { 'band':str(band+1), diff --git a/mrf_apps/tiles2mrf.py b/mrf_apps/tiles2mrf.py index 5b4de30..ec58806 100755 --- a/mrf_apps/tiles2mrf.py +++ b/mrf_apps/tiles2mrf.py @@ -40,8 +40,8 @@ def option_error(parser, msg): sys.exit(1) def half(val): - 'Divide by two with roundup, returns at least 1' - return 1 + (val - 1 )/2 + 'Divide by two with roundup, returns integer value at least 1' + return 1 + (val - 1 ) // 2 def hash_tile(tile): h = hashlib.sha256() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..28b2bfa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "mrf_utilities" +version = "0.1.0" + +[tool.setuptools] +packages = ["mrf_apps", "tests"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..767288d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +#requirements.txt +pytest +numpy +Pillow diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 0000000..3e22908 --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,16 @@ +# mrf/tests/Dockerfile +# This file builds the test runner image. + +# Start from application image mrf/Dockerfile +# Need to make sure the tag in the build step is matching. +FROM mrf-app:latest + +# The WORKDIR and ENV variables are inherited from the base image. + +# Copy the test directory into the image. +# This assumes the `docker build` command was run from the project root. +COPY tests/ ./tests/ + +# The source code and tests were copied in the base image, +# so we just defining the command to run the tests. +CMD ["pytest"] diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..e673cca --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,70 @@ +# tests/helpers.py + +import unittest +import os +import shutil +import struct +from xml.etree import ElementTree as ET +from PIL import Image + +class MRFTestCase(unittest.TestCase): + """ + A base class for MRF utility tests that handles temporary directory + creation and provides helper methods for creating mock MRF files. + """ + def setUp(self): + """Set up a temporary directory for test files.""" + self.test_dir = "mrf_test_temp_dir" + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + os.makedirs(self.test_dir) + + # Assume C++ utilities are compiled and in the system PATH + self.can_executable = "can" + self.mrf_insert_executable = "mrf_insert" + + def tearDown(self): + """Clean up the temporary directory.""" + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + def create_mock_mrf_xml(self, path, xsize=512, ysize=512, channels=1, pagesize=512, data_ext="dat"): + """Creates a minimal MRF metadata file.""" + root = ET.Element("MRF_META") + raster = ET.SubElement(root, "Raster") + ET.SubElement(raster, "Size", x=str(xsize), y=str(ysize), c=str(channels)) + ET.SubElement(raster, "PageSize", x=str(pagesize), y=str(pagesize), c=str(channels)) + data_file = ET.SubElement(raster, "DataFile") + base_name = os.path.basename(path).replace('.mrf', '') + data_file.text = f"{base_name}.{data_ext}" + + tree = ET.ElementTree(root) + tree.write(path) + + def create_mock_idx(self, path, tiles): + """Creates a mock index file from a list of (offset, size) tuples.""" + with open(path, "wb") as f: + for offset, size in tiles: + f.write(struct.pack('>QQ', offset, size)) + + def create_mock_data(self, path, content_list): + """Creates a mock data file from a list of byte strings.""" + with open(path, "wb") as f: + for content in content_list: + f.write(content) + + def read_idx_file(self, path): + """Reads an index file and returns a list of (offset, size) tuples.""" + tiles = [] + with open(path, 'rb') as f: + while True: + chunk = f.read(16) + if not chunk: + break + tiles.append(struct.unpack('>QQ', chunk)) + return tiles + + def create_mock_jpeg(self, path, size=(16, 16), color='black'): + """Creates a simple, valid JPEG file using Pillow.""" + with Image.new('RGB', size, color) as img: + img.save(path, 'jpeg') diff --git a/tests/test_can.py b/tests/test_can.py new file mode 100644 index 0000000..a4c7234 --- /dev/null +++ b/tests/test_can.py @@ -0,0 +1,33 @@ +# tests/test_can.py + +import os +import shutil +import subprocess +import struct +import filecmp +from tests.helpers import MRFTestCase + +class TestCanUtility(MRFTestCase): + def test_can_uncan_cycle(self): + """Test that canning and uncanning an index file restores it perfectly.""" + if not shutil.which(self.can_executable): + self.skipTest(f"'{self.can_executable}' executable not found in PATH.") + + idx_path = os.path.join(self.test_dir, "test.idx") + can_path = os.path.join(self.test_dir, "test.ix") + out_idx_path = os.path.join(self.test_dir, "test.out.idx") + + # Create a sparse index with data in the first and last blocks + empty_block = b'\x00' * 512 + data_block = struct.pack('>QQ', 123, 456) * (512 // 16) + with open(idx_path, 'wb') as f: + f.write(data_block) + for _ in range(94): + f.write(empty_block) + f.write(data_block) + + subprocess.run([self.can_executable, "-g", idx_path, can_path], check=True) + self.assertLess(os.path.getsize(can_path), os.path.getsize(idx_path)) + + subprocess.run([self.can_executable, "-u", "-g", can_path, out_idx_path], check=True) + self.assertTrue(filecmp.cmp(idx_path, out_idx_path, shallow=False)) diff --git a/tests/test_clean.py b/tests/test_clean.py new file mode 100644 index 0000000..e05df10 --- /dev/null +++ b/tests/test_clean.py @@ -0,0 +1,41 @@ +# tests/test_clean.py + +import os +from tests.helpers import MRFTestCase +from mrf_apps import mrf_clean + +class TestMRFClean(MRFTestCase): + def test_mrf_clean_copy(self): + """Test the 'copy' mode of mrf_clean.py.""" + source_base = os.path.join(self.test_dir, "source") + dest_base = os.path.join(self.test_dir, "dest") + + tile1, tile2 = b'\x01' * 10, b'\x02' * 20 + self.create_mock_data(source_base + ".dat", [tile1, b'\x00' * 5, tile2]) + self.create_mock_idx(source_base + ".idx", [(0, 10), (15, 20)]) + + mrf_clean.mrf_clean(source_base + ".dat", dest_base + ".dat") + + self.assertEqual(os.path.getsize(dest_base + ".dat"), 30) + with open(dest_base + ".dat", "rb") as f: + self.assertEqual(f.read(), tile1 + tile2) + + new_idx = self.read_idx_file(dest_base + ".idx") + self.assertEqual(new_idx, [(0, 10), (10, 20)]) + + def test_mrf_clean_trim(self): + """Test the 'trim' in-place mode of mrf_clean.py.""" + source_base = os.path.join(self.test_dir, "source") + + self.create_mock_data(source_base + ".dat", [b'\x01' * 10, b'\x00' * 5, b'\x02' * 20]) + self.create_mock_idx(source_base + ".idx", [(0, 10), (15, 20)]) + + class Args: + source = source_base + ".dat" + empty_file = 0 + + mrf_clean.mrf_trim(Args()) + + self.assertEqual(os.path.getsize(source_base + ".dat"), 30) + new_idx = self.read_idx_file(source_base + ".idx") + self.assertEqual(new_idx, [(0, 10), (10, 20)]) diff --git a/tests/test_join.md b/tests/test_join.md new file mode 100644 index 0000000..b016430 --- /dev/null +++ b/tests/test_join.md @@ -0,0 +1,63 @@ +# Test Suite: `mrf_join.py` + +This is documentation of the test suite for the `mrf_join.py` utility. The purpose of this suite is to provide comprehensive testing for the script's two primary modes of operation: +1. **Join Mode:** Merging multiple 2D MRFs into a single composite 2D MRF. +2. **Append Mode:** Stacking multiple 2D MRFs to create a single 3D MRF with multiple Z-slices. + +The tests are designed to verify data integrity, the correctness of complex index file manipulations, and proper metadata updates. + +--- + +## Test Setup and Helpers + +All tests in this suite inherit from `MRFTestCase`, a base class defined in `tests/helpers.py`. This class provides: +* Automatic creation and cleanup of a temporary directory for each test, ensuring **test isolation**. +* Helper functions (`create_mock_mrf_xml`, `create_mock_data`, `create_mock_idx`) for generating the necessary MRF component files for each test scenario. + +The tests directly import and call the functions from the `mrf_join.py` script, allowing for fast and focused unit testing without relying on command-line subprocesses. + +--- + +## Test Cases + +### `test_mrf_join_simple_merge()` + +* **Purpose:** This test validates the behavior of the **join mode**, where two MRFs containing data for different tiles are merged into one. +* **Scenario:** + 1. An MRF (`in1.mrf`) is created with space for two tiles, but only the **first** tile contains data. + 2. A second MRF (`in2.mrf`) is created with the same structure, but only the **second** tile contains data. +* **Assertions:** + 1. It verifies that the output data file (`out.dat`) contains the simple concatenation of the tile data from both inputs. + 2. It asserts that the output index file (`out.idx`) has been correctly merged, with the tile from `in1` at the beginning and the tile from `in2` following it, with its data offset correctly recalculated. + +### `test_mrf_join_overwrite()` + +* **Purpose:** This test validates the "last-in-wins" overwrite logic of the **join mode**. +* **Scenario:** + 1. An MRF (`in1_overwrite.mrf`) is created with data for a single tile (version A). + 2. A second MRF (`in2_overwrite.mrf`) is created with different data for the **same** tile (version B). +* **Assertions:** + 1. It verifies that the output data file contains the data from both inputs concatenated. The data from the first input becomes inaccessible "slack space." + 2. It asserts that the final index record correctly points to the data from the **second** input (`in2_overwrite.mrf`), confirming that it overwrote the entry from the first input. + +### `test_mrf_append_z_dimension()` + +* **Purpose:** This test validates the core functionality of the **append mode**—stacking 2D MRFs into a 3D MRF. +* **Scenario:** + 1. Two simple 2D MRFs (`in_z1.mrf` and `in_z2.mrf`) are created, each containing a single tile. + 2. The `mrf_append` function is called to stack these into a new 3D MRF with a Z-size of 2. +* **Assertions:** + 1. It verifies that the output data file contains the concatenated tile data from both inputs. + 2. It asserts that the output index file contains two records, one for each Z-slice, with correctly calculated offsets. + 3. It parses the output metadata file (`out_3d.mrf`) and confirms that the `` element has been correctly updated with the `z="2"` attribute. + +### `test_mrf_append_with_overviews()` + +* **Purpose:** This test is designed to validate the **append mode**'s handling of MRFs that contain overviews (pyramids). It verifies the script's ability to correctly calculate the complex interleaved index structure for a 3D MRF with multiple levels. +* **Scenario:** + 1. Two identical MRFs are created, each having a structure that produces one overview level (e.g., a base resolution of 2x1 tiles and an overview of 1x1 tiles). + 2. The `mrf_append` function is called to stack these into a 2-slice, 3D MRF. +* **Assertions:** + 1. It verifies that the final index file has the correct total number of records (2 slices * 3 records/slice = 6 records). + 2. It asserts that the records are interleaved in the correct order as required by the MRF specification for 3D pyramids: **[L0S0T0, L0S0T1, L0S1T0, L0S1T1, L1S0T0, L1S1T0]**, where `L` is level, `S` is slice, and `T` is tile. + 3. It confirms that the data offsets for each record have been correctly recalculated to point to the right location in the final concatenated data file. diff --git a/tests/test_join.py b/tests/test_join.py new file mode 100644 index 0000000..75a3d71 --- /dev/null +++ b/tests/test_join.py @@ -0,0 +1,208 @@ +# tests/test_join.py + +import os +import shutil +from xml.etree import ElementTree as ET +from tests.helpers import MRFTestCase +from mrf_apps import mrf_join # Import the script to test its functions directly + +class TestMRFJoin(MRFTestCase): + + def setUp(self): + """Set up test environment by calling parent setUp.""" + super().setUp() + # Mock class to simulate argparse Namespace object for direct function calls + class MockArgs: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + self.MockArgs = MockArgs + + # Helper function to create MRFs for append tests + def create_append_input(self, base_name, tile_data, index_layout, xsize=512): + """Helper to create a single MRF for append tests.""" + self.create_mock_mrf_xml(base_name + ".mrf", xsize=xsize) + self.create_mock_data(base_name + ".dat", tile_data) + self.create_mock_idx(base_name + ".idx", index_layout) + + def test_mrf_join_simple_merge(self): + """Test simple merge where inputs provide data for different tiles.""" + input1_base = os.path.join(self.test_dir, "in1") + input2_base = os.path.join(self.test_dir, "in2") + output_base = os.path.join(self.test_dir, "out") + + # Create two simple MRFs for a 2-tile image (xsize=1024, pagesize=512) + # Input 1 provides data for the FIRST tile and leaves the second empty. + tile1_data = [b'A' * 10] + tile1_index = [(0, 10), (0, 0)] # Tile 1 has data, Tile 2 is empty + self.create_mock_mrf_xml(input1_base + ".mrf", xsize=1024) + self.create_mock_data(input1_base + ".dat", tile1_data) + self.create_mock_idx(input1_base + ".idx", tile1_index) + + # Input 2 provides data for the SECOND tile and leaves the first empty. + tile2_data = [b'B' * 20] + tile2_index = [(0, 0), (0, 20)] # Tile 1 is empty, Tile 2 has data + self.create_mock_mrf_xml(input2_base + ".mrf", xsize=1024) + self.create_mock_data(input2_base + ".dat", tile2_data) + self.create_mock_idx(input2_base + ".idx", tile2_index) + + # Run the join script in default mode + argv = [input1_base + ".dat", input2_base + ".dat", output_base + ".dat"] + mrf_join.mrf_join(argv) + + # Verify the results + # 1. Data file should contain the concatenated tile data from both inputs + with open(output_base + ".dat", "rb") as f: + self.assertEqual(f.read(), b'A' * 10 + b'B' * 20) + + # 2. Final index should merge tile info from both inputs, updating offsets + final_idx = self.read_idx_file(output_base + ".idx") + # Tile 1 from input1: (offset=0, size=10) + # Tile 2 from input2: offset_of_input2_data + tile2_offset = 10 + 0 = 10 + self.assertEqual(final_idx, [(0, 10), (10, 20)]) + + def test_mrf_join_overwrite(self): + """Test that later inputs overwrite tiles from earlier inputs.""" + input1_base = os.path.join(self.test_dir, "in1_overwrite") + input2_base = os.path.join(self.test_dir, "in2_overwrite") + output_base = os.path.join(self.test_dir, "out_overwrite") + + # Input 1 has data for tile 1: version 'A' + tile_v1_data = [b'A_v1' * 10] + tile_v1_index = [(0, 6)] + self.create_mock_mrf_xml(input1_base + ".mrf", xsize=512) + self.create_mock_data(input1_base + ".dat", tile_v1_data) + self.create_mock_idx(input1_base + ".idx", tile_v1_index) + + # Input 2 also has data for tile 1: version 'B' + tile_v2_data = [b'B_v2' * 20] + tile_v2_index = [(0, 6)] + self.create_mock_mrf_xml(input2_base + ".mrf", xsize=512) + self.create_mock_data(input2_base + ".dat", tile_v2_data) + self.create_mock_idx(input2_base + ".idx", tile_v2_index) + + # Run the join script in default mode + argv = [input1_base + ".dat", input2_base + ".dat", output_base + ".dat"] + mrf_join.mrf_join(argv) + + # Verify the results + # 1. Data file contains data from both inputs. The data from input1 is now slack space. + with open(output_base + ".dat", "rb") as f: + self.assertEqual(f.read(), tile_v1_data[0] + tile_v2_data[0]) + + # 2. Final index should point to the data from input2 (the last one processed) + final_idx = self.read_idx_file(output_base + ".idx") + # Offset calculation: len(input1_data) + offset_from_input2 = len(b'A_v1' * 10) + 0 = 6 + expected_offset = len(tile_v1_data[0]) + self.assertEqual(final_idx, [(expected_offset, 6)]) + + def test_mrf_append_z_dimension(self): + """Test stacking two 2D MRFs into one 3D MRF using append mode.""" + input1_base = os.path.join(self.test_dir, "in_z1") + input2_base = os.path.join(self.test_dir, "in_z2") + output_data_path = os.path.join(self.test_dir, "out_3d.dat") + + # Create input 1 (for slice 0) + self.create_append_input(input1_base, [b'Slice1Data'], [(0, 10)]) + + # Create input 2 (for slice 1) + self.create_append_input(input2_base, [b'Slice2Data'], [(0, 10)]) + + # Run mrf_append via main function simulation + args = self.MockArgs( + zsize=2, + output=output_data_path, + forceoffset=None, + slice=0, + fnames=[input1_base + ".dat", input2_base + ".dat"] + ) + mrf_join.mrf_append(args.fnames, args.output, args.zsize, args.slice) + + # Verify results + # 1. Data file check + with open(output_data_path, "rb") as f: + self.assertEqual(f.read(), b'Slice1Data' + b'Slice2Data') + + # 2. Index check: The index for a 2-slice MRF with 1 tile per slice + # should contain two records. + final_idx = self.read_idx_file(os.path.join(self.test_dir, "out_3d.idx")) + self.assertEqual(len(final_idx), 2) + # Slice 0 (from input 1) offset = 0. Data size = 10. + self.assertEqual(final_idx[0], (0, 10)) + # Slice 1 (from input 2) offset = len(Slice1Data) = 10. Data size = 10. + self.assertEqual(final_idx[1], (10, 10)) + + # 3. Metadata check: Verify the zsize attribute was added to the MRF XML. + tree = ET.parse(os.path.join(self.test_dir, "out_3d.mrf")) + size_node = tree.find("./Raster/Size") + self.assertEqual(size_node.get('z'), '2') + + def test_mrf_append_with_overviews(self): + """Test stacking MRFs that contain overviews to verify complex index layout.""" + # Setup: Create MRFs with overviews. + # Image size: 1024x512 pixels. Tile size: 512x512 pixels. + # Level 0 (base resolution) dimensions: 2x1 tiles = 2 tiles. + # Overview Level 1 dimensions: ceil(2/2)xceil(1/2) = 1x1 tiles = 1 tile. + # Total index records per slice = 2 + 1 = 3. + + input1_base = os.path.join(self.test_dir, "in_ov1") + input2_base = os.path.join(self.test_dir, "in_ov2") + output_data_path = os.path.join(self.test_dir, "out_3d_ov.dat") + + # Input 1 data: [L0T0, L0T1, L1T0] data. Offsets are 0, 10, 20. Sizes are 10, 10, 5. + data1 = [b'A' * 10, b'B' * 10, b'C' * 5] + index1 = [(0, 10), (10, 10), (20, 5)] + self.create_append_input(input1_base, data1, index1, xsize=1024) + + # Input 2 data: [L0T0, L0T1, L1T0] data. Offsets are 0, 8, 16. Sizes are 8, 8, 4. + data2 = [b'X' * 8, b'Y' * 8, b'Z' * 4] + index2 = [(0, 8), (8, 8), (16, 4)] + self.create_append_input(input2_base, data2, index2, xsize=1024) + + # Run append for 2 slices + args = self.MockArgs( + zsize=2, + output=output_data_path, + forceoffset=None, + slice=0, + fnames=[input1_base + ".dat", input2_base + ".dat"] + ) + # Need to mock getmrfinfo to return correct overview structure + # mrfinfo['pages'] = [pagecount_level_0, pagecount_level_1] = [2, 1] + original_getmrfinfo = mrf_join.getmrfinfo + def mock_getmrfinfo(fname, *args_): + info, tree = original_getmrfinfo(fname) + info['pages'] = [2, 1] # Override page count calculation: [L0 count, L1 count] + info['totalpages'] = 3 + return info, tree + mrf_join.getmrfinfo = mock_getmrfinfo + + mrf_join.mrf_append(args.fnames, args.output, args.zsize, args.slice) + + # Restore original function + mrf_join.getmrfinfo = original_getmrfinfo + + # ASSERT: Verify interleaved index structure for 2 slices, 3 records per slice. + final_idx = self.read_idx_file(os.path.join(self.test_dir, "out_3d_ov.idx")) + self.assertEqual(len(final_idx), 6) # 2 slices * 3 records/slice + + # Expected index layout trace: [L0S0T0, L0S0T1, L0S1T0, L0S1T1, L1S0T0, L1S1T0] + # Data offsets: Input 1 starts at 0. Input 2 starts at len(data1) = 10+10+5 = 25. + + # Slice 0 records (from input 1) with offsets relative to 0: + expected_s0_l0t0 = index1[0] # (0, 10) + expected_s0_l0t1 = index1[1] # (10, 10) + expected_s0_l1t0 = index1[2] # (20, 5) + + # Slice 1 records (from input 2) with offsets relative to 25: + data2_start_offset = sum(len(d) for d in data1) # 25 + expected_s1_l0t0 = (index2[0][0] + data2_start_offset, index2[0][1]) # (25, 8) + expected_s1_l0t1 = (index2[1][0] + data2_start_offset, index2[1][1]) # (33, 8) + expected_s1_l1t0 = (index2[2][0] + data2_start_offset, index2[2][1]) # (41, 4) + + # Verify interleaved records based on trace logic from mrf_join code review + self.assertEqual(final_idx[0], expected_s0_l0t0) # L0S0T0 + self.assertEqual(final_idx[1], expected_s0_l0t1) # L0S0T1 + self.assertEqual(final_idx[2], expected_s1_l0t0) # L0S1T0 + self.assertEqual(final_idx[3], expected_s1_l0t1) # L0S1T1 + self.assertEqual(final_idx[4], expected_s0_l1t0) # L1S0T0 + self.assertEqual(final_idx[5], expected_s1_l1t0) # L1S1T0 diff --git a/tests/test_jxl.py b/tests/test_jxl.py new file mode 100644 index 0000000..aac74d3 --- /dev/null +++ b/tests/test_jxl.py @@ -0,0 +1,77 @@ +import os +import shutil +import subprocess +import filecmp +from tests.helpers import MRFTestCase + +class TestJXLUtility(MRFTestCase): + """ + Tests for the jxl utility which converts between JFIF-JPEG and JPEG-XL (brunsli). + """ + + def setUp(self): + """Extend setUp to define the jxl executable path.""" + super().setUp() + self.jxl_executable = "jxl" + + def test_jxl_mrf_round_trip(self): + """Test round-trip conversion for an MRF data/index file pair.""" + if not shutil.which(self.jxl_executable): + self.skipTest(f"'{self.jxl_executable}' executable not found in PATH.") + + # ARRANGE: Create mock MRF files and a path for a backup of the original + data_path = os.path.join(self.test_dir, "test.pjg") + idx_path = os.path.join(self.test_dir, "test.idx") + jxl_data_path = data_path + ".jxl" + original_data_backup_path = os.path.join(self.test_dir, "original.pjg") + + # Generate the initial JPEG file and its index + self.create_mock_jpeg(data_path, size=(32, 32), color='blue') + jpeg_size = os.path.getsize(data_path) + self.create_mock_idx(idx_path, [(0, jpeg_size)]) + + # Back up the original file for the final comparison + shutil.copy(data_path, original_data_backup_path) + + # ACT: Convert MRF to JXL + subprocess.run([self.jxl_executable, data_path], check=True) + + # ASSERT: Check that the JXL file was created and is smaller + self.assertTrue(os.path.exists(jxl_data_path)) + self.assertLess(os.path.getsize(jxl_data_path), jpeg_size, + "JXL file should be smaller than the original JPEG MRF data.") + + # ACT: Convert back to MRF/JFIF. This will overwrite the original data_path file. + subprocess.run([self.jxl_executable, "-r", jxl_data_path], check=True) + + # ASSERT: The overwritten data file should be identical to our backup of the original + self.assertTrue(filecmp.cmp(original_data_backup_path, data_path, shallow=False), + "Round-trip conversion of MRF data file failed.") + + def test_jxl_single_file_round_trip(self): + """Test round-trip conversion for a single JPEG file using the -s flag.""" + if not shutil.which(self.jxl_executable): + self.skipTest(f"'{self.jxl_executable}' executable not found in PATH.") + + # ARRANGE: Create a mock single JPEG file + jpeg_path = os.path.join(self.test_dir, "single.jpg") + jxl_path = jpeg_path + ".jxl" + final_jpeg_path = os.path.join(self.test_dir, "final_single.jpg") + self.create_mock_jpeg(jpeg_path) + + # ACT: Convert JPEG to JXL in single file mode + subprocess.run([self.jxl_executable, "-s", jpeg_path], check=True) + + # ASSERT: Check that the JXL file exists + self.assertTrue(os.path.exists(jxl_path)) + + # ACT: Convert back to JPEG + subprocess.run([self.jxl_executable, "-s", "-r", jxl_path], check=True) + os.rename(jxl_path + ".jfif", final_jpeg_path) + + # ASSERT: The final file should be identical to the original + self.assertTrue(filecmp.cmp(jpeg_path, final_jpeg_path, shallow=False), + "Round-trip conversion of single JPEG file failed.") + def test_jxl_bundle_mode(self): + """Placeholder test for Esri bundle conversion.""" + self.skipTest("Skipping bundle test: Creating a mock Esri bundle is not yet implemented.") diff --git a/tests/test_mrf_insert.md b/tests/test_mrf_insert.md new file mode 100644 index 0000000..fb4e651 --- /dev/null +++ b/tests/test_mrf_insert.md @@ -0,0 +1,50 @@ +# Test Suite: `mrf_insert` + +This document outlines the test suite for the `mrf_insert` C++ utility. The purpose of this suite is to provide comprehensive **integration testing** that validates the end-to-end functionality of the tool. + +The tests use the `osgeo.gdal` and `numpy` Python libraries to dynamically create georeferenced test rasters. The `mrf_insert` executable is called as a `subprocess`, and the results are verified by reading the output MRF's pixel data back into memory. + +--- + +## Setup and Helpers + +All tests in this suite inherit from `MRFTestCase`, a base class that automatically handles the creation and cleanup of a temporary directory for each test, ensuring complete isolation. + +A helper method, `_create_geotiff`, is used to reduce code duplication by providing a simple interface for generating TIFF files with specific dimensions, fill values, and georeferencing. + +--- + +## Test Cases + +### `test_mrf_insert_simple_patch()` + +* **Purpose:** This test validates the most basic functionality of `mrf_insert`: patching a source image into a target MRF at the base resolution level. +* **Scenario:** + 1. A large target GeoTIFF (1024x1024) is created and filled with a background value of **0**. + 2. This target is converted into a base MRF. + 3. A smaller source GeoTIFF (512x512) is created, filled with a patch value of **255**, and georeferenced to the top-left corner of the target. +* **Assertions:** + 1. It asserts that the top-left 512x512 region of the modified MRF now contains the patch value (**255**). + 2. It asserts that a region outside the patch area remains unchanged, still containing the background value (**0**). + +### `test_mrf_insert_with_overviews()` + +* **Purpose:** This is a critical test that validates the utility's most powerful feature: the intelligent and efficient regeneration of **overviews** (pyramids) after an insert. +* **Scenario:** + 1. A large target MRF (2048x2048) is created with pre-built overview levels using the `UNIFORM_SCALE` creation option. The MRF is filled with a background value of **0**. + 2. A 512x512 source image filled with **255** is inserted into the target using the `-r Avg` command-line flag, which triggers the overview update logic. +* **Assertions:** + 1. It verifies that the **base resolution** is correctly patched. + 2. It then reads the first overview level (which is 2x downsampled) and asserts that the corresponding, smaller region in the overview has been correctly updated to the new average value (**255**). + 3. It also checks an unpatched area in the overview to ensure it was not modified. + +### `test_mrf_insert_partial_tile_overlap()` + +* **Purpose:** This test targets an important **edge case**: inserting a source image that only partially covers a tile in the target MRF. This validates the `ClippedRasterIO` logic within the C++ code, which must correctly merge old and new data within a single tile. +* **Scenario:** + 1. A target MRF is created with a single 512x512 tile, filled with a background value of **100**. + 2. A smaller 256x256 source image, filled with a patch value of **255**, is georeferenced to cover only the top-left quadrant of the target's single tile. +* **Assertions:** + 1. It reads the entire 512x512 tile from the modified MRF. + 2. It asserts that the top-left quadrant of the tile now contains the patch value (**255**). + 3. It asserts that the other three quadrants of the tile were not affected and still contain the original background value (**100**). diff --git a/tests/test_mrf_insert.py b/tests/test_mrf_insert.py new file mode 100644 index 0000000..55c0936 --- /dev/null +++ b/tests/test_mrf_insert.py @@ -0,0 +1,149 @@ +# tests/test_mrf_insert.py + +import os +import shutil +import subprocess +import numpy as np +from tests.helpers import MRFTestCase + +# The osgeo library is available inside the Docker container +try: + from osgeo import gdal +except ImportError: + gdal = None + +class TestMRFInsert(MRFTestCase): + """ + Test the mrf_insert utility, including base level patching, + overview regeneration, and partial tile updates. + """ + + def _create_geotiff(self, path, width, height, bands, fill_value, geo_transform): + """Helper function to create a georeferenced TIFF file.""" + driver = gdal.GetDriverByName('GTiff') + dataset = driver.Create(path, width, height, bands, gdal.GDT_Byte) + dataset.SetGeoTransform(geo_transform) + + fill_array = np.full((height, width), fill_value, dtype=np.uint8) + for i in range(1, bands + 1): + band = dataset.GetRasterBand(i) + band.WriteArray(fill_array) + + dataset = None # Close and save + + def setUp(self): + """Extend setUp to skip all tests if dependencies are missing.""" + super().setUp() + if not gdal: + self.skipTest("GDAL Python bindings are not available.") + if not shutil.which(self.mrf_insert_executable): + self.skipTest(f"'{self.mrf_insert_executable}' executable not found in PATH.") + + def test_mrf_insert_simple_patch(self): + """Test patching a small image into a larger MRF at the base resolution.""" + # Define file paths + target_tiff_path = os.path.join(self.test_dir, "target.tif") + source_tiff_path = os.path.join(self.test_dir, "source.tif") + target_mrf_path = os.path.join(self.test_dir, "target.mrf") + + # Create a large target raster (1024x1024) filled with zeros + self._create_geotiff(target_tiff_path, 1024, 1024, 1, 0, [0, 1, 0, 1024, 0, -1]) + + # Create a smaller source raster (512x512) filled with 255 + # Place it in the top-left corner of the target + self._create_geotiff(source_tiff_path, 512, 512, 1, 255, [0, 1, 0, 1024, 0, -1]) + + # Convert the target TIFF to an MRF + gdal.Translate(target_mrf_path, target_tiff_path, options='-f MRF -co BLOCKSIZE=512') + + # Run the mrf_insert utility + subprocess.run([self.mrf_insert_executable, source_tiff_path, target_mrf_path], check=True) + + # Verify the result + result_ds = gdal.Open(target_mrf_path) + result_band = result_ds.GetRasterBand(1) + + # All values in the patched area should be 255 + data_patched = result_band.ReadAsArray(0, 0, 512, 512) + self.assertTrue(np.all(data_patched == 255)) + + # Check an un-patched area to ensure it's still zero + data_unpatched = result_band.ReadAsArray(513, 513, 10, 10) + self.assertTrue(np.all(data_unpatched == 0)) + + result_ds = None + + def test_mrf_insert_with_overviews(self): + """Test that inserting a patch correctly updates the MRF's overviews.""" + # ARRANGE: Create a large target MRF with pre-built overviews + target_tiff_path = os.path.join(self.test_dir, "target_ov.tif") + source_tiff_path = os.path.join(self.test_dir, "source_ov.tif") + target_mrf_path = os.path.join(self.test_dir, "target_ov.mrf") + + self._create_geotiff(target_tiff_path, 2048, 2048, 1, 0, [0, 1, 0, 2048, 0, -1]) + # Insert a 512x512 patch of 255 into the second tile column, first row + self._create_geotiff(source_tiff_path, 512, 512, 1, 255, [512, 1, 0, 2048, 0, -1]) + + # Create an MRF with UNIFORM_SCALE to generate overview levels + gdal.Translate(target_mrf_path, target_tiff_path, options='-f MRF -co BLOCKSIZE=512 -co UNIFORM_SCALE=2') + + # ACT: Run mrf_insert with the '-r' flag to trigger overview regeneration + subprocess.run([self.mrf_insert_executable, "-r", "Avg", source_tiff_path, target_mrf_path], check=True) + + # ASSERT: Verify both the base level and the first overview level + result_ds = gdal.Open(target_mrf_path) + + # 1. Verify base level (as in the simple test) + base_band = result_ds.GetRasterBand(1) + patched_base_data = base_band.ReadAsArray(512, 0, 512, 512) + self.assertTrue(np.all(patched_base_data == 255), "Base level was not patched correctly.") + + # 2. Verify overview level + self.assertEqual(base_band.GetOverviewCount(), 2, "MRF should have overviews.") + ov_band = base_band.GetOverview(0) # First overview (2x downsampled) + self.assertEqual(ov_band.XSize, 1024, "Overview width is incorrect.") + + # The 512x512 patch at (512,0) on the base becomes a 256x256 patch at (256,0) on the overview + patched_ov_data = ov_band.ReadAsArray(256, 0, 256, 256) + self.assertTrue(np.all(patched_ov_data == 255), "Overview level was not updated correctly.") + + unpatched_ov_data = ov_band.ReadAsArray(0, 0, 10, 10) + self.assertTrue(np.all(unpatched_ov_data == 0), "Unpatched overview area was modified.") + + result_ds = None + + def test_mrf_insert_partial_tile_overlap(self): + """Test inserting a source that only partially covers a target tile.""" + # ARRANGE: Create a target with one 512x512 tile filled with value 100 + target_tiff_path = os.path.join(self.test_dir, "target_partial.tif") + source_tiff_path = os.path.join(self.test_dir, "source_partial.tif") + target_mrf_path = os.path.join(self.test_dir, "target_partial.mrf") + + self._create_geotiff(target_tiff_path, 512, 512, 1, 100, [0, 1, 0, 512, 0, -1]) + + # Create a 256x256 source to patch over the top-left quadrant of the target tile + self._create_geotiff(source_tiff_path, 256, 256, 1, 255, [0, 1, 0, 512, 0, -1]) + + gdal.Translate(target_mrf_path, target_tiff_path, options='-f MRF -co BLOCKSIZE=512') + + # ACT: Run mrf_insert + subprocess.run([self.mrf_insert_executable, source_tiff_path, target_mrf_path], check=True) + + # ASSERT: Check that the single tile is a mix of old and new data + result_ds = gdal.Open(target_mrf_path) + result_band = result_ds.GetRasterBand(1) + final_tile_data = result_band.ReadAsArray() + + # Top-left quadrant should be the new value (255) + top_left = final_tile_data[0:256, 0:256] + self.assertTrue(np.all(top_left == 255), "Top-left quadrant was not patched.") + + # Top-right quadrant should be the original value (100) + top_right = final_tile_data[0:256, 256:512] + self.assertTrue(np.all(top_right == 100), "Top-right quadrant was incorrectly modified.") + + # Bottom-left quadrant should be the original value (100) + bottom_left = final_tile_data[256:512, 0:256] + self.assertTrue(np.all(bottom_left == 100), "Bottom-left quadrant was incorrectly modified.") + + result_ds = None diff --git a/tests/test_mrf_size.py b/tests/test_mrf_size.py new file mode 100644 index 0000000..ee932d5 --- /dev/null +++ b/tests/test_mrf_size.py @@ -0,0 +1,120 @@ +import os +import subprocess +import sys +from xml.etree import ElementTree as ET +from tests.helpers import MRFTestCase +# Import the script to test its functions directly +from mrf_apps import mrf_size + +class TestMRFSize(MRFTestCase): + """ + Tests for the mrf_size.py script, which generates a VRT to visualize tile sizes. + """ + + def _create_full_mrf_xml(self, path, xsize=512, ysize=512, channels=1, pagesize=512, include_pagesize_tag=True): + """Creates a more complete MRF metadata file with GeoTags for testing.""" + root = ET.Element("MRF_META") + raster = ET.SubElement(root, "Raster") + ET.SubElement(raster, "Size", x=str(xsize), y=str(ysize), c=str(channels)) + if include_pagesize_tag: + ET.SubElement(raster, "PageSize", x=str(pagesize), y=str(pagesize), c=str(channels)) + + geotags = ET.SubElement(root, "GeoTags") + # Define a simple BoundingBox and Projection for testing GeoTransform + ET.SubElement(geotags, "BoundingBox", minx="0", miny="0", maxx=str(xsize), maxy=str(ysize)) + ET.SubElement(geotags, "Projection").text = "LOCAL_CS[\"Pseudo-Mercator\",UNIT[\"metre\",1]]" + + tree = ET.ElementTree(root) + tree.write(path) + + def test_vrt_creation_single_band(self): + """Test VRT generation for a single-band 2x1 tile MRF.""" + # ARRANGE: Create a mock MRF file for a 1024x512 image (2x1 tiles) + mrf_path = os.path.join(self.test_dir, "test.mrf") + vrt_path = os.path.join(self.test_dir, "test_size.vrt") + self._create_full_mrf_xml(mrf_path, xsize=1024, ysize=512, pagesize=512) + + # ACT: Run the script's main function + sys.argv = ["mrf_size.py", mrf_path] + mrf_size.main() + + # ASSERT + self.assertTrue(os.path.exists(vrt_path)) + tree = ET.parse(vrt_path) + root = tree.getroot() + + # 1. Check VRT dimensions (should be 2x1 pixels, one for each tile) + self.assertEqual(root.tag, "VRTDataset") + self.assertEqual(root.get("rasterXSize"), "2") + self.assertEqual(root.get("rasterYSize"), "1") + + # 2. Check GeoTransform (should be scaled by pagesize) + # Original pixel res: x=1, y=-1. Scaled by 512 -> x=512, y=-512 + geotransform = root.find("GeoTransform").text + self.assertEqual(geotransform, "0.0,512.0,0,512.0,0,-512.0") + + # 3. Check VRTRasterBand properties + band = root.find("VRTRasterBand") + self.assertIsNotNone(band) + self.assertEqual(band.get("dataType"), "UInt32") + self.assertEqual(band.find("SourceFilename").text, "test.idx") + + # ImageOffset should be 12 to read the lower 4 bytes of the 8-byte size field + self.assertEqual(band.find("ImageOffset").text, "12") + # PixelOffset is 16 bytes (size of one index record) + self.assertEqual(band.find("PixelOffset").text, "16") + # LineOffset is 16 * rasterXSize * num_bands = 16 * 2 * 1 = 32 + self.assertEqual(band.find("LineOffset").text, "32") + self.assertEqual(band.find("ByteOrder").text, "MSB") + + def test_vrt_creation_multi_band(self): + """Test VRT generation for a 3-band, single-tile MRF.""" + # ARRANGE + mrf_path = os.path.join(self.test_dir, "multiband.mrf") + vrt_path = os.path.join(self.test_dir, "multiband_size.vrt") + self._create_full_mrf_xml(mrf_path, xsize=512, ysize=512, channels=3, pagesize=512) + + # ACT + sys.argv = ["mrf_size.py", mrf_path] + mrf_size.main() + + # ASSERT + self.assertTrue(os.path.exists(vrt_path)) + tree = ET.parse(vrt_path) + root = tree.getroot() + + # 1. VRT should be 1x1 pixels + self.assertEqual(root.get("rasterXSize"), "1") + self.assertEqual(root.get("rasterYSize"), "1") + + # 2. There should be 3 raster bands + bands = root.findall("VRTRasterBand") + self.assertEqual(len(bands), 3) + + # 3. Check offsets for each band + # PixelOffset and LineOffset are 16 * num_bands = 48 + # ImageOffset increments by 16 for each band + self.assertEqual(bands[0].find("ImageOffset").text, "12") # 12 + 16 * 0 + self.assertEqual(bands[0].find("PixelOffset").text, "48") + self.assertEqual(bands[0].find("LineOffset").text, "48") + + self.assertEqual(bands[1].find("ImageOffset").text, "28") # 12 + 16 * 1 + self.assertEqual(bands[2].find("ImageOffset").text, "44") # 12 + 16 * 2 + + def test_vrt_default_pagesize(self): + """Test that a default PageSize of 512 is used when the tag is absent.""" + # ARRANGE: Create a 1024x1024 MRF without a tag + mrf_path = os.path.join(self.test_dir, "default.mrf") + vrt_path = os.path.join(self.test_dir, "default_size.vrt") + self._create_full_mrf_xml(mrf_path, xsize=1024, ysize=1024, include_pagesize_tag=False) + + # ACT + sys.argv = ["mrf_size.py", mrf_path] + mrf_size.main() + + # ASSERT: The VRT should be 2x2, based on the default 512x512 page size + self.assertTrue(os.path.exists(vrt_path)) + tree = ET.parse(vrt_path) + root = tree.getroot() + self.assertEqual(root.get("rasterXSize"), "2") + self.assertEqual(root.get("rasterYSize"), "2") diff --git a/tests/test_read_data.py b/tests/test_read_data.py new file mode 100644 index 0000000..8bb1b5d --- /dev/null +++ b/tests/test_read_data.py @@ -0,0 +1,110 @@ +import os +import subprocess +import struct +from tests.helpers import MRFTestCase + +class TestMRFReadData(MRFTestCase): + """ + Tests for the mrf_read_data.py script, which extracts tile data. + """ + + def test_read_with_offset_and_size(self): + """Test reading a data segment using direct offset and size.""" + # ARRANGE: Create a mock data file with three distinct parts + data_path = os.path.join(self.test_dir, "test.dat") + output_path = os.path.join(self.test_dir, "output.dat") + + part1 = b'START' + part2 = b'MIDDLE_CHUNK' + part3 = b'END' + self.create_mock_data(data_path, [part1, part2, part3]) + + # ACT: Run the script to extract the middle part + offset = len(part1) + size = len(part2) + + cmd = [ + "python3", "mrf_apps/mrf_read_data.py", + "--input", data_path, + "--output", output_path, + "--offset", str(offset), + "--size", str(size) + ] + # Hide the script's version printout for cleaner test logs + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + self.assertTrue("Wrote" in result.stdout) + + # ASSERT: The output file should contain only the middle part + self.assertTrue(os.path.exists(output_path)) + with open(output_path, 'rb') as f: + content = f.read() + self.assertEqual(content, part2) + + def test_read_with_index_and_tile(self): + """Test reading a tile using an index file.""" + # ARRANGE: Create a data file and an index file pointing to tiles within it + data_path = os.path.join(self.test_dir, "test.dat") + idx_path = os.path.join(self.test_dir, "test.idx") + output_path = os.path.join(self.test_dir, "output.dat") + + tile1_content = b'TILE_ONE_DATA' + tile2_content = b'TILE_TWO_DATA' + self.create_mock_data(data_path, [tile1_content, tile2_content]) + + # Index records: (offset, size) + # Tile 1 starts at offset 0 + # Tile 2 starts after tile 1 + idx_layout = [ + (0, len(tile1_content)), + (len(tile1_content), len(tile2_content)) + ] + self.create_mock_idx(idx_path, idx_layout) + + # ACT: Run the script to read the second tile (tile number is 1-based) + cmd = [ + "python3", "mrf_apps/mrf_read_data.py", + "--input", data_path, + "--output", output_path, + "--index", idx_path, + "--tile", "2" + ] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + self.assertTrue("Wrote" in result.stdout) + + # ASSERT: The output file should contain the content of the second tile + self.assertTrue(os.path.exists(output_path)) + with open(output_path, 'rb') as f: + content = f.read() + self.assertEqual(content, tile2_content) + + def test_read_with_little_endian_index(self): + """Test reading with a little-endian formatted index file.""" + # ARRANGE + data_path = os.path.join(self.test_dir, "test.dat") + idx_path = os.path.join(self.test_dir, "test.idx") + output_path = os.path.join(self.test_dir, "output.dat") + + tile_content = b'LITTLE_ENDIAN_TEST' + self.create_mock_data(data_path, [tile_content]) + + # Create the index file using little-endian byte order (' col 1, etc.) + # z=0 is highest res level in script logic (reversed list) + idx_records = self.read_idx_file(idx_path) + expected_idx = [ + (0, 1), # Tile 0,0 (A) + (1, 2), # Tile 0,1 (B) + (3, 3), # Tile 1,0 (C) + (6, 4), # Tile 1,1 (D) + (0, 0), + (0, 0), + (0, 0), + (0, 0) + ] + self.assertEqual(idx_records, expected_idx) + + def test_with_overviews_and_padding(self): + """Test a 2-level pyramid that requires index padding.""" + # ARRANGE + # Level 0 (overview, z=0): 2x1 tiles + # Level 1 (highest res, z=1): 3x2 tiles + template_dir = os.path.join(self.test_dir, "tiles_ov") + output_base = os.path.join(self.test_dir, "output_ov") + self._create_tile_files(template_dir, 2, [(2, 1), (3, 2)]) + + template = os.path.join(template_dir, "{z}", "{x}_{y}.ppg") + + # ACT + cmd = [ + "python3", "mrf_apps/tiles2mrf.py", + "--levels", "2", + "--width", "3", + "--height", "2", + template, + output_base + ] + subprocess.run(cmd, check=True, capture_output=True) + + # ASSERT + # Total tiles: 6 (z=1) + 2 (z=0) = 8. Data file should contain 8 tiles' data. + # Padding: Level 0 is 2x1. To get to 1x1, one more logical level is needed. + # The script pads the index to fill out this 2x1 level. Total pads = 2. + # Expected index records = 8 data records + 2 padding records = 10. + idx_records = self.read_idx_file(output_base + ".idx") + self.assertEqual(len(idx_records), 10, "Index should contain 8 data records and 2 padding records.") + + # Check the padding records at the end + self.assertEqual(idx_records[-2], (0, 0)) + self.assertEqual(idx_records[-1], (0, 0)) + + # Verify the offset and size of the last data tile. + # The script processes z=1 then z=0, so the last data tile is the + # last tile of level 0 (tile "B", which is the 2nd tile created). + total_data_size = sum(range(1, 9)) # Sum of sizes A(1) through H(8) + last_data_tile = idx_records[7] # The 8th data record corresponds to tile "B" + + # The size of tile "B" is 2 bytes + self.assertEqual(last_data_tile[1], 2) + # Its offset should be the total size minus its own size + self.assertEqual(last_data_tile[0], total_data_size - 2) + + def test_blank_tile_handling(self): + """Test that blank tiles are correctly identified and skipped.""" + # ARRANGE + template_dir = os.path.join(self.test_dir, "tiles_blank") + output_base = os.path.join(self.test_dir, "output_blank") + blank_tile_path = os.path.join(self.test_dir, "blank.ppg") + + # Create a 2x1 grid + os.makedirs(os.path.join(template_dir, "0")) + with open(os.path.join(template_dir, "0", "0_0.ppg"), 'wb') as f: + f.write(b'DATATILE') + with open(os.path.join(template_dir, "0", "1_0.ppg"), 'wb') as f: + f.write(b'BLANK') + with open(blank_tile_path, 'wb') as f: + f.write(b'BLANK') + + template = os.path.join(template_dir, "{z}", "{x}_{y}.ppg") + + # ACT + cmd = [ + "python3", "mrf_apps/tiles2mrf.py", + "--levels", "1", + "--width", "2", + "--height", "1", + "--blank-tile", blank_tile_path, + template, + output_base + ] + subprocess.run(cmd, check=True, capture_output=True) + + # ASSERT + data_path = output_base + ".ppg" + idx_path = output_base + ".idx" + + # 1. Data file should only contain the non-blank tile's data + self.assertEqual(os.path.getsize(data_path), 8) + with open(data_path, 'rb') as f: + self.assertEqual(f.read(), b'DATATILE') + + # 2. Index should have one data record and one blank record + idx_records = self.read_idx_file(idx_path) + expected_idx = [ + (0, 8), # Data tile at offset 0, size 8 + (0, 0), # Blank tile + (0, 0), + (0, 0) + ] + self.assertEqual(idx_records, expected_idx)