From 0dc4e06d1db876c225092ac195db55c95f8375fc Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Fri, 30 Oct 2026 07:44:18 +0100 Subject: [PATCH 1/6] export_workspace: support not merging derived_from links https://github.com/Open-EO/openeo-geopyspark-driver/issues/1922 --- openeo_driver/workspace.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/openeo_driver/workspace.py b/openeo_driver/workspace.py index 426b6c16..5c72de48 100644 --- a/openeo_driver/workspace.py +++ b/openeo_driver/workspace.py @@ -2,6 +2,7 @@ import logging import os.path import shutil +from dataclasses import dataclass from pathlib import Path, PurePath from typing import Optional, Union from urllib.parse import urlparse @@ -42,6 +43,11 @@ def merges_by_default(self) -> bool: return False +@dataclass(frozen=True) +class MergeCollectionMetadataOptions: + merge_derived_from_links: bool = True + + class DiskWorkspace(Workspace): def __init__(self, root_directory: Path): self.root_directory = root_directory @@ -67,7 +73,13 @@ def import_file(self, common_path: Union[str, Path], file: Path, merge: str, rem def import_object(self, common_path: str, s3_uri: str, merge: str, remove_original: bool = False): raise NotImplementedError(f"importing objects is not supported yet") - def merge(self, stac_resource: STACObject, target: PurePath, remove_original: bool = False) -> STACObject: + def merge( + self, + stac_resource: STACObject, + target: PurePath, + remove_original: bool = False, + merge_collection_metadata_options: MergeCollectionMetadataOptions = MergeCollectionMetadataOptions(), + ) -> STACObject: stac_resource = stac_resource.full_copy() target = self.root_directory / os.path.normpath(target).lstrip("/") @@ -135,7 +147,9 @@ def replace_asset_href(asset: Asset, src_collection_path: Path) -> Asset: assert isinstance( new_collection, Collection ), f"Can only merge a Collection into an existing Collection. Was type: {new_collection}" - merged_collection = _merge_collection_metadata(existing_collection, new_collection) + merged_collection = _merge_collection_metadata( + existing_collection, new_collection, merge_collection_metadata_options + ) new_collection = new_collection.map_assets( lambda _, asset: replace_asset_href(asset, src_collection_path=new_collection_path) ) @@ -168,7 +182,11 @@ def replace_asset_href(asset: Asset, src_collection_path: Path) -> Asset: raise NotImplementedError(stac_resource) -def _merge_collection_metadata(existing_collection: Collection, new_collection: Collection) -> Collection: +def _merge_collection_metadata( + existing_collection: Collection, + new_collection: Collection, + options: MergeCollectionMetadataOptions = MergeCollectionMetadataOptions(), +) -> Collection: existing_collection.extent.spatial = _merge_spatial_extents( existing_collection.extent.spatial, new_collection.extent.spatial ) @@ -177,7 +195,8 @@ def _merge_collection_metadata(existing_collection: Collection, new_collection: existing_collection.extent.temporal, new_collection.extent.temporal ) - _merge_derived_from_links(existing_collection, new_collection) + if options.merge_derived_from_links: + _merge_derived_from_links(existing_collection, new_collection) _merge_item_assets(existing_collection, new_collection) From 719aa7c5d784a6f17898899fe1267cbd854920b6 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Fri, 30 Oct 2026 09:04:44 +0100 Subject: [PATCH 2/6] bugfix: do not add new Collection's derived_from links https://github.com/Open-EO/openeo-geopyspark-driver/issues/1922 --- openeo_driver/workspace.py | 16 ++++++-------- tests/test_workspace.py | 44 ++++++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/openeo_driver/workspace.py b/openeo_driver/workspace.py index 5c72de48..b77c7d62 100644 --- a/openeo_driver/workspace.py +++ b/openeo_driver/workspace.py @@ -89,6 +89,8 @@ def merge( if isinstance(stac_resource, Catalog): # includes Collection new_collection = stac_resource + if not merge_collection_metadata_options.merge_derived_from_links: + new_collection.clear_links(rel=RelType.DERIVED_FROM) existing_collection = None try: @@ -133,6 +135,7 @@ def replace_asset_href(asset: Asset, src_collection_path: Path) -> Asset: new_collection = new_collection.map_assets( lambda _, asset: replace_asset_href(asset, src_collection_path=new_collection_path) ) + new_collection.save(CatalogType.SELF_CONTAINED) for new_item in new_collection.get_items(): @@ -147,9 +150,7 @@ def replace_asset_href(asset: Asset, src_collection_path: Path) -> Asset: assert isinstance( new_collection, Collection ), f"Can only merge a Collection into an existing Collection. Was type: {new_collection}" - merged_collection = _merge_collection_metadata( - existing_collection, new_collection, merge_collection_metadata_options - ) + merged_collection = _merge_collection_metadata(existing_collection, new_collection) new_collection = new_collection.map_assets( lambda _, asset: replace_asset_href(asset, src_collection_path=new_collection_path) ) @@ -182,11 +183,7 @@ def replace_asset_href(asset: Asset, src_collection_path: Path) -> Asset: raise NotImplementedError(stac_resource) -def _merge_collection_metadata( - existing_collection: Collection, - new_collection: Collection, - options: MergeCollectionMetadataOptions = MergeCollectionMetadataOptions(), -) -> Collection: +def _merge_collection_metadata(existing_collection: Collection, new_collection: Collection) -> Collection: existing_collection.extent.spatial = _merge_spatial_extents( existing_collection.extent.spatial, new_collection.extent.spatial ) @@ -195,8 +192,7 @@ def _merge_collection_metadata( existing_collection.extent.temporal, new_collection.extent.temporal ) - if options.merge_derived_from_links: - _merge_derived_from_links(existing_collection, new_collection) + _merge_derived_from_links(existing_collection, new_collection) _merge_item_assets(existing_collection, new_collection) diff --git a/tests/test_workspace.py b/tests/test_workspace.py index c53abdba..b9033fe2 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -9,7 +9,7 @@ import pytest from openeo_driver.util.date_math import now_utc -from openeo_driver.workspace import DiskWorkspace +from openeo_driver.workspace import DiskWorkspace, MergeCollectionMetadataOptions @pytest.mark.parametrize("merge", [ @@ -160,7 +160,8 @@ def test_merge_catalog_from_disk_new(tmp_path): assert f.read() == "catalog-asset\n" -def test_merge_from_disk_into_existing(tmp_path): +@pytest.mark.parametrize("merge_derived_from_links", [True, False]) +def test_merge_from_disk_into_existing(tmp_path, merge_derived_from_links: bool): existing_collection = _collection( root_path=tmp_path / "src" / "existing_collection", collection_id="existing_collection", @@ -219,10 +220,21 @@ def test_merge_from_disk_into_existing(tmp_path): ) target = Path("path") / "to" / "collection.json" + merge_collection_metadata_options = MergeCollectionMetadataOptions( + merge_derived_from_links=merge_derived_from_links + ) workspace = DiskWorkspace(root_directory=tmp_path) - workspace.merge(stac_resource=existing_collection, target=target) - imported_collection = workspace.merge(stac_resource=new_collection, target=target) + workspace.merge( + stac_resource=existing_collection, + target=target, + merge_collection_metadata_options=merge_collection_metadata_options, + ) + imported_collection = workspace.merge( + stac_resource=new_collection, + target=target, + merge_collection_metadata_options=merge_collection_metadata_options, + ) workspace_dir = (workspace.root_directory / target).parent assert _paths_relative_to(workspace_dir) == { @@ -260,16 +272,20 @@ def test_merge_from_disk_into_existing(tmp_path): dt.datetime.fromisoformat("2024-11-04T00:00:00+00:00")] ] - assert [link.to_dict() for link in exported_collection.get_links(rel=RelType.DERIVED_FROM)] == [ - { - "rel": "derived_from", - "href": "https://src.test/asset1.tif", - }, - { - "rel": "derived_from", - "href": "https://src.test/asset2.tif", - }, - ] + assert [link.to_dict() for link in exported_collection.get_links(rel=RelType.DERIVED_FROM)] == ( + [ + { + "rel": "derived_from", + "href": "https://src.test/asset1.tif", + }, + { + "rel": "derived_from", + "href": "https://src.test/asset2.tif", + }, + ] + if merge_derived_from_links + else [] + ) for item in exported_collection.get_items(): for asset in item.get_assets().values(): From 4afad1cd6546de3ffee5815680ddc0d8c89386f0 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Fri, 30 Oct 2026 09:09:55 +0100 Subject: [PATCH 3/6] adapt version and CHANGELOG https://github.com/Open-EO/openeo-geopyspark-driver/issues/1922 --- CHANGELOG.md | 1 + openeo_driver/_version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fffa9f41..62c510c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ and start a new "In Progress" section above it. - `run_udf` on the result of `read_vector`/`get_geometries` no longer passes the geometries as feature collection to the UDF, but follows the generic vector cube handling (like `load_uploaded_files`). - Use `save_result` node id in synchronous download filename hint (content-disposition header), and align filename extension handling with batch jobs ([#548](https://github.com/Open-EO/openeo-python-driver/issues/548)) - Support extraction of default job options from predefined (and "custom") processes, and do basic merging ([#483](https://github.com/Open-EO/openeo-python-driver/issues/483)) +- `export_workspace`: support not merging `derived_from` links into a STAC Collection. ([Open-EO/openeo-geopyspark-driver#1922](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1922)) ## 0.141.0 diff --git a/openeo_driver/_version.py b/openeo_driver/_version.py index 96657c89..829f0459 100644 --- a/openeo_driver/_version.py +++ b/openeo_driver/_version.py @@ -1 +1 @@ -__version__ = "0.142.0a4" +__version__ = "0.142.0a5" From e9a97c41d6a2df0e59cad140fa538d276426e4c4 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Fri, 30 Oct 2026 10:27:37 +0100 Subject: [PATCH 4/6] fix warning re: base signature https://github.com/Open-EO/openeo-geopyspark-driver/issues/1922 --- openeo_driver/workspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo_driver/workspace.py b/openeo_driver/workspace.py index b77c7d62..118b7fd9 100644 --- a/openeo_driver/workspace.py +++ b/openeo_driver/workspace.py @@ -70,7 +70,7 @@ def import_file(self, common_path: Union[str, Path], file: Path, merge: str, rem _log.debug(f"{'moved' if remove_original else 'copied'} {file_absolute} to {target_directory}") return f"file:{target_directory / file.name}" - def import_object(self, common_path: str, s3_uri: str, merge: str, remove_original: bool = False): + def import_object(self, common_path: Union[str, Path], s3_uri: str, merge: str, remove_original: bool = False): raise NotImplementedError(f"importing objects is not supported yet") def merge( From 1dac4ac60c80ed1a0d2acaa7a2cf483c95023a03 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Fri, 30 Oct 2026 10:29:37 +0100 Subject: [PATCH 5/6] adapt base signature https://github.com/Open-EO/openeo-geopyspark-driver/issues/1922 --- openeo_driver/workspace.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/openeo_driver/workspace.py b/openeo_driver/workspace.py index 118b7fd9..924792b5 100644 --- a/openeo_driver/workspace.py +++ b/openeo_driver/workspace.py @@ -16,6 +16,11 @@ _log = logging.getLogger(__name__) +@dataclass(frozen=True) +class MergeCollectionMetadataOptions: + merge_derived_from_links: bool = True + + class Workspace(abc.ABC): @abc.abstractmethod def import_file(self, common_path: Union[str, Path], file: Path, merge: str, remove_original: bool = False) -> str: @@ -28,13 +33,20 @@ def import_object( raise NotImplementedError @abc.abstractmethod - def merge(self, stac_resource: STACObject, target: PurePath, remove_original: bool = False) -> STACObject: + def merge( + self, + stac_resource: STACObject, + target: PurePath, + remove_original: bool = False, + merge_collection_metadata_options: MergeCollectionMetadataOptions = MergeCollectionMetadataOptions(), + ) -> STACObject: """ Merges a STAC resource, its children and their assets into this workspace at the given path, possibly removing the original assets. :param stac_resource: a STAC resource, typically a Collection :param target: a path identifier to a STAC resource to merge the given STAC resource into :param remove_original: remove the original assets? + :param merge_collection_metadata_options: options for merging collection metadata """ raise NotImplementedError @@ -43,11 +55,6 @@ def merges_by_default(self) -> bool: return False -@dataclass(frozen=True) -class MergeCollectionMetadataOptions: - merge_derived_from_links: bool = True - - class DiskWorkspace(Workspace): def __init__(self, root_directory: Path): self.root_directory = root_directory From acbf930565d160869fa29e86f4b4f01f1398773c Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Fri, 30 Oct 2026 14:08:20 +0100 Subject: [PATCH 6/6] cleanup https://github.com/Open-EO/openeo-geopyspark-driver/issues/1922 --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2d262d9..ca6250fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,6 @@ and start a new "In Progress" section above it. - Use `save_result` node id in synchronous download filename hint (content-disposition header), and align filename extension handling with batch jobs ([#548](https://github.com/Open-EO/openeo-python-driver/issues/548)) - Support extraction of default job options from predefined (and "custom") processes, and do basic merging ([#483](https://github.com/Open-EO/openeo-python-driver/issues/483)) - `load_stac` with STAC metadata where "bands" dimension is erroneously missing: "bands" dimension will still be forced in dry run if "bands" argument is given to `load_stac` ([Open-EO/openeo-geopyspark-driver#1907](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1907)) - - `export_workspace`: support not merging `derived_from` links into a STAC Collection. ([Open-EO/openeo-geopyspark-driver#1922](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1922)) ## 0.141.0