From c791f85926570d3845c20feaf3b43d528e2f5b93 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 10:37:37 -0400 Subject: [PATCH 01/18] update storage extension to v2.0.0 --- pystac/extensions/ext.py | 24 +- pystac/extensions/storage.py | 475 +++++++++++++----- tests/data-files/storage/collection-naip.json | 63 ++- tests/data-files/storage/item-naip.json | 65 ++- .../test_storage/test_refs_apply.yaml | 332 ++++++++++++ .../test_storage/test_validate_storage.yaml | 330 ++++++++++++ tests/extensions/test_storage.py | 323 +++++------- 7 files changed, 1237 insertions(+), 375 deletions(-) create mode 100644 tests/extensions/cassettes/test_storage/test_refs_apply.yaml diff --git a/pystac/extensions/ext.py b/pystac/extensions/ext.py index 84f60c39d..db487e93c 100644 --- a/pystac/extensions/ext.py +++ b/pystac/extensions/ext.py @@ -31,7 +31,7 @@ from pystac.extensions.sar import SarExtension from pystac.extensions.sat import SatExtension from pystac.extensions.scientific import ScientificExtension -from pystac.extensions.storage import StorageExtension +from pystac.extensions.storage import StorageRefsExtension, StorageSchemesExtension from pystac.extensions.table import TableExtension from pystac.extensions.timestamps import TimestampsExtension from pystac.extensions.version import BaseVersionExtension, VersionExtension @@ -85,7 +85,7 @@ SarExtension.name: SarExtension, SatExtension.name: SatExtension, ScientificExtension.name: ScientificExtension, - StorageExtension.name: StorageExtension, + StorageSchemesExtension.name: StorageSchemesExtension, TableExtension.name: TableExtension, TimestampsExtension.name: TimestampsExtension, VersionExtension.name: VersionExtension, @@ -172,6 +172,10 @@ def render(self) -> dict[str, Render]: def sci(self) -> ScientificExtension[Collection]: return ScientificExtension.ext(self.stac_object) + @property + def storage(self) -> StorageSchemesExtension[Collection]: + return StorageSchemesExtension.ext(self.stac_object) + @property def table(self) -> TableExtension[Collection]: return TableExtension.ext(self.stac_object) @@ -265,8 +269,8 @@ def sci(self) -> ScientificExtension[Item]: return ScientificExtension.ext(self.stac_object) @property - def storage(self) -> StorageExtension[Item]: - return StorageExtension.ext(self.stac_object) + def storage(self) -> StorageSchemesExtension[Item]: + return StorageSchemesExtension.ext(self.stac_object) @property def table(self) -> TableExtension[Item]: @@ -376,8 +380,8 @@ def sat(self) -> SatExtension[U]: return SatExtension.ext(self.stac_object) @property - def storage(self) -> StorageExtension[U]: - return StorageExtension.ext(self.stac_object) + def storage(self) -> StorageRefsExtension[U]: + return StorageRefsExtension.ext(self.stac_object) @property def table(self) -> TableExtension[U]: @@ -432,6 +436,10 @@ class ItemAssetExt(_AssetExt[ItemAssetDefinition]): def mlm(self) -> MLMExtension[ItemAssetDefinition]: return MLMExtension.ext(self.stac_object) + @property + def storage(self) -> StorageRefsExtension[ItemAssetDefinition]: + return StorageRefsExtension.ext(self.stac_object) + @dataclass class LinkExt(_AssetsExt[Link]): @@ -444,3 +452,7 @@ class LinkExt(_AssetsExt[Link]): @property def file(self) -> FileExtension[Link]: return FileExtension.ext(self.stac_object) + + @property + def storage(self) -> StorageRefsExtension[Link]: + return StorageRefsExtension.ext(self.stac_object) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 4270a9dc3..3a6a3c523 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -5,7 +5,7 @@ from __future__ import annotations -from collections.abc import Iterable +from abc import ABC from typing import ( Any, Generic, @@ -15,134 +15,253 @@ ) import pystac +from pystac.errors import RequiredPropertyMissing from pystac.extensions.base import ( ExtensionManagementMixin, PropertiesExtension, SummariesExtension, ) from pystac.extensions.hooks import ExtensionHooks -from pystac.utils import StringEnum +from pystac.utils import StringEnum, get_required, map_opt #: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Asset` or #: :class:`~pystac.ItemAssetDefinition` -T = TypeVar("T", pystac.Item, pystac.Asset, pystac.ItemAssetDefinition) +T = TypeVar("T", pystac.Item, pystac.Catalog, pystac.Collection) +U = TypeVar("U", pystac.Asset, pystac.Link, pystac.ItemAssetDefinition) -SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v1.0.0/schema.json" +SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v2.0.0/schema.json" PREFIX: str = "storage:" # Field names -PLATFORM_PROP: str = PREFIX + "platform" -REGION_PROP: str = PREFIX + "region" -REQUESTER_PAYS_PROP: str = PREFIX + "requester_pays" -TIER_PROP: str = PREFIX + "tier" +REFS_PROP: str = PREFIX + "refs" +SCHEMES_PROP: str = PREFIX + "schemes" +# Storage scheme object names +TYPE_PROP: str = "type" +PLATFORM_PROP: str = "platform" +REGION_PROP: str = "region" +REQUESTER_PAYS_PROP: str = "requester_pays" -class CloudPlatform(StringEnum): - ALIBABA = "ALIBABA" - AWS = "AWS" - AZURE = "AZURE" - GCP = "GCP" - IBM = "IBM" - ORACLE = "ORACLE" - OTHER = "OTHER" +class StorageSchemeType(StringEnum): + AWS_S3 = "aws-s3" + CUSTOM_S3 = "custom-s3" + AZURE = "ms-azure" -class StorageExtension( - Generic[T], - PropertiesExtension, - ExtensionManagementMixin[pystac.Item | pystac.Collection], -): - """An abstract class that can be used to extend the properties of an - :class:`~pystac.Item` or :class:`~pystac.Asset` with properties from the - :stac-ext:`Storage Extension `. This class is generic over the type of - STAC Object to be extended (e.g. :class:`~pystac.Item`, - :class:`~pystac.Asset`). - To create a concrete instance of :class:`StorageExtension`, use the - :meth:`StorageExtension.ext` method. For example: +class StorageScheme: + properties: dict[str, Any] - .. code-block:: python + def __init__(self, properties: dict[str, Any]): + super().__setattr__("properties", properties) - >>> item: pystac.Item = ... - >>> storage_ext = StorageExtension.ext(item) - """ + def __eq__(self, other: object) -> bool: + if not isinstance(other, StorageScheme): + raise NotImplementedError + return self.properties == other.properties - name: Literal["storage"] = "storage" + def __getattr__(self, name: str) -> Any: + if name in self.properties: + return self.properties[name] + raise AttributeError(f"StorageScheme does not have attribute '{name}'") + + def __setattr__(self, name: str, value: Any) -> None: + self.properties[name] = value def apply( self, - platform: CloudPlatform | None = None, + type: str, + platform: str, region: str | None = None, requester_pays: bool | None = None, - tier: str | None = None, + **kwargs: dict[str, Any], ) -> None: - """Applies Storage Extension properties to the extended :class:`~pystac.Item` or - :class:`~pystac.Asset`. - - Args: - platform (str, CloudPlatform) : The cloud provider where data is stored. - region (str) : The region where the data is stored. Relevant to speed of - access and inter-region egress costs (as defined by PaaS provider). - requester_pays (bool) : Is the data requester pays or is it data - manager/cloud provider pays. - tier (str) : The title for the tier type (as defined by PaaS provider). - """ + self.type = type self.platform = platform self.region = region self.requester_pays = requester_pays - self.tier = tier + self.properties.update(kwargs) - @property - def platform(self) -> CloudPlatform | None: - """Get or sets the cloud provider where data is stored. + @classmethod + def create( + cls, + type: str, + platform: str, + region: str | None = None, + requester_pays: bool | None = None, + **kwargs: dict[str, Any], + ) -> StorageScheme: + """Set the properties for a new StorageScheme object. + + Additional properties can be set through kwargs to fulfill + any additional variables in a templated uri. + + Args: + type (str): Type identifier for the platform. + platform (str): The cloud provider where data is stored as URI or URI + template to the API. + region (str | None, optional): The region where the data is stored. + Defaults to None. + requester_pays (bool | None, optional): requester pays or data manager/cloud + provider pays. Defaults to None. + kwargs (dict[str | Any]): Additional properties to set on scheme Returns: - str or None + StorageScheme: storage scheme """ - return self._get_property(PLATFORM_PROP, CloudPlatform) + c = cls({}) + c.apply( + type=type, + platform=platform, + region=region, + requester_pays=requester_pays, + **kwargs, + ) + return c + + @property + def type(self) -> str: + """ + Get or set the required type property + """ + return get_required( + self.properties.get(TYPE_PROP), + self, + TYPE_PROP, + ) + + @type.setter + def type(self, v: str) -> None: + self.properties[TYPE_PROP] = v + + @property + def platform(self) -> str: + """ + Get or set the required platform property + """ + return get_required( + self.properties.get(PLATFORM_PROP), + self, + PLATFORM_PROP, + ) @platform.setter - def platform(self, v: CloudPlatform | None) -> None: - self._set_property(PLATFORM_PROP, v) + def platform(self, v: str) -> None: + self.properties[PLATFORM_PROP] = v @property def region(self) -> str | None: - """Gets or sets the region where the data is stored. Relevant to speed of - access and inter-region egress costs (as defined by PaaS provider).""" - return self._get_property(REGION_PROP, str) + """ + Get or set the optional region property + """ + return self.properties.get(REGION_PROP) @region.setter - def region(self, v: str | None) -> None: - self._set_property(REGION_PROP, v) + def region(self, v: str) -> None: + if v is not None: + self.properties[REGION_PROP] = v + else: + self.properties.pop(REGION_PROP, None) @property def requester_pays(self) -> bool | None: - # This value "defaults to false", according to the extension spec. - return self._get_property(REQUESTER_PAYS_PROP, bool) + """ + Get or set the optional requester_pays property + """ + return self.properties.get(REQUESTER_PAYS_PROP) @requester_pays.setter - def requester_pays(self, v: bool | None) -> None: - self._set_property(REQUESTER_PAYS_PROP, v) + def requester_pays(self, v: bool) -> None: + if v is not None: + self.properties[REQUESTER_PAYS_PROP] = v + else: + self.properties.pop(REQUESTER_PAYS_PROP, None) + + def to_dict(self) -> dict[str, Any]: + """ + Returns the dictionary encoding of this object + + Returns: + dict[str, Any + """ + return self.properties - @property - def tier(self) -> str | None: - return self._get_property(TIER_PROP, str) - @tier.setter - def tier(self, v: str | None) -> None: - self._set_property(TIER_PROP, v) +class _StorageExtension(ABC): + name: Literal["storage"] = "storage" @classmethod def get_schema_uri(cls) -> str: return SCHEMA_URI + +class StorageSchemesExtension( + _StorageExtension, + Generic[T], + PropertiesExtension, + ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog], +): + """An abstract class that can be used to extend the properties of an + :class:`~pystac.Collection`, :class:`~pystac.Catalog`, or :class:`~pystac.Item` + with properties from the :stac-ext:`Storage Extension `. + This class is generic over the type of STAC Object to be extended (e.g. + :class:`~pystac.Item`, :class:`~pystac.Collection`). + + To create a concrete instance of :class:`StorageExtension`, use the + :meth:`StorageExtension.ext` method. For example: + + .. code-block:: python + + >>> item: pystac.Item = ... + >>> storage_ext = StorageExtension.ext(item) + """ + + def apply( + self, + schemes: dict[str, StorageScheme], + ) -> None: + """Applies Storage Extension properties to the extended + :class:`~pystac.Catalog`, :class:`~pystac.Collection`, + or :class:`~pystac.Item`. + + Args: + schemes (dict[str, StorageScheme]): Storage schemes used by Assets and Links + in the STAC Item, Catalog or Collection. + """ + self.schemes = schemes + + @property + def schemes(self) -> dict[str, StorageScheme]: + """Get or sets the schemes used by Assets and Links. + + Returns: + dict[str, StorageScheme]: storage schemes + """ + schemes: dict[str, dict[str, Any]] = get_required( + self.properties.get(SCHEMES_PROP), + self, + SCHEMES_PROP, + ) + return {k: StorageScheme(v) for k, v in schemes.items()} + + @schemes.setter + def schemes(self, v: dict[str, StorageScheme]) -> None: + v_trans = {k: c.to_dict() for k, c in v.items()} + self._set_property(SCHEMES_PROP, v_trans) + + def add_scheme(self, key: str, scheme: StorageScheme) -> None: + try: + self.schemes = {**self.schemes, **{key: scheme}} + except RequiredPropertyMissing: + self.schemes = {key: scheme} + @classmethod - def ext(cls, obj: T, add_if_missing: bool = False) -> StorageExtension[T]: + def ext(cls, obj: T, add_if_missing: bool = False) -> StorageSchemesExtension[T]: """Extends the given STAC Object with properties from the :stac-ext:`Storage Extension `. - This extension can be applied to instances of :class:`~pystac.Item` or - :class:`~pystac.Asset`. + This extension can be applied to instances of :class:`~pystac.Catalog`, + :class:`~pystac.Collection`, or :class:`~pystac.Item`. Raises: @@ -150,13 +269,13 @@ def ext(cls, obj: T, add_if_missing: bool = False) -> StorageExtension[T]: """ if isinstance(obj, pystac.Item): cls.ensure_has_extension(obj, add_if_missing) - return cast(StorageExtension[T], ItemStorageExtension(obj)) - elif isinstance(obj, pystac.Asset): - cls.ensure_owner_has_extension(obj, add_if_missing) - return cast(StorageExtension[T], AssetStorageExtension(obj)) - elif isinstance(obj, pystac.ItemAssetDefinition): - cls.ensure_owner_has_extension(obj, add_if_missing) - return cast(StorageExtension[T], ItemAssetsStorageExtension(obj)) + return cast(StorageSchemesExtension[T], ItemStorageExtension(obj)) + elif isinstance(obj, pystac.Collection): + cls.ensure_has_extension(obj, add_if_missing) + return cast(StorageSchemesExtension[T], CollectionStorageExtension(obj)) + elif isinstance(obj, pystac.Catalog): + cls.ensure_has_extension(obj, add_if_missing) + return cast(StorageSchemesExtension[T], CatalogStorageExtension(obj)) else: raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) @@ -169,13 +288,13 @@ def summaries( return SummariesStorageExtension(obj) -class ItemStorageExtension(StorageExtension[pystac.Item]): - """A concrete implementation of :class:`StorageExtension` on an +class ItemStorageExtension(StorageSchemesExtension[pystac.Item]): + """A concrete implementation of :class:`StorageSchemesExtension` on an :class:`~pystac.Item` that extends the properties of the Item to include properties defined in the :stac-ext:`Storage Extension `. This class should generally not be instantiated directly. Instead, call - :meth:`StorageExtension.ext` on an :class:`~pystac.Item` to extend it. + :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Item` to extend it. """ item: pystac.Item @@ -192,25 +311,122 @@ def __repr__(self) -> str: return f"" -class AssetStorageExtension(StorageExtension[pystac.Asset]): - """A concrete implementation of :class:`StorageExtension` on an - :class:`~pystac.Asset` that extends the Asset fields to include properties defined - in the :stac-ext:`Storage Extension `. +class CollectionStorageExtension(StorageSchemesExtension[pystac.Collection]): + """A concrete implementation of :class:`StorageSchemesExtension` on an + :class:`~pystac.Collection` that extends the properties of the Collection to include + properties defined in the :stac-ext:`Storage Extension `. This class should generally not be instantiated directly. Instead, call - :meth:`StorageExtension.ext` on an :class:`~pystac.Asset` to extend it. + :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Collection` to extend it. """ - asset_href: str - """The ``href`` value of the :class:`~pystac.Asset` being extended.""" + collection: pystac.Collection + """The :class:`~pystac.Collection` being extended.""" properties: dict[str, Any] - """The :class:`~pystac.Asset` fields, including extension properties.""" + """The :class:`~pystac.Collection` properties, including extension properties.""" + + def __init__(self, collection: pystac.Collection): + self.collection = collection + self.properties = collection.extra_fields + + def __repr__(self) -> str: + return f"" + + +class CatalogStorageExtension(StorageSchemesExtension[pystac.Catalog]): + """A concrete implementation of :class:`StorageSchemesExtension` on an + :class:`~pystac.Catalog` that extends the properties of the Catalog to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Catalog` to extend it. + """ + + catalog: pystac.Catalog + """The :class:`~pystac.Catalog` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Catalog` properties, including extension properties.""" + + def __init__(self, catalog: pystac.Catalog): + self.catalog = catalog + self.properties = catalog.extra_fields + + def __repr__(self) -> str: + return f"" + + +class StorageRefsExtension( + _StorageExtension, + Generic[U], + PropertiesExtension, + ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog], +): + def apply( + self, + refs: list[str], + ) -> None: + """Applies Storage Extension properties to the extended :class:`~pystac.Asset`, + :class:`~pystac.Link`, or :class:`~pystac.ItemAssetDefinition`. - additional_read_properties: Iterable[dict[str, Any]] | None = None - """If present, this will be a list containing 1 dictionary representing the - properties of the owning :class:`~pystac.Item`.""" + Args: + refs (list[str]): specifies which schemes in storage:schemes may be used to + access an Asset or Link. Each value must be one of the keys defined in + storage:schemes. + """ + self.refs = refs + + @property + def refs(self) -> list[str]: + """Get or sets the keys of the schemes that may be used to access an Asset + or Link. + + Returns: + list[str] + """ + return get_required( + self.properties.get(REFS_PROP), + self, + REFS_PROP, + ) + + @refs.setter + def refs(self, v: list[str]) -> None: + self._set_property(REFS_PROP, v) + + def add_ref(self, ref: str) -> None: + try: + self.refs.append(ref) + except RequiredPropertyMissing: + self.refs = [ref] + + @classmethod + def ext(cls, obj: U, add_if_missing: bool = False) -> StorageRefsExtension[U]: + """Extends the given STAC Object with properties from the :stac-ext:`Storage + Extension `. + This extension can be applied to instances of :class:`~pystac.Item` or + :class:`~pystac.Asset`. + + Raises: + + pystac.ExtensionTypeError : If an invalid object type is passed. + """ + if isinstance(obj, pystac.Asset): + cls.ensure_owner_has_extension(obj, add_if_missing) + return AssetStorageExtension(obj) + if isinstance(obj, pystac.Link): + cls.ensure_owner_has_extension(obj, add_if_missing) + return LinkStorageExtension(obj) + if isinstance(obj, pystac.ItemAssetDefinition): + cls.ensure_owner_has_extension(obj, add_if_missing) + return ItemAssetsStorageExtension(obj) + else: + raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) + + +class AssetStorageExtension(StorageRefsExtension[pystac.Asset]): def __init__(self, asset: pystac.Asset): self.asset_href = asset.href self.properties = asset.extra_fields @@ -221,7 +437,7 @@ def __repr__(self) -> str: return f"" -class ItemAssetsStorageExtension(StorageExtension[pystac.ItemAssetDefinition]): +class ItemAssetsStorageExtension(StorageRefsExtension[pystac.ItemAssetDefinition]): properties: dict[str, Any] asset_defn: pystac.ItemAssetDefinition @@ -230,6 +446,15 @@ def __init__(self, item_asset: pystac.ItemAssetDefinition): self.properties = item_asset.properties +class LinkStorageExtension(StorageRefsExtension[pystac.Link]): + properties: dict[str, Any] + link: pystac.Link + + def __init__(self, link: pystac.Link): + self.link = link + self.properties = link.extra_fields + + class SummariesStorageExtension(SummariesExtension): """A concrete implementation of :class:`~pystac.extensions.base.SummariesExtension` that extends the ``summaries`` field of a :class:`~pystac.Collection` to include @@ -237,54 +462,38 @@ class SummariesStorageExtension(SummariesExtension): """ @property - def platform(self) -> list[CloudPlatform] | None: + def schemes(self) -> list[dict[str, StorageScheme]] | None: """Get or sets the summary of :attr:`StorageExtension.platform` values for this Collection. """ - return self.summaries.get_list(PLATFORM_PROP) - - @platform.setter - def platform(self, v: list[CloudPlatform] | None) -> None: - self._set_summary(PLATFORM_PROP, v) - - @property - def region(self) -> list[str] | None: - """Get or sets the summary of :attr:`StorageExtension.region` values - for this Collection. - """ - return self.summaries.get_list(REGION_PROP) - - @region.setter - def region(self, v: list[str] | None) -> None: - self._set_summary(REGION_PROP, v) - - @property - def requester_pays(self) -> list[bool] | None: - """Get or sets the summary of :attr:`StorageExtension.requester_pays` values - for this Collection. - """ - return self.summaries.get_list(REQUESTER_PAYS_PROP) - - @requester_pays.setter - def requester_pays(self, v: list[bool] | None) -> None: - self._set_summary(REQUESTER_PAYS_PROP, v) - - @property - def tier(self) -> list[str] | None: - """Get or sets the summary of :attr:`StorageExtension.tier` values - for this Collection. - """ - return self.summaries.get_list(TIER_PROP) - - @tier.setter - def tier(self, v: list[str] | None) -> None: - self._set_summary(TIER_PROP, v) + v = map_opt( + lambda schemes: [ + {k: StorageScheme(v) for k, v in x.items()} for x in schemes + ], + self.summaries.get_list(SCHEMES_PROP), + ) + + print(v) + return v + + @schemes.setter + def schemes(self, v: list[dict[str, StorageScheme]] | None) -> None: + self._set_summary( + SCHEMES_PROP, + map_opt( + lambda schemes: [ + {k: c.to_dict() for k, c in x.items()} for x in schemes + ], + v, + ), + ) class StorageExtensionHooks(ExtensionHooks): schema_uri: str = SCHEMA_URI prev_extension_ids: set[str] = set() stac_object_types = { + pystac.STACObjectType.CATALOG, pystac.STACObjectType.COLLECTION, pystac.STACObjectType.ITEM, } diff --git a/tests/data-files/storage/collection-naip.json b/tests/data-files/storage/collection-naip.json index c24f38276..c3afac3da 100644 --- a/tests/data-files/storage/collection-naip.json +++ b/tests/data-files/storage/collection-naip.json @@ -16,29 +16,50 @@ } ], "stac_extensions": [ - "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" ], "summaries": { - "storage:platform": [ - "AZURE", - "GCP", - "AWS" - ], - "storage:region": [ - "westus2", - "us-central1", - "us-west-2", - "eastus" - ], - "storage:requester_pays": [ - true, - false - ], - "storage:tier": [ - "archive", - "COLDLINE", - "Standard", - "hot" + "storage:schemes": [ + { + "naip-azure-nsl": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naip-nsl" + } + }, + { + "naip-azure-eu": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipeuwest", + "requester_pays": false + } + }, + { + "naip-azure-blobs": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipblobs", + "region": "eastus" + } + }, + { + "naip-gcs": { + "type": "gcp", + "platform": "https://storage.googleapis.com/{bucket}", + "bucket": "naip-nsl", + "requester_pays": true + } + }, + { + "naip-aws": { + "type": "gcp", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "naip-visualization", + "region": "us-west-2", + "requester_pays": true + } + } ] }, "extent": { diff --git a/tests/data-files/storage/item-naip.json b/tests/data-files/storage/item-naip.json index 22069111d..59ebefb14 100644 --- a/tests/data-files/storage/item-naip.json +++ b/tests/data-files/storage/item-naip.json @@ -1,7 +1,7 @@ { "stac_version": "1.1.0", "stac_extensions": [ - "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" ], "id": "m_3009743_sw_14_1_20160928_20161129", "collection": "NAIP_MOSAIC", @@ -43,55 +43,70 @@ "datetime": "2016-09-28T00:00:00+00:00", "mission": "NAIP", "platform": "UNKNOWN_PLATFORM", - "gsd": 1 + "gsd": 1, + "storage:schemes": { + "naip-azure-nsl": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naip-nsl" + }, + "naip-azure-eu": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipeuwest", + "requester_pays": false + }, + "naip-azure-blobs": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipblobs", + "region": "eastus" + }, + "naip-gcs": { + "type": "gcp", + "platform": "https://storage.googleapis.com/{bucket}", + "bucket": "naip-nsl", + "requester_pays": true + }, + "naip-aws": { + "type": "gcp", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "naip-visualization", + "region": "us-west-2", + "requester_pays": true + } + } }, "assets": { "GEOTIFF_AZURE_RGBIR": { "href": "https://naip-nsl.blob.core.windows.net/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff", - "storage:platform": "AZURE", - "storage:region": "westus2", - "storage:tier": "archive" + "storage:refs": ["naip-azure-nsl"] }, "CO_GEOTIFF_GCP_RGB": { "href": "gs://naip-data/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "GCP", - "storage:region": "us-central1", - "storage:requester_pays": true, - "storage:tier": "COLDLINE" + "storage:refs": ["naip-gcs"] }, "CO_GEOTIFF_AWS_RGB": { "href": "s3://naip-visualization/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AWS", - "storage:region": "us-west-2", - "storage:requester_pays": true, - "storage:tier": "Standard" + "storage:refs": ["naip-aws"] }, "CO_GEOTIFF_AZURE_RGB": { "href": "https://naipeuwest.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AZURE", - "storage:region": "westeurope", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-eu"] }, "CO_GEOTIFF_AZURE_RGB_DEPRECATED": { "href": "https://naipblobs.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AZURE", - "storage:region": "eastus", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-blobs"] }, "THUMBNAIL_AZURE_DEPRECATED": { "href": "https://naipblobs.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.200.jpg", "type": "image/jpeg", - "storage:platform": "AZURE", - "storage:region": "eastus", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-blobs"] } }, "links": [ diff --git a/tests/extensions/cassettes/test_storage/test_refs_apply.yaml b/tests/extensions/cassettes/test_storage/test_refs_apply.yaml new file mode 100644 index 000000000..e5afbfe61 --- /dev/null +++ b/tests/extensions/cassettes/test_storage/test_refs_apply.yaml @@ -0,0 +1,332 @@ +interactions: +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n + \ }\n ]\n },\n {\n \"$comment\": \"This is the schema + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '4259' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-10a3"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - af64a991591f50737fadd2e89371cc12c8a2bb7d + X-GitHub-Request-Id: + - 591F:3B4DB0:4904A90:4C8204C:683CAF52 + X-Served-By: + - cache-iad-kcgs7200048-IAD + X-Timer: + - S1748816209.717952,VS0,VE1 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '706' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-2c2"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - 43d2b7092055b40e12b92793975aae512e94642f + X-GitHub-Request-Id: + - 87AB:3BCE73:4C312A6:4FAEA0D:683CAF53 + X-Served-By: + - cache-iad-kiad7000035-IAD + X-Timer: + - S1748816209.780679,VS0,VE3 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '353' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-161"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - c0e4c57aae853bd12f2127008255fb0d13a187fd + X-GitHub-Request-Id: + - 3461:2BF8CE:4BDFA81:4F5D0D4:683CAF53 + X-Served-By: + - cache-iad-kiad7000157-IAD + X-Timer: + - S1748816209.846702,VS0,VE4 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '469' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-1d5"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - ea7c52a0de24dedf14c95d765216ad65b6bfa746 + X-GitHub-Request-Id: + - D40E:3B1629:4B5EDFA:4EDC767:683CAF53 + X-Served-By: + - cache-iad-kiad7000135-IAD + X-Timer: + - S1748816209.915699,VS0,VE2 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml index e0f739b8a..ad4cfad1d 100644 --- a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml +++ b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml @@ -95,4 +95,334 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n + \ }\n ]\n },\n {\n \"$comment\": \"This is the schema + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '4259' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-10a3"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 855a50e2b2d2411c04eaca800019a8413b70b9a8 + X-GitHub-Request-Id: + - 591F:3B4DB0:4904A90:4C8204C:683CAF52 + X-Served-By: + - cache-iad-kiad7000152-IAD + X-Timer: + - S1748816208.373228,VS0,VE14 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '706' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-2c2"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 278480db61c132221d05f66f4d67dd50f9669b9a + X-GitHub-Request-Id: + - 87AB:3BCE73:4C312A6:4FAEA0D:683CAF53 + X-Served-By: + - cache-iad-kiad7000064-IAD + X-Timer: + - S1748816208.451944,VS0,VE18 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '353' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-161"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 17cf88621af9473909a54f09b7280d23a8d03f5b + X-GitHub-Request-Id: + - 3461:2BF8CE:4BDFA81:4F5D0D4:683CAF53 + X-Served-By: + - cache-iad-kiad7000129-IAD + X-Timer: + - S1748816209.531423,VS0,VE21 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '469' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-1d5"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - ff0d43db7da3291d7786a451b605ce5d2e96e1c4 + X-GitHub-Request-Id: + - D40E:3B1629:4B5EDFA:4EDC767:683CAF53 + X-Served-By: + - cache-iad-kiad7000060-IAD + X-Timer: + - S1748816209.609643,VS0,VE15 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK version: 1 diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index 4c1ba2322..e843ca0b9 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -7,7 +7,12 @@ import pystac from pystac import ExtensionTypeError, Item from pystac.collection import Collection -from pystac.extensions.storage import CloudPlatform, StorageExtension +from pystac.extensions.storage import ( + StorageRefsExtension, + StorageScheme, + StorageSchemesExtension, + StorageSchemeType, +) from tests.utils import TestCases, assert_to_from_dict NAIP_EXAMPLE_URI = TestCases.get_path("data-files/storage/item-naip.json") @@ -31,20 +36,20 @@ def test_to_from_dict() -> None: def test_add_to(sample_item: Item) -> None: - assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions # Check that the URI gets added to stac_extensions - StorageExtension.add_to(sample_item) - assert StorageExtension.get_schema_uri() in sample_item.stac_extensions + StorageSchemesExtension.add_to(sample_item) + assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions # Check that the URI only gets added once, regardless of how many times add_to # is called. - StorageExtension.add_to(sample_item) - StorageExtension.add_to(sample_item) + StorageSchemesExtension.add_to(sample_item) + StorageSchemesExtension.add_to(sample_item) eo_uris = [ uri for uri in sample_item.stac_extensions - if uri == StorageExtension.get_schema_uri() + if uri == StorageSchemesExtension.get_schema_uri() ] assert len(eo_uris) == 1 @@ -58,40 +63,60 @@ def test_extend_invalid_object() -> None: link = pystac.Link("child", "https://some-domain.com/some/path/to.json") with pytest.raises(pystac.ExtensionTypeError): - StorageExtension.ext(link) # type: ignore + StorageSchemesExtension.ext(link) # type: ignore def test_extension_not_implemented(sample_item: Item) -> None: # Should raise exception if Item does not include extension URI with pytest.raises(pystac.ExtensionNotImplemented): - _ = StorageExtension.ext(sample_item) + _ = StorageSchemesExtension.ext(sample_item) # Should raise exception if owning Item does not include extension URI asset = sample_item.assets["thumbnail"] with pytest.raises(pystac.ExtensionNotImplemented): - _ = StorageExtension.ext(asset) + _ = StorageRefsExtension.ext(asset) # Should succeed if Asset has no owner ownerless_asset = pystac.Asset.from_dict(asset.to_dict()) - _ = StorageExtension.ext(ownerless_asset) + _ = StorageRefsExtension.ext(ownerless_asset) + + +def test_collection_ext_add_to(naip_collection: Collection) -> None: + naip_collection.stac_extensions = [] + assert ( + StorageSchemesExtension.get_schema_uri() not in naip_collection.stac_extensions + ) + + _ = StorageSchemesExtension.ext(naip_collection, add_if_missing=True) + + assert StorageSchemesExtension.get_schema_uri() in naip_collection.stac_extensions def test_item_ext_add_to(sample_item: Item) -> None: - assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions - _ = StorageExtension.ext(sample_item, add_if_missing=True) + _ = StorageSchemesExtension.ext(sample_item, add_if_missing=True) - assert StorageExtension.get_schema_uri() in sample_item.stac_extensions + assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions def test_asset_ext_add_to(sample_item: Item) -> None: - assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions asset = sample_item.assets["thumbnail"] - _ = StorageExtension.ext(asset, add_if_missing=True) + _ = StorageRefsExtension.ext(asset, add_if_missing=True) + + assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions + - assert StorageExtension.get_schema_uri() in sample_item.stac_extensions +def test_link_ext_add_to(sample_item: Item) -> None: + assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions + asset = sample_item.links[0] + + _ = StorageRefsExtension.ext(asset, add_if_missing=True) + + assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: @@ -99,84 +124,44 @@ def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: asset = pystac.Asset.from_dict(asset_dict) with pytest.raises(pystac.STACError): - _ = StorageExtension.ext(asset, add_if_missing=True) + _ = StorageRefsExtension.ext(asset, add_if_missing=True) def test_should_raise_exception_when_passing_invalid_extension_object() -> None: with pytest.raises( - ExtensionTypeError, match=r"^StorageExtension does not apply to type 'object'$" + ExtensionTypeError, + match=r"^StorageRefsExtension does not apply to type 'object'$", ): # calling it wrong purposely so ---------v - StorageExtension.ext(object()) # type: ignore - - -def test_summaries_platform(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.platform == col_dict["summaries"]["storage:platform"] - # Set - new_platform_summary = [random.choice([v for v in CloudPlatform])] - assert storage_summaries.platform != new_platform_summary - storage_summaries.platform = new_platform_summary - assert storage_summaries.platform == new_platform_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:platform"] == new_platform_summary - - -def test_summaries_region(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.region == col_dict["summaries"]["storage:region"] - # Set - new_region_summary = [random.choice(ascii_letters)] - assert storage_summaries.region != new_region_summary - storage_summaries.region = new_region_summary - assert storage_summaries.region == new_region_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:region"] == new_region_summary + StorageRefsExtension.ext(object()) # type: ignore -def test_summaries_requester_pays(naip_collection: Collection) -> None: +def test_summaries_schemes(naip_collection: Collection) -> None: col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - + storage_summaries = StorageSchemesExtension.summaries(naip_collection) + print(naip_collection.summaries) # Get assert ( - storage_summaries.requester_pays - == col_dict["summaries"]["storage:requester_pays"] + list( + map( + lambda x: {k: c.to_dict() for k, c in x.items()}, + storage_summaries.schemes or [], + ) + ) + == col_dict["summaries"]["storage:schemes"] ) - # Set - new_requester_pays_summary = [True] - assert storage_summaries.requester_pays != new_requester_pays_summary - storage_summaries.requester_pays = new_requester_pays_summary - assert storage_summaries.requester_pays == new_requester_pays_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:requester_pays"] == new_requester_pays_summary - - -def test_summaries_tier(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.tier == col_dict["summaries"]["storage:tier"] - - # Set - new_tier_summary = [random.choice(ascii_letters)] - assert storage_summaries.tier != new_tier_summary - storage_summaries.tier = new_tier_summary - assert storage_summaries.tier == new_tier_summary + new_schemes_summary = [ + {"key": StorageScheme.create("aws-s3", "https://a.platform.example.com")} + ] + assert storage_summaries.schemes != new_schemes_summary + storage_summaries.schemes = new_schemes_summary + assert storage_summaries.schemes == new_schemes_summary col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:tier"] == new_tier_summary + assert col_dict["summaries"]["storage:schemes"] == [ + {k: c.to_dict() for k, c in x.items()} for x in new_schemes_summary + ] def test_summaries_adds_uri(naip_collection: Collection) -> None: @@ -185,152 +170,110 @@ def test_summaries_adds_uri(naip_collection: Collection) -> None: pystac.ExtensionNotImplemented, match="Extension 'storage' is not implemented", ): - StorageExtension.summaries(naip_collection, add_if_missing=False) - - _ = StorageExtension.summaries(naip_collection, add_if_missing=True) + StorageSchemesExtension.summaries(naip_collection, add_if_missing=False) - assert StorageExtension.get_schema_uri() in naip_collection.stac_extensions + _ = StorageSchemesExtension.summaries(naip_collection, add_if_missing=True) - StorageExtension.remove_from(naip_collection) - assert StorageExtension.get_schema_uri() not in naip_collection.stac_extensions + assert StorageSchemesExtension.get_schema_uri() in naip_collection.stac_extensions + StorageSchemesExtension.remove_from(naip_collection) + assert ( + StorageSchemesExtension.get_schema_uri() not in naip_collection.stac_extensions + ) -def test_item_apply(naip_item: Item) -> None: - asset = random.choice(list(naip_item.assets.values())) - - storage_ext = StorageExtension.ext(asset) - new_platform = random.choice( - [v for v in CloudPlatform if v != storage_ext.platform] - ) +def test_schemes_apply(naip_item: Item) -> None: + storage_ext = StorageSchemesExtension.ext(naip_item) + new_key = random.choice(ascii_letters) + new_type = random.choice(ascii_letters) + new_platform = random.choice(ascii_letters) new_region = random.choice(ascii_letters) - new_requestor_pays = random.choice( - [v for v in {True, False} if v != storage_ext.requester_pays] - ) - new_tier = random.choice(ascii_letters) + new_requestor_pays = random.choice([v for v in {True, False}]) storage_ext.apply( - platform=new_platform, - region=new_region, - requester_pays=new_requestor_pays, - tier=new_tier, + schemes={ + new_key: StorageScheme.create( + new_type, new_platform, new_region, new_requestor_pays + ), + } ) - assert storage_ext.platform == new_platform - assert storage_ext.region == new_region - assert storage_ext.requester_pays == new_requestor_pays - assert storage_ext.tier == new_tier + applied_schemes = storage_ext.schemes + assert list(applied_schemes) == [new_key] + assert applied_schemes[new_key].type == new_type + assert applied_schemes[new_key].platform == new_platform + assert applied_schemes[new_key].region == new_region + assert applied_schemes[new_key].requester_pays == new_requestor_pays @pytest.mark.vcr() -def test_asset_platform(naip_item: Item) -> None: +def test_refs_apply(naip_item: Item) -> None: # Grab a random asset with the platform property asset = random.choice( [ _asset for _asset in naip_item.assets.values() - if "storage:platform" in _asset.to_dict() + if "storage:refs" in _asset.to_dict() ] ) - storage_ext = StorageExtension.ext(asset) + storage_ext = StorageRefsExtension.ext(asset) # Get - assert storage_ext.platform == asset.extra_fields.get("storage:platform") + assert storage_ext.refs == asset.extra_fields.get("storage:refs") # Set - new_platform = random.choice( - [val for val in CloudPlatform if val != storage_ext.platform] - ) - storage_ext.platform = new_platform - assert storage_ext.platform == new_platform + new_refs = [random.choice(ascii_letters)] + storage_ext.refs = new_refs + assert storage_ext.refs == new_refs naip_item.validate() -@pytest.mark.vcr() -def test_asset_region(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:region" in _asset.to_dict() - ] - ) +def test_add_storage_scheme(naip_item: Item) -> None: + storage_ext = naip_item.ext.storage + storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) + assert "new_scheme" in storage_ext.schemes - storage_ext = StorageExtension.ext(asset) + storage_ext.properties.pop("storage:schemes") + storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) + assert len(storage_ext.schemes) == 1 + assert "new_scheme" in storage_ext.schemes - # Get - assert storage_ext.region == asset.extra_fields.get("storage:region") - # Set - new_region = random.choice( - [val for val in CloudPlatform if val != storage_ext.region] - ) - storage_ext.region = new_region - assert storage_ext.region == new_region +def test_add_refs(naip_item: Item) -> None: + scheme_name = random.choice(ascii_letters) + asset = naip_item.assets["GEOTIFF_AZURE_RGBIR"] + storage_ext = asset.ext.storage + storage_ext.add_ref(scheme_name) + assert scheme_name in storage_ext.refs - naip_item.validate() - - # Set to None - storage_ext.region = None - assert "storage:region" not in asset.extra_fields + storage_ext.properties.pop("storage:refs") + scheme_name_2 = random.choice(ascii_letters) + storage_ext.add_ref(scheme_name_2) + assert len(storage_ext.refs) == 1 + assert scheme_name_2 in storage_ext.refs -@pytest.mark.vcr() -def test_asset_requester_pays(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:requester_pays" in _asset.to_dict() - ] +def test_storage_scheme_create() -> None: + scheme = StorageScheme.create( + type=StorageSchemeType.AWS_S3, + platform="https://{bucket}.s3.{region}.amazonaws.com", + region="us-west-2", + requester_pays=True, ) - storage_ext = StorageExtension.ext(asset) + assert scheme.type == StorageSchemeType.AWS_S3 + assert scheme.platform == "https://{bucket}.s3.{region}.amazonaws.com" + assert scheme.region == "us-west-2" + assert scheme.requester_pays is True - # Get - assert storage_ext.requester_pays == asset.extra_fields.get( - "storage:requester_pays" - ) - - # Set - new_requester_pays = True if not storage_ext.requester_pays else False - storage_ext.requester_pays = new_requester_pays - assert storage_ext.requester_pays == new_requester_pays - - naip_item.validate() - - # Set to None - storage_ext.requester_pays = None - assert "storage:requester_pays" not in asset.extra_fields - - -@pytest.mark.vcr() -def test_asset_tier(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:tier" in _asset.to_dict() - ] - ) - - storage_ext = StorageExtension.ext(asset) - - # Get - assert storage_ext.tier == asset.extra_fields.get("storage:tier") - - # Set - new_tier = random.choice(ascii_letters) - storage_ext.tier = new_tier - assert storage_ext.tier == new_tier - - naip_item.validate() + scheme.type = StorageSchemeType.AZURE + scheme.platform = "https://{account}.blob.core.windows.net" + scheme.region = "eastus" + scheme.requester_pays = False - # Set to None - storage_ext.tier = None - assert "storage:tier" not in asset.extra_fields + assert scheme.type == StorageSchemeType.AZURE + assert scheme.platform == "https://{account}.blob.core.windows.net" + assert scheme.region == "eastus" + assert scheme.requester_pays is False From 489918d12e833c9392c418a34bf8e96da0333768 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 10:59:14 -0400 Subject: [PATCH 02/18] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a21d5f16..93a54b15a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changed - Remove unused pystac.validation import ([#1583](https://github.com/stac-utils/pystac/pull/1583)) +- Updated storage extension to v2.0.0 ### Fixed From c2b0b47ddce62da79a46f05c4d442e2cdd3198d1 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 20:10:23 -0400 Subject: [PATCH 03/18] improve equality and setting/getting attribute --- pystac/extensions/storage.py | 48 +++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 3a6a3c523..a0e443639 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -10,6 +10,7 @@ Any, Generic, Literal, + SupportsIndex, TypeVar, cast, ) @@ -50,23 +51,31 @@ class StorageSchemeType(StringEnum): class StorageScheme: - properties: dict[str, Any] + _properties: dict[str, Any] def __init__(self, properties: dict[str, Any]): - super().__setattr__("properties", properties) + super().__setattr__("_properties", properties) def __eq__(self, other: object) -> bool: if not isinstance(other, StorageScheme): - raise NotImplementedError - return self.properties == other.properties + return NotImplemented + + return bool(self.__dict__["_properties"] == other.__dict__["_properties"]) def __getattr__(self, name: str) -> Any: - if name in self.properties: - return self.properties[name] + properties = self.__dict__["_properties"] + if name in properties: + return properties[name] raise AttributeError(f"StorageScheme does not have attribute '{name}'") def __setattr__(self, name: str, value: Any) -> None: - self.properties[name] = value + if name.startswith("_") or hasattr(type(self), name): + super().__setattr__(name, value) + else: + self._properties[name] = value + + def __reduce_ex__(self, protocol: SupportsIndex) -> Any: + return (self.__class__, (self.__dict__["_properties"],), None) def apply( self, @@ -80,7 +89,7 @@ def apply( self.platform = platform self.region = region self.requester_pays = requester_pays - self.properties.update(kwargs) + self._properties.update(kwargs) @classmethod def create( @@ -125,14 +134,14 @@ def type(self) -> str: Get or set the required type property """ return get_required( - self.properties.get(TYPE_PROP), + self._properties.get(TYPE_PROP), self, TYPE_PROP, ) @type.setter def type(self, v: str) -> None: - self.properties[TYPE_PROP] = v + self._properties[TYPE_PROP] = v @property def platform(self) -> str: @@ -140,42 +149,42 @@ def platform(self) -> str: Get or set the required platform property """ return get_required( - self.properties.get(PLATFORM_PROP), + self._properties.get(PLATFORM_PROP), self, PLATFORM_PROP, ) @platform.setter def platform(self, v: str) -> None: - self.properties[PLATFORM_PROP] = v + self._properties[PLATFORM_PROP] = v @property def region(self) -> str | None: """ Get or set the optional region property """ - return self.properties.get(REGION_PROP) + return self._properties.get(REGION_PROP) @region.setter def region(self, v: str) -> None: if v is not None: - self.properties[REGION_PROP] = v + self._properties[REGION_PROP] = v else: - self.properties.pop(REGION_PROP, None) + self._properties.pop(REGION_PROP, None) @property def requester_pays(self) -> bool | None: """ Get or set the optional requester_pays property """ - return self.properties.get(REQUESTER_PAYS_PROP) + return self._properties.get(REQUESTER_PAYS_PROP) @requester_pays.setter def requester_pays(self, v: bool) -> None: if v is not None: - self.properties[REQUESTER_PAYS_PROP] = v + self._properties[REQUESTER_PAYS_PROP] = v else: - self.properties.pop(REQUESTER_PAYS_PROP, None) + self._properties.pop(REQUESTER_PAYS_PROP, None) def to_dict(self) -> dict[str, Any]: """ @@ -184,7 +193,7 @@ def to_dict(self) -> dict[str, Any]: Returns: dict[str, Any """ - return self.properties + return self._properties class _StorageExtension(ABC): @@ -473,7 +482,6 @@ def schemes(self) -> list[dict[str, StorageScheme]] | None: self.summaries.get_list(SCHEMES_PROP), ) - print(v) return v @schemes.setter From 723a1482179ed6c13ff45605dbb4f58b9ee9e3c8 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 22:38:28 -0400 Subject: [PATCH 04/18] increase test coverage --- tests/extensions/test_storage.py | 95 +++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index e843ca0b9..d176e6f54 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -1,5 +1,6 @@ import json import random +from copy import deepcopy from string import ascii_letters import pytest @@ -29,6 +30,28 @@ def naip_collection() -> Collection: return Collection.from_file(NAIP_COLLECTION_URI) +@pytest.fixture +def sample_scheme() -> StorageScheme: + return StorageScheme.create( + type=StorageSchemeType.AWS_S3, + platform="https://{bucket}.s3.{region}.amazonaws.com", + region="us-west-2", + requester_pays=True, + ) + + +@pytest.fixture +def naip_asset(naip_item: Item) -> pystac.Asset: + # Grab a random asset with the platform property + return random.choice( + [ + _asset + for _asset in naip_item.assets.values() + if "storage:refs" in _asset.to_dict() + ] + ) + + def test_to_from_dict() -> None: with open(NAIP_EXAMPLE_URI) as f: item_dict = json.load(f) @@ -101,6 +124,16 @@ def test_item_ext_add_to(sample_item: Item) -> None: assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions +def test_catalog_ext_add_to() -> None: + catalog = pystac.Catalog("stac", "a catalog") + + assert StorageSchemesExtension.get_schema_uri() not in catalog.stac_extensions + + _ = StorageSchemesExtension.ext(catalog, add_if_missing=True) + + assert StorageSchemesExtension.get_schema_uri() in catalog.stac_extensions + + def test_asset_ext_add_to(sample_item: Item) -> None: assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions asset = sample_item.assets["thumbnail"] @@ -207,28 +240,21 @@ def test_schemes_apply(naip_item: Item) -> None: @pytest.mark.vcr() -def test_refs_apply(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:refs" in _asset.to_dict() - ] - ) +def test_refs_apply(naip_asset: pystac.Asset) -> None: + test_refs = ["a_ref", "b_ref"] + + storage_ext = StorageRefsExtension.ext(naip_asset) - storage_ext = StorageRefsExtension.ext(asset) + storage_ext.apply(test_refs) # Get - assert storage_ext.refs == asset.extra_fields.get("storage:refs") + assert storage_ext.refs == test_refs # Set new_refs = [random.choice(ascii_letters)] storage_ext.refs = new_refs assert storage_ext.refs == new_refs - naip_item.validate() - def test_add_storage_scheme(naip_item: Item) -> None: storage_ext = naip_item.ext.storage @@ -255,25 +281,30 @@ def test_add_refs(naip_item: Item) -> None: assert scheme_name_2 in storage_ext.refs -def test_storage_scheme_create() -> None: - scheme = StorageScheme.create( - type=StorageSchemeType.AWS_S3, - platform="https://{bucket}.s3.{region}.amazonaws.com", - region="us-west-2", - requester_pays=True, - ) +def test_storage_scheme_create(sample_scheme: StorageScheme) -> None: + assert sample_scheme.type == StorageSchemeType.AWS_S3 + assert sample_scheme.platform == "https://{bucket}.s3.{region}.amazonaws.com" + assert sample_scheme.region == "us-west-2" + assert sample_scheme.requester_pays is True + + sample_scheme.type = StorageSchemeType.AZURE + sample_scheme.platform = "https://{account}.blob.core.windows.net" + sample_scheme.region = "eastus" + sample_scheme.account = "account" + sample_scheme.requester_pays = False + + assert sample_scheme.type == StorageSchemeType.AZURE + assert sample_scheme.platform == "https://{account}.blob.core.windows.net" + assert sample_scheme.region == "eastus" + assert sample_scheme.account == "account" + assert sample_scheme.requester_pays is False + - assert scheme.type == StorageSchemeType.AWS_S3 - assert scheme.platform == "https://{bucket}.s3.{region}.amazonaws.com" - assert scheme.region == "us-west-2" - assert scheme.requester_pays is True +def test_storage_scheme_equality(sample_scheme: StorageScheme) -> None: + other = deepcopy(sample_scheme) + assert sample_scheme == other - scheme.type = StorageSchemeType.AZURE - scheme.platform = "https://{account}.blob.core.windows.net" - scheme.region = "eastus" - scheme.requester_pays = False + other.requester_pays = False + assert sample_scheme != other - assert scheme.type == StorageSchemeType.AZURE - assert scheme.platform == "https://{account}.blob.core.windows.net" - assert scheme.region == "eastus" - assert scheme.requester_pays is False + assert sample_scheme != object() From 3df49bc41b49ec9101c8b33eedb572519671e63b Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:40:38 -0400 Subject: [PATCH 05/18] documentation fixes --- docs/api/extensions.rst | 3 ++- pystac/extensions/storage.py | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/api/extensions.rst b/docs/api/extensions.rst index ce37d3215..19d17c84a 100644 --- a/docs/api/extensions.rst +++ b/docs/api/extensions.rst @@ -30,7 +30,8 @@ pystac.extensions sar.SarExtension sat.SatExtension scientific.ScientificExtension - storage.StorageExtension + storage.StorageSchemesExtension + storage.StorageRefsExtension table.TableExtension timestamps.TimestampsExtension version.VersionExtension diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index a0e443639..15f0ff37b 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -25,9 +25,11 @@ from pystac.extensions.hooks import ExtensionHooks from pystac.utils import StringEnum, get_required, map_opt -#: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Asset` or -#: :class:`~pystac.ItemAssetDefinition` +#: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Catalog` or +#: :class:`~pystac.Collection` T = TypeVar("T", pystac.Item, pystac.Catalog, pystac.Collection) +#: Generalized version of :class:`~pystac.Asset`, :class:`~pystac.Link` or +#: :class:`~pystac.ItemAssetDefinition` U = TypeVar("U", pystac.Asset, pystac.Link, pystac.ItemAssetDefinition) SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v2.0.0/schema.json" @@ -109,9 +111,9 @@ def create( type (str): Type identifier for the platform. platform (str): The cloud provider where data is stored as URI or URI template to the API. - region (str | None, optional): The region where the data is stored. + region (str | None): The region where the data is stored. Defaults to None. - requester_pays (bool | None, optional): requester pays or data manager/cloud + requester_pays (bool | None): requester pays or data manager/cloud provider pays. Defaults to None. kwargs (dict[str | Any]): Additional properties to set on scheme @@ -216,13 +218,13 @@ class StorageSchemesExtension( This class is generic over the type of STAC Object to be extended (e.g. :class:`~pystac.Item`, :class:`~pystac.Collection`). - To create a concrete instance of :class:`StorageExtension`, use the - :meth:`StorageExtension.ext` method. For example: + To create a concrete instance of :class:`StorageSchemesExtension`, use the + :meth:`StorageSchemesExtension.ext` method. For example: .. code-block:: python >>> item: pystac.Item = ... - >>> storage_ext = StorageExtension.ext(item) + >>> storage_ext = StorageSchemesExtension.ext(item) """ def apply( @@ -472,7 +474,7 @@ class SummariesStorageExtension(SummariesExtension): @property def schemes(self) -> list[dict[str, StorageScheme]] | None: - """Get or sets the summary of :attr:`StorageExtension.platform` values + """Get or sets the summary of :attr:`StorageScheme.platform` values for this Collection. """ v = map_opt( From 141b639b125af7c95a11fbc5a4ec9a777228c2fc Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Wed, 18 Jun 2025 15:35:24 -0400 Subject: [PATCH 06/18] increase coverage, fix naming --- tests/extensions/test_storage.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index d176e6f54..4974bbfe0 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -6,7 +6,7 @@ import pytest import pystac -from pystac import ExtensionTypeError, Item +from pystac import ExtensionTypeError, Item, ItemAssetDefinition from pystac.collection import Collection from pystac.extensions.storage import ( StorageRefsExtension, @@ -69,12 +69,12 @@ def test_add_to(sample_item: Item) -> None: StorageSchemesExtension.add_to(sample_item) StorageSchemesExtension.add_to(sample_item) - eo_uris = [ + uris = [ uri for uri in sample_item.stac_extensions if uri == StorageSchemesExtension.get_schema_uri() ] - assert len(eo_uris) == 1 + assert len(uris) == 1 @pytest.mark.vcr() @@ -271,6 +271,8 @@ def test_add_refs(naip_item: Item) -> None: scheme_name = random.choice(ascii_letters) asset = naip_item.assets["GEOTIFF_AZURE_RGBIR"] storage_ext = asset.ext.storage + assert isinstance(storage_ext, StorageRefsExtension) + storage_ext.add_ref(scheme_name) assert scheme_name in storage_ext.refs @@ -308,3 +310,10 @@ def test_storage_scheme_equality(sample_scheme: StorageScheme) -> None: assert sample_scheme != other assert sample_scheme != object() + + +def test_item_asset_accessor() -> None: + item_asset = ItemAssetDefinition.create( + title="title", description="desc", media_type="media", roles=["a_role"] + ) + assert isinstance(item_asset.ext.storage, StorageRefsExtension) From 41f04ed04e246cb1c63e5c2231d38addf3180405 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Tue, 24 Jun 2025 17:20:29 -0400 Subject: [PATCH 07/18] retrigger ci --- tests/extensions/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index 4974bbfe0..6f696c60d 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -244,7 +244,6 @@ def test_refs_apply(naip_asset: pystac.Asset) -> None: test_refs = ["a_ref", "b_ref"] storage_ext = StorageRefsExtension.ext(naip_asset) - storage_ext.apply(test_refs) # Get From a0267df6517af87686031c631942d33b711aa400 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Sat, 6 Dec 2025 20:09:37 -0500 Subject: [PATCH 08/18] rework: single entrypoint --- pystac/extensions/ext.py | 24 +- pystac/extensions/storage.py | 430 ++++++++++++++++++------------- tests/extensions/test_storage.py | 133 ++++++---- 3 files changed, 338 insertions(+), 249 deletions(-) diff --git a/pystac/extensions/ext.py b/pystac/extensions/ext.py index db487e93c..0b72e9feb 100644 --- a/pystac/extensions/ext.py +++ b/pystac/extensions/ext.py @@ -31,7 +31,7 @@ from pystac.extensions.sar import SarExtension from pystac.extensions.sat import SatExtension from pystac.extensions.scientific import ScientificExtension -from pystac.extensions.storage import StorageRefsExtension, StorageSchemesExtension +from pystac.extensions.storage import StorageExtension from pystac.extensions.table import TableExtension from pystac.extensions.timestamps import TimestampsExtension from pystac.extensions.version import BaseVersionExtension, VersionExtension @@ -85,7 +85,7 @@ SarExtension.name: SarExtension, SatExtension.name: SatExtension, ScientificExtension.name: ScientificExtension, - StorageSchemesExtension.name: StorageSchemesExtension, + StorageExtension.name: StorageExtension, TableExtension.name: TableExtension, TimestampsExtension.name: TimestampsExtension, VersionExtension.name: VersionExtension, @@ -173,8 +173,8 @@ def sci(self) -> ScientificExtension[Collection]: return ScientificExtension.ext(self.stac_object) @property - def storage(self) -> StorageSchemesExtension[Collection]: - return StorageSchemesExtension.ext(self.stac_object) + def storage(self) -> StorageExtension[Collection]: + return StorageExtension.ext(self.stac_object) @property def table(self) -> TableExtension[Collection]: @@ -269,8 +269,8 @@ def sci(self) -> ScientificExtension[Item]: return ScientificExtension.ext(self.stac_object) @property - def storage(self) -> StorageSchemesExtension[Item]: - return StorageSchemesExtension.ext(self.stac_object) + def storage(self) -> StorageExtension[Item]: + return StorageExtension.ext(self.stac_object) @property def table(self) -> TableExtension[Item]: @@ -380,8 +380,8 @@ def sat(self) -> SatExtension[U]: return SatExtension.ext(self.stac_object) @property - def storage(self) -> StorageRefsExtension[U]: - return StorageRefsExtension.ext(self.stac_object) + def storage(self) -> StorageExtension[U]: + return StorageExtension.ext(self.stac_object) @property def table(self) -> TableExtension[U]: @@ -437,8 +437,8 @@ def mlm(self) -> MLMExtension[ItemAssetDefinition]: return MLMExtension.ext(self.stac_object) @property - def storage(self) -> StorageRefsExtension[ItemAssetDefinition]: - return StorageRefsExtension.ext(self.stac_object) + def storage(self) -> StorageExtension[ItemAssetDefinition]: + return StorageExtension.ext(self.stac_object) @dataclass @@ -454,5 +454,5 @@ def file(self) -> FileExtension[Link]: return FileExtension.ext(self.stac_object) @property - def storage(self) -> StorageRefsExtension[Link]: - return StorageRefsExtension.ext(self.stac_object) + def storage(self) -> StorageExtension[Link]: + return StorageExtension.ext(self.stac_object) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 15f0ff37b..b3d5c66d6 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -5,12 +5,10 @@ from __future__ import annotations -from abc import ABC from typing import ( Any, Generic, Literal, - SupportsIndex, TypeVar, cast, ) @@ -25,12 +23,15 @@ from pystac.extensions.hooks import ExtensionHooks from pystac.utils import StringEnum, get_required, map_opt -#: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Catalog` or -#: :class:`~pystac.Collection` -T = TypeVar("T", pystac.Item, pystac.Catalog, pystac.Collection) -#: Generalized version of :class:`~pystac.Asset`, :class:`~pystac.Link` or -#: :class:`~pystac.ItemAssetDefinition` -U = TypeVar("U", pystac.Asset, pystac.Link, pystac.ItemAssetDefinition) +T = TypeVar( + "T", + pystac.Catalog, + pystac.Collection, + pystac.Item, + pystac.Asset, + pystac.Link, + pystac.ItemAssetDefinition, +) SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v2.0.0/schema.json" PREFIX: str = "storage:" @@ -53,31 +54,47 @@ class StorageSchemeType(StringEnum): class StorageScheme: + """ + Helper class for storage scheme objects. + + Can set well-defined properties, or if needed, + any arbitrary property. + """ + + _known_fields = {"type", "platform", "region", "requester_pays"} _properties: dict[str, Any] def __init__(self, properties: dict[str, Any]): super().__setattr__("_properties", properties) - def __eq__(self, other: object) -> bool: - if not isinstance(other, StorageScheme): - return NotImplemented + def __setattr__(self, name: str, value: Any) -> None: + if hasattr(type(self), name): + object.__setattr__(self, name, value) + return + + if name in self._known_fields: + prop = getattr(type(self), name) + prop.fset(self, value) + return - return bool(self.__dict__["_properties"] == other.__dict__["_properties"]) + props = object.__getattribute__(self, "_properties") + props[name] = value def __getattr__(self, name: str) -> Any: - properties = self.__dict__["_properties"] - if name in properties: - return properties[name] - raise AttributeError(f"StorageScheme does not have attribute '{name}'") + props = object.__getattribute__(self, "_properties") - def __setattr__(self, name: str, value: Any) -> None: - if name.startswith("_") or hasattr(type(self), name): - super().__setattr__(name, value) - else: - self._properties[name] = value + if name in props: + return props[name] - def __reduce_ex__(self, protocol: SupportsIndex) -> Any: - return (self.__class__, (self.__dict__["_properties"],), None) + raise AttributeError(name) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, StorageScheme): + return NotImplemented + return bool(self._properties == other._properties) + + def __repr__(self) -> str: + return f"" def apply( self, @@ -85,7 +102,7 @@ def apply( platform: str, region: str | None = None, requester_pays: bool | None = None, - **kwargs: dict[str, Any], + **kwargs: Any, ) -> None: self.type = type self.platform = platform @@ -100,7 +117,7 @@ def create( platform: str, region: str | None = None, requester_pays: bool | None = None, - **kwargs: dict[str, Any], + **kwargs: Any, ) -> StorageScheme: """Set the properties for a new StorageScheme object. @@ -168,7 +185,7 @@ def region(self) -> str | None: return self._properties.get(REGION_PROP) @region.setter - def region(self, v: str) -> None: + def region(self, v: str | None) -> None: if v is not None: self._properties[REGION_PROP] = v else: @@ -182,7 +199,7 @@ def requester_pays(self) -> bool | None: return self._properties.get(REQUESTER_PAYS_PROP) @requester_pays.setter - def requester_pays(self, v: bool) -> None: + def requester_pays(self, v: bool | None) -> None: if v is not None: self._properties[REQUESTER_PAYS_PROP] = v else: @@ -198,95 +215,100 @@ def to_dict(self) -> dict[str, Any]: return self._properties -class _StorageExtension(ABC): - name: Literal["storage"] = "storage" - - @classmethod - def get_schema_uri(cls) -> str: - return SCHEMA_URI - - -class StorageSchemesExtension( - _StorageExtension, +class StorageExtension( Generic[T], PropertiesExtension, ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog], ): - """An abstract class that can be used to extend the properties of an - :class:`~pystac.Collection`, :class:`~pystac.Catalog`, or :class:`~pystac.Item` - with properties from the :stac-ext:`Storage Extension `. + """An class that can be used to extend the properties of an + :class:`~pystac.Catalog`, :class:`~pystac.Collection`, :class:`~pystac.Item`, + :class:`~pystac.Asset`, :class:`~pystac.Link`, or + :class:`~pystac.ItemAssetDefinition` with properties from the + :stac-ext:`Storage Extension `. This class is generic over the type of STAC Object to be extended (e.g. :class:`~pystac.Item`, :class:`~pystac.Collection`). - - To create a concrete instance of :class:`StorageSchemesExtension`, use the - :meth:`StorageSchemesExtension.ext` method. For example: + To create a concrete instance of :class:`StorageExtension`, use the + :meth:`StorageExtension.ext` method. For example: .. code-block:: python >>> item: pystac.Item = ... - >>> storage_ext = StorageSchemesExtension.ext(item) + >>> storage_ext = StorageExtension.ext(item) """ + name: Literal["storage"] = "storage" + + @classmethod + def get_schema_uri(cls) -> str: + return SCHEMA_URI + + # For type checking purposes only, these methods are overridden in mixins def apply( self, - schemes: dict[str, StorageScheme], + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, ) -> None: - """Applies Storage Extension properties to the extended - :class:`~pystac.Catalog`, :class:`~pystac.Collection`, - or :class:`~pystac.Item`. - - Args: - schemes (dict[str, StorageScheme]): Storage schemes used by Assets and Links - in the STAC Item, Catalog or Collection. - """ - self.schemes = schemes + raise NotImplementedError() @property def schemes(self) -> dict[str, StorageScheme]: - """Get or sets the schemes used by Assets and Links. - - Returns: - dict[str, StorageScheme]: storage schemes - """ - schemes: dict[str, dict[str, Any]] = get_required( - self.properties.get(SCHEMES_PROP), - self, - SCHEMES_PROP, - ) - return {k: StorageScheme(v) for k, v in schemes.items()} + raise NotImplementedError() @schemes.setter def schemes(self, v: dict[str, StorageScheme]) -> None: - v_trans = {k: c.to_dict() for k, c in v.items()} - self._set_property(SCHEMES_PROP, v_trans) + raise NotImplementedError() def add_scheme(self, key: str, scheme: StorageScheme) -> None: - try: - self.schemes = {**self.schemes, **{key: scheme}} - except RequiredPropertyMissing: - self.schemes = {key: scheme} + raise NotImplementedError() + + @property + def refs(self) -> list[str]: + raise NotImplementedError() + + @refs.setter + def refs(self, v: list[str]) -> None: + raise NotImplementedError() + + def add_ref(self, ref: str) -> None: + raise NotImplementedError() @classmethod - def ext(cls, obj: T, add_if_missing: bool = False) -> StorageSchemesExtension[T]: + def ext(cls, obj: T, add_if_missing: bool = False) -> StorageExtension[T]: """Extends the given STAC Object with properties from the :stac-ext:`Storage Extension `. This extension can be applied to instances of :class:`~pystac.Catalog`, - :class:`~pystac.Collection`, or :class:`~pystac.Item`. + :class:`~pystac.Collection`, :class:`~pystac.Item`, :class:`~pystac.Asset`, + :class:`~pystac.Link`, or :class:`~pystac.ItemAssetDefinition`. Raises: - pystac.ExtensionTypeError : If an invalid object type is passed. """ if isinstance(obj, pystac.Item): cls.ensure_has_extension(obj, add_if_missing) - return cast(StorageSchemesExtension[T], ItemStorageExtension(obj)) + return cast(StorageExtension[T], ItemStorageExtension(obj)) + elif isinstance(obj, pystac.Collection): cls.ensure_has_extension(obj, add_if_missing) - return cast(StorageSchemesExtension[T], CollectionStorageExtension(obj)) + return cast(StorageExtension[T], CollectionStorageExtension(obj)) + elif isinstance(obj, pystac.Catalog): cls.ensure_has_extension(obj, add_if_missing) - return cast(StorageSchemesExtension[T], CatalogStorageExtension(obj)) + return cast(StorageExtension[T], CatalogStorageExtension(obj)) + + elif isinstance(obj, pystac.Asset): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], AssetStorageExtension(obj)) + + elif isinstance(obj, pystac.Link): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], LinkStorageExtension(obj)) + + elif isinstance(obj, pystac.ItemAssetDefinition): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], ItemAssetsStorageExtension(obj)) + else: raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) @@ -299,21 +321,85 @@ def summaries( return SummariesStorageExtension(obj) -class ItemStorageExtension(StorageSchemesExtension[pystac.Item]): - """A concrete implementation of :class:`StorageSchemesExtension` on an - :class:`~pystac.Item` that extends the properties of the Item to include - properties defined in the :stac-ext:`Storage Extension `. +class _SchemesMixin: + """Mixin for objects that support Storage Schemes (Items, Collections, Catalogs).""" - This class should generally not be instantiated directly. Instead, call - :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Item` to extend it. - """ + properties: dict[str, Any] + _set_property: Any - item: pystac.Item - """The :class:`~pystac.Item` being extended.""" + def apply( + self, + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, + ) -> None: + if refs is not None: + raise ValueError("'refs' cannot be applied with this STAC object type.") + if schemes is None: + raise RequiredPropertyMissing( + self, + SCHEMES_PROP, + "'schemes' property is required for this object type.", + ) + self.schemes = schemes + + @property + def schemes(self) -> dict[str, StorageScheme]: + schemes_dict: dict[str, Any] = get_required( + self.properties.get(SCHEMES_PROP), self, SCHEMES_PROP + ) + return {k: StorageScheme(v) for k, v in schemes_dict.items()} + + @schemes.setter + def schemes(self, v: dict[str, StorageScheme]) -> None: + v_trans = {k: c.to_dict() for k, c in v.items()} + self._set_property(SCHEMES_PROP, v_trans) + + def add_scheme(self, key: str, scheme: StorageScheme) -> None: + current = self.properties.get(SCHEMES_PROP, {}) + current[key] = scheme.to_dict() + self._set_property(SCHEMES_PROP, current) + + +class _RefsMixin: + """Mixin for objects that support Storage Refs (Assets, Links).""" properties: dict[str, Any] - """The :class:`~pystac.Item` properties, including extension properties.""" + _set_property: Any + + def apply( + self, + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, + ) -> None: + if schemes is not None: + raise ValueError("'schemes' cannot be applied with this STAC object type.") + if refs is None: + raise RequiredPropertyMissing( + self, REFS_PROP, "'refs' property is required for this object type." + ) + self.refs = refs + + @property + def refs(self) -> list[str]: + return get_required(self.properties.get(REFS_PROP), self, REFS_PROP) + + @refs.setter + def refs(self, v: list[str]) -> None: + self._set_property(REFS_PROP, v) + def add_ref(self, ref: str) -> None: + try: + current = self.refs + if ref not in current: + current.append(ref) + self.refs = current + except RequiredPropertyMissing: + self.refs = [ref] + + +class ItemStorageExtension(_SchemesMixin, StorageExtension[pystac.Item]): def __init__(self, item: pystac.Item): self.item = item self.properties = item.properties @@ -322,13 +408,36 @@ def __repr__(self) -> str: return f"" -class CollectionStorageExtension(StorageSchemesExtension[pystac.Collection]): - """A concrete implementation of :class:`StorageSchemesExtension` on an +class CatalogStorageExtension(_SchemesMixin, StorageExtension[pystac.Catalog]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Catalog` that extends the properties of the Catalog to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.Catalog` to extend it. + """ + + catalog: pystac.Catalog + """The :class:`~pystac.Catalog` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Catalog` properties, including extension properties.""" + + def __init__(self, catalog: pystac.Catalog): + self.catalog = catalog + self.properties = catalog.extra_fields + + def __repr__(self) -> str: + return f"" + + +class CollectionStorageExtension(_SchemesMixin, StorageExtension[pystac.Collection]): + """A concrete implementation of :class:`StorageExtension` on an :class:`~pystac.Collection` that extends the properties of the Collection to include properties defined in the :stac-ext:`Storage Extension `. This class should generally not be instantiated directly. Instead, call - :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Collection` to extend it. + :meth:`StorageExtension.ext` on an :class:`~pystac.Collection` to extend it. """ collection: pystac.Collection @@ -345,125 +454,78 @@ def __repr__(self) -> str: return f"" -class CatalogStorageExtension(StorageSchemesExtension[pystac.Catalog]): - """A concrete implementation of :class:`StorageSchemesExtension` on an - :class:`~pystac.Catalog` that extends the properties of the Catalog to include +class AssetStorageExtension(_RefsMixin, StorageExtension[pystac.Asset]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Asset` that extends the properties of the Asset to include properties defined in the :stac-ext:`Storage Extension `. This class should generally not be instantiated directly. Instead, call - :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Catalog` to extend it. + :meth:`StorageExtension.ext` on an :class:`~pystac.Asset` to extend it. """ - catalog: pystac.Catalog - """The :class:`~pystac.Catalog` being extended.""" + asset: pystac.Asset + """The :class:`~pystac.Asset` being extended.""" properties: dict[str, Any] - """The :class:`~pystac.Catalog` properties, including extension properties.""" + """The :class:`~pystac.Asset` properties, including extension properties.""" - def __init__(self, catalog: pystac.Catalog): - self.catalog = catalog - self.properties = catalog.extra_fields + def __init__(self, asset: pystac.Asset): + self.asset = asset + self.properties = asset.extra_fields def __repr__(self) -> str: - return f"" - - -class StorageRefsExtension( - _StorageExtension, - Generic[U], - PropertiesExtension, - ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog], -): - def apply( - self, - refs: list[str], - ) -> None: - """Applies Storage Extension properties to the extended :class:`~pystac.Asset`, - :class:`~pystac.Link`, or :class:`~pystac.ItemAssetDefinition`. + return f"" - Args: - refs (list[str]): specifies which schemes in storage:schemes may be used to - access an Asset or Link. Each value must be one of the keys defined in - storage:schemes. - """ - self.refs = refs - - @property - def refs(self) -> list[str]: - """Get or sets the keys of the schemes that may be used to access an Asset - or Link. - Returns: - list[str] - """ - return get_required( - self.properties.get(REFS_PROP), - self, - REFS_PROP, - ) - - @refs.setter - def refs(self, v: list[str]) -> None: - self._set_property(REFS_PROP, v) +class LinkStorageExtension(_RefsMixin, StorageExtension[pystac.Link]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Link` that extends the properties of the Link to include + properties defined in the :stac-ext:`Storage Extension `. - def add_ref(self, ref: str) -> None: - try: - self.refs.append(ref) - except RequiredPropertyMissing: - self.refs = [ref] + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.Link` to extend it. + """ - @classmethod - def ext(cls, obj: U, add_if_missing: bool = False) -> StorageRefsExtension[U]: - """Extends the given STAC Object with properties from the :stac-ext:`Storage - Extension `. + link: pystac.Link + """The :class:`~pystac.Link` being extended.""" - This extension can be applied to instances of :class:`~pystac.Item` or - :class:`~pystac.Asset`. + properties: dict[str, Any] + """The :class:`~pystac.Link` properties, including extension properties.""" - Raises: + def __init__(self, link: pystac.Link): + self.link = link + self.properties = link.extra_fields - pystac.ExtensionTypeError : If an invalid object type is passed. - """ - if isinstance(obj, pystac.Asset): - cls.ensure_owner_has_extension(obj, add_if_missing) - return AssetStorageExtension(obj) - if isinstance(obj, pystac.Link): - cls.ensure_owner_has_extension(obj, add_if_missing) - return LinkStorageExtension(obj) - if isinstance(obj, pystac.ItemAssetDefinition): - cls.ensure_owner_has_extension(obj, add_if_missing) - return ItemAssetsStorageExtension(obj) - else: - raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) + def __repr__(self) -> str: + return f"" -class AssetStorageExtension(StorageRefsExtension[pystac.Asset]): - def __init__(self, asset: pystac.Asset): - self.asset_href = asset.href - self.properties = asset.extra_fields - if asset.owner and isinstance(asset.owner, pystac.Item): - self.additional_read_properties = [asset.owner.properties] +class ItemAssetsStorageExtension( + _RefsMixin, StorageExtension[pystac.ItemAssetDefinition] +): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.ItemAssetDefinition` that extends the properties of the + ItemAssetDefinition to include properties defined in the + :stac-ext:`Storage Extension `. - def __repr__(self) -> str: - return f"" + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.ItemAssetDefinition` + to extend it. + """ + item_asset: pystac.ItemAssetDefinition + """The :class:`~pystac.ItemAssetDefinition` being extended.""" -class ItemAssetsStorageExtension(StorageRefsExtension[pystac.ItemAssetDefinition]): properties: dict[str, Any] - asset_defn: pystac.ItemAssetDefinition + """The :class:`~pystac.ItemAssetDefinition` properties, + including extension properties.""" def __init__(self, item_asset: pystac.ItemAssetDefinition): - self.asset_defn = item_asset + self.item_asset = item_asset self.properties = item_asset.properties - -class LinkStorageExtension(StorageRefsExtension[pystac.Link]): - properties: dict[str, Any] - link: pystac.Link - - def __init__(self, link: pystac.Link): - self.link = link - self.properties = link.extra_fields + def __repr__(self) -> str: + return f"" class SummariesStorageExtension(SummariesExtension): @@ -477,15 +539,13 @@ def schemes(self) -> list[dict[str, StorageScheme]] | None: """Get or sets the summary of :attr:`StorageScheme.platform` values for this Collection. """ - v = map_opt( + return map_opt( lambda schemes: [ {k: StorageScheme(v) for k, v in x.items()} for x in schemes ], self.summaries.get_list(SCHEMES_PROP), ) - return v - @schemes.setter def schemes(self, v: list[dict[str, StorageScheme]] | None) -> None: self._set_summary( diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index 6f696c60d..43dd54f7e 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -8,10 +8,10 @@ import pystac from pystac import ExtensionTypeError, Item, ItemAssetDefinition from pystac.collection import Collection +from pystac.errors import RequiredPropertyMissing from pystac.extensions.storage import ( - StorageRefsExtension, + StorageExtension, StorageScheme, - StorageSchemesExtension, StorageSchemeType, ) from tests.utils import TestCases, assert_to_from_dict @@ -59,20 +59,20 @@ def test_to_from_dict() -> None: def test_add_to(sample_item: Item) -> None: - assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions # Check that the URI gets added to stac_extensions - StorageSchemesExtension.add_to(sample_item) - assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions + StorageExtension.add_to(sample_item) + assert StorageExtension.get_schema_uri() in sample_item.stac_extensions # Check that the URI only gets added once, regardless of how many times add_to # is called. - StorageSchemesExtension.add_to(sample_item) - StorageSchemesExtension.add_to(sample_item) + StorageExtension.add_to(sample_item) + StorageExtension.add_to(sample_item) uris = [ uri for uri in sample_item.stac_extensions - if uri == StorageSchemesExtension.get_schema_uri() + if uri == StorageExtension.get_schema_uri() ] assert len(uris) == 1 @@ -82,74 +82,65 @@ def test_validate_storage(naip_item: Item) -> None: naip_item.validate() -def test_extend_invalid_object() -> None: - link = pystac.Link("child", "https://some-domain.com/some/path/to.json") - - with pytest.raises(pystac.ExtensionTypeError): - StorageSchemesExtension.ext(link) # type: ignore - - def test_extension_not_implemented(sample_item: Item) -> None: # Should raise exception if Item does not include extension URI with pytest.raises(pystac.ExtensionNotImplemented): - _ = StorageSchemesExtension.ext(sample_item) + _ = StorageExtension.ext(sample_item) # Should raise exception if owning Item does not include extension URI asset = sample_item.assets["thumbnail"] with pytest.raises(pystac.ExtensionNotImplemented): - _ = StorageRefsExtension.ext(asset) + _ = StorageExtension.ext(asset) # Should succeed if Asset has no owner ownerless_asset = pystac.Asset.from_dict(asset.to_dict()) - _ = StorageRefsExtension.ext(ownerless_asset) + _ = StorageExtension.ext(ownerless_asset) def test_collection_ext_add_to(naip_collection: Collection) -> None: naip_collection.stac_extensions = [] - assert ( - StorageSchemesExtension.get_schema_uri() not in naip_collection.stac_extensions - ) + assert StorageExtension.get_schema_uri() not in naip_collection.stac_extensions - _ = StorageSchemesExtension.ext(naip_collection, add_if_missing=True) + _ = StorageExtension.ext(naip_collection, add_if_missing=True) - assert StorageSchemesExtension.get_schema_uri() in naip_collection.stac_extensions + assert StorageExtension.get_schema_uri() in naip_collection.stac_extensions def test_item_ext_add_to(sample_item: Item) -> None: - assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions - _ = StorageSchemesExtension.ext(sample_item, add_if_missing=True) + _ = StorageExtension.ext(sample_item, add_if_missing=True) - assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions + assert StorageExtension.get_schema_uri() in sample_item.stac_extensions def test_catalog_ext_add_to() -> None: catalog = pystac.Catalog("stac", "a catalog") - assert StorageSchemesExtension.get_schema_uri() not in catalog.stac_extensions + assert StorageExtension.get_schema_uri() not in catalog.stac_extensions - _ = StorageSchemesExtension.ext(catalog, add_if_missing=True) + _ = StorageExtension.ext(catalog, add_if_missing=True) - assert StorageSchemesExtension.get_schema_uri() in catalog.stac_extensions + assert StorageExtension.get_schema_uri() in catalog.stac_extensions def test_asset_ext_add_to(sample_item: Item) -> None: - assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions asset = sample_item.assets["thumbnail"] - _ = StorageRefsExtension.ext(asset, add_if_missing=True) + _ = StorageExtension.ext(asset, add_if_missing=True) - assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions + assert StorageExtension.get_schema_uri() in sample_item.stac_extensions def test_link_ext_add_to(sample_item: Item) -> None: - assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions asset = sample_item.links[0] - _ = StorageRefsExtension.ext(asset, add_if_missing=True) + _ = StorageExtension.ext(asset, add_if_missing=True) - assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions + assert StorageExtension.get_schema_uri() in sample_item.stac_extensions def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: @@ -157,22 +148,21 @@ def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: asset = pystac.Asset.from_dict(asset_dict) with pytest.raises(pystac.STACError): - _ = StorageRefsExtension.ext(asset, add_if_missing=True) + _ = StorageExtension.ext(asset, add_if_missing=True) def test_should_raise_exception_when_passing_invalid_extension_object() -> None: with pytest.raises( ExtensionTypeError, - match=r"^StorageRefsExtension does not apply to type 'object'$", + match=r"^StorageExtension does not apply to type 'object'$", ): # calling it wrong purposely so ---------v - StorageRefsExtension.ext(object()) # type: ignore + StorageExtension.ext(object()) # type: ignore def test_summaries_schemes(naip_collection: Collection) -> None: col_dict = naip_collection.to_dict() - storage_summaries = StorageSchemesExtension.summaries(naip_collection) - print(naip_collection.summaries) + storage_summaries = StorageExtension.summaries(naip_collection) # Get assert ( list( @@ -203,20 +193,18 @@ def test_summaries_adds_uri(naip_collection: Collection) -> None: pystac.ExtensionNotImplemented, match="Extension 'storage' is not implemented", ): - StorageSchemesExtension.summaries(naip_collection, add_if_missing=False) + StorageExtension.summaries(naip_collection, add_if_missing=False) - _ = StorageSchemesExtension.summaries(naip_collection, add_if_missing=True) + _ = StorageExtension.summaries(naip_collection, add_if_missing=True) - assert StorageSchemesExtension.get_schema_uri() in naip_collection.stac_extensions + assert StorageExtension.get_schema_uri() in naip_collection.stac_extensions - StorageSchemesExtension.remove_from(naip_collection) - assert ( - StorageSchemesExtension.get_schema_uri() not in naip_collection.stac_extensions - ) + StorageExtension.remove_from(naip_collection) + assert StorageExtension.get_schema_uri() not in naip_collection.stac_extensions def test_schemes_apply(naip_item: Item) -> None: - storage_ext = StorageSchemesExtension.ext(naip_item) + storage_ext = StorageExtension.ext(naip_item) new_key = random.choice(ascii_letters) new_type = random.choice(ascii_letters) new_platform = random.choice(ascii_letters) @@ -243,8 +231,8 @@ def test_schemes_apply(naip_item: Item) -> None: def test_refs_apply(naip_asset: pystac.Asset) -> None: test_refs = ["a_ref", "b_ref"] - storage_ext = StorageRefsExtension.ext(naip_asset) - storage_ext.apply(test_refs) + storage_ext = StorageExtension.ext(naip_asset) + storage_ext.apply(refs=test_refs) # Get assert storage_ext.refs == test_refs @@ -255,6 +243,47 @@ def test_refs_apply(naip_asset: pystac.Asset) -> None: assert storage_ext.refs == new_refs +def test_schemes_apply_raises(naip_item: Item) -> None: + storage_ext = StorageExtension.ext(naip_item) + + with pytest.raises( + ValueError, + match="'refs' cannot be applied with this STAC object type.", + ): + storage_ext.apply( + schemes={ + "a_key": StorageScheme.create("a_type", "a_platform"), + }, + refs=["a_ref"], + ) + with pytest.raises( + RequiredPropertyMissing, + match="'schemes' property is required for this object type.", + ): + storage_ext.apply(refs=None) + + +def test_refs_apply_raises(naip_asset: Item) -> None: + storage_ext = StorageExtension.ext(naip_asset) + + with pytest.raises( + ValueError, + match="'schemes' cannot be applied with this STAC object type.", + ): + storage_ext.apply( + schemes={ + "a_key": StorageScheme.create("a_type", "a_platform"), + }, + refs=["a_ref"], + ) + + with pytest.raises( + RequiredPropertyMissing, + match="'refs' property is required for this object type.", + ): + storage_ext.apply(schemes=None) + + def test_add_storage_scheme(naip_item: Item) -> None: storage_ext = naip_item.ext.storage storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) @@ -270,7 +299,7 @@ def test_add_refs(naip_item: Item) -> None: scheme_name = random.choice(ascii_letters) asset = naip_item.assets["GEOTIFF_AZURE_RGBIR"] storage_ext = asset.ext.storage - assert isinstance(storage_ext, StorageRefsExtension) + assert isinstance(storage_ext, StorageExtension) storage_ext.add_ref(scheme_name) assert scheme_name in storage_ext.refs @@ -315,4 +344,4 @@ def test_item_asset_accessor() -> None: item_asset = ItemAssetDefinition.create( title="title", description="desc", media_type="media", roles=["a_role"] ) - assert isinstance(item_asset.ext.storage, StorageRefsExtension) + assert isinstance(item_asset.ext.storage, StorageExtension) From 8d5e9967dd99f20d4383c1b19a06130c919fbcb6 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Sat, 6 Dec 2025 20:30:06 -0500 Subject: [PATCH 09/18] cast after get_required --- pystac/extensions/storage.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index b3d5c66d6..f8064d23b 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -152,10 +152,13 @@ def type(self) -> str: """ Get or set the required type property """ - return get_required( - self._properties.get(TYPE_PROP), - self, - TYPE_PROP, + return cast( + str, + get_required( + self._properties.get(TYPE_PROP), + self, + TYPE_PROP, + ), ) @type.setter @@ -167,10 +170,13 @@ def platform(self) -> str: """ Get or set the required platform property """ - return get_required( - self._properties.get(PLATFORM_PROP), - self, - PLATFORM_PROP, + return cast( + str, + get_required( + self._properties.get(PLATFORM_PROP), + self, + PLATFORM_PROP, + ), ) @platform.setter From 1e7974bcf0a9bd2f6c184ef9a13c9de30c756800 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:22:28 -0500 Subject: [PATCH 10/18] fix docs class --- docs/api/extensions.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/api/extensions.rst b/docs/api/extensions.rst index 19d17c84a..ce37d3215 100644 --- a/docs/api/extensions.rst +++ b/docs/api/extensions.rst @@ -30,8 +30,7 @@ pystac.extensions sar.SarExtension sat.SatExtension scientific.ScientificExtension - storage.StorageSchemesExtension - storage.StorageRefsExtension + storage.StorageExtension table.TableExtension timestamps.TimestampsExtension version.VersionExtension From d4bf741f097de2120beec1de36df58627cbd6aea Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Fri, 9 Jan 2026 11:54:22 -0500 Subject: [PATCH 11/18] fix: to_dict docstring --- pystac/extensions/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index f8064d23b..1eeab6903 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -216,7 +216,7 @@ def to_dict(self) -> dict[str, Any]: Returns the dictionary encoding of this object Returns: - dict[str, Any + dict[str, Any]: The dictionary encoding of this object """ return self._properties From 091c02d6f3ee479645f2c80024e2a81331d4191f Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Tue, 20 Jan 2026 14:23:49 -0500 Subject: [PATCH 12/18] add best effort migration from v1 to v2 --- pystac/extensions/storage.py | 148 ++++++++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 4 deletions(-) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 1eeab6903..6256aaba0 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -5,6 +5,7 @@ from __future__ import annotations +import warnings from typing import ( Any, Generic, @@ -21,6 +22,7 @@ SummariesExtension, ) from pystac.extensions.hooks import ExtensionHooks +from pystac.serialization.identify import STACJSONDescription, STACVersionID from pystac.utils import StringEnum, get_required, map_opt T = TypeVar( @@ -33,7 +35,12 @@ pystac.ItemAssetDefinition, ) -SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v2.0.0/schema.json" +SCHEMA_URI_PATTERN: str = ( + "https://stac-extensions.github.io/storage/v{version}/schema.json" +) +DEFAULT_VERSION: str = "2.0.0" +SUPPORTED_VERSIONS: list[str] = ["2.0.0", "1.0.0"] + PREFIX: str = "storage:" # Field names @@ -246,7 +253,8 @@ class StorageExtension( @classmethod def get_schema_uri(cls) -> str: - return SCHEMA_URI + return SCHEMA_URI_PATTERN.format(version=DEFAULT_VERSION) + # For type checking purposes only, these methods are overridden in mixins def apply( @@ -566,13 +574,145 @@ def schemes(self, v: list[dict[str, StorageScheme]] | None) -> None: class StorageExtensionHooks(ExtensionHooks): - schema_uri: str = SCHEMA_URI - prev_extension_ids: set[str] = set() + schema_uri: str = SCHEMA_URI_PATTERN.format(version=DEFAULT_VERSION) + prev_extension_ids = { + SCHEMA_URI_PATTERN.format(version=v) + for v in SUPPORTED_VERSIONS + if v != DEFAULT_VERSION + } stac_object_types = { pystac.STACObjectType.CATALOG, pystac.STACObjectType.COLLECTION, pystac.STACObjectType.ITEM, } + # Mapping from v1.0.0 platform enum values to v2.0.0 type identifiers + # Only AWS and Azure have defined v2.0.0 platform definitions + _PLATFORM_TYPE_MAP: dict[str, str] = { + "AWS": "aws-s3", + "AZURE": "ms-azure", + } + + # Mapping from v1.0.0 platform enum values to v2.0.0 platform URI templates + _PLATFORM_URI_MAP: dict[str, str] = { + "AWS": "https://{bucket}.s3.{region}.amazonaws.com", + "AZURE": "https://{account}.blob.core.windows.net", + } + + # Mapping from v1.0.0 platform enum values to scheme key prefixes + _PLATFORM_KEY_PREFIX: dict[str, str] = { + "AWS": "aws", + "AZURE": "azure", + } + + # Platforms that cannot be automatically migrated + _UNSUPPORTED_PLATFORMS: set[str] = {"GCP", "IBM", "ALIBABA", "ORACLE", "OTHER"} + + def migrate( + self, obj: dict[str, Any], version: STACVersionID, info: STACJSONDescription + ) -> None: + if SCHEMA_URI_PATTERN.format(version="1.0.0") in info.extensions: + props = obj.get("properties", obj) + + # v1 defined item level storage properties can + # be used across all assets + item_platform = props.get(PREFIX + "platform") + item_region = props.get(PREFIX + "region") + item_requester_pays = props.get(PREFIX + "requester_pays") + item_tier = props.get(PREFIX + "tier") + + schemes: dict[str, dict[str, Any]] = {} + scheme_hash_to_key: dict[int, str] = {} + assets_with_tier: list[str] = [] + unsupported_platforms: set[str] = set() + migrated_assets: list[str] = [] + + for asset_key, asset in obj.get("assets", {}).items(): + platform = asset.get(PREFIX + "platform", item_platform) + region = asset.get(PREFIX + "region", item_region) + requester_pays = asset.get( + PREFIX + "requester_pays", item_requester_pays + ) + tier = asset.get(PREFIX + "tier", item_tier) + + if tier is not None: + assets_with_tier.append(asset_key) + + # cannot migrate assets without a platform + if platform is None: + continue + + # cannot migrate assets with unsupported platforms + platform_upper = platform.upper() + if ( + platform_upper in self._UNSUPPORTED_PLATFORMS + or platform_upper not in self._PLATFORM_TYPE_MAP + ): + unsupported_platforms.add(platform_upper) + continue + + scheme: dict[str, Any] = { + "type": self._PLATFORM_TYPE_MAP[platform_upper], + "platform": self._PLATFORM_URI_MAP[platform_upper], + } + if region is not None: + scheme["region"] = region + if requester_pays is not None: + scheme["requester_pays"] = requester_pays + + # Deduplicate schemes by content hash + scheme_hash = hash(frozenset(scheme.items())) + + if scheme_hash in scheme_hash_to_key: + scheme_key = scheme_hash_to_key[scheme_hash] + else: + # Generate scheme key: provider-region or provider + # if key would collide, appends an int suffix + key_prefix = self._PLATFORM_KEY_PREFIX[platform_upper] + base_key = ( + f"{key_prefix}-{region.lower()}" if region else key_prefix + ) + + scheme_key = base_key + counter = 1 + + while scheme_key in schemes: + scheme_key = f"{base_key}-{counter}" + counter += 1 + + schemes[scheme_key] = scheme + scheme_hash_to_key[scheme_hash] = scheme_key + + asset.pop(PREFIX + "platform", None) + asset.pop(PREFIX + "region", None) + asset.pop(PREFIX + "requester_pays", None) + asset[REFS_PROP] = [scheme_key] + migrated_assets.append(asset_key) + + if assets_with_tier: + warnings.warn( + "storage:tier was removed in storage extension v2.0.0 and cannot " + f"be migrated. Property left in place for: {assets_with_tier}", + UserWarning, + ) + + if unsupported_platforms: + warnings.warn( + "The following platforms cannot be automatically migrated to " + f"storage extension v2.0.0: {unsupported_platforms}", + UserWarning, + ) + + # Only remove item-level properties if all assets were migrated + if migrated_assets and not unsupported_platforms: + props.pop(PREFIX + "platform", None) + props.pop(PREFIX + "region", None) + props.pop(PREFIX + "requester_pays", None) + + if schemes: + props[SCHEMES_PROP] = schemes + + super().migrate(obj, version, info) + STORAGE_EXTENSION_HOOKS: ExtensionHooks = StorageExtensionHooks() From 9f7aef364ecffc45ae794dd0167b8ac328023033 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:29:40 -0500 Subject: [PATCH 13/18] docs fix --- pystac/extensions/storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 6256aaba0..89236908f 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -25,6 +25,9 @@ from pystac.serialization.identify import STACJSONDescription, STACVersionID from pystac.utils import StringEnum, get_required, map_opt +#: Generalized version of :class:`~pystac.Catalog`, :class:`~pystac.Collection`, +#: :class:`~pystac.Item`, :class:`~pystac.Asset`, :class:`~pystac.Link`, +#: or :class:`~pystac.ItemAssetDefinition` T = TypeVar( "T", pystac.Catalog, From dace9d2520183f0b39cce7ab0a9b6cb6428491d0 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:30:53 -0500 Subject: [PATCH 14/18] fmt --- pystac/extensions/storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 89236908f..669696daf 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -258,7 +258,6 @@ class StorageExtension( def get_schema_uri(cls) -> str: return SCHEMA_URI_PATTERN.format(version=DEFAULT_VERSION) - # For type checking purposes only, these methods are overridden in mixins def apply( self, From a900c3bcdffecca60bd062cd2c80344bc58aad68 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Tue, 20 Jan 2026 16:05:35 -0500 Subject: [PATCH 15/18] update storage test cassettes --- .../test_storage/test_asset_platform.yaml | 52 --- .../test_storage/test_asset_region.yaml | 52 --- .../test_asset_requester_pays.yaml | 52 --- .../test_storage/test_asset_tier.yaml | 52 --- .../test_storage/test_refs_apply.yaml | 332 ---------------- .../test_storage/test_validate_storage.yaml | 372 ++++++------------ 6 files changed, 122 insertions(+), 790 deletions(-) delete mode 100644 tests/extensions/cassettes/test_storage/test_asset_platform.yaml delete mode 100644 tests/extensions/cassettes/test_storage/test_asset_region.yaml delete mode 100644 tests/extensions/cassettes/test_storage/test_asset_requester_pays.yaml delete mode 100644 tests/extensions/cassettes/test_storage/test_asset_tier.yaml delete mode 100644 tests/extensions/cassettes/test_storage/test_refs_apply.yaml diff --git a/tests/extensions/cassettes/test_storage/test_asset_platform.yaml b/tests/extensions/cassettes/test_storage/test_asset_platform.yaml deleted file mode 100644 index df56e24ae..000000000 --- a/tests/extensions/cassettes/test_storage/test_asset_platform.yaml +++ /dev/null @@ -1,52 +0,0 @@ -interactions: -- request: - body: null - headers: {} - method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_asset_region.yaml b/tests/extensions/cassettes/test_storage/test_asset_region.yaml deleted file mode 100644 index df56e24ae..000000000 --- a/tests/extensions/cassettes/test_storage/test_asset_region.yaml +++ /dev/null @@ -1,52 +0,0 @@ -interactions: -- request: - body: null - headers: {} - method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_asset_requester_pays.yaml b/tests/extensions/cassettes/test_storage/test_asset_requester_pays.yaml deleted file mode 100644 index df56e24ae..000000000 --- a/tests/extensions/cassettes/test_storage/test_asset_requester_pays.yaml +++ /dev/null @@ -1,52 +0,0 @@ -interactions: -- request: - body: null - headers: {} - method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_asset_tier.yaml b/tests/extensions/cassettes/test_storage/test_asset_tier.yaml deleted file mode 100644 index df56e24ae..000000000 --- a/tests/extensions/cassettes/test_storage/test_asset_tier.yaml +++ /dev/null @@ -1,52 +0,0 @@ -interactions: -- request: - body: null - headers: {} - method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_refs_apply.yaml b/tests/extensions/cassettes/test_storage/test_refs_apply.yaml deleted file mode 100644 index e5afbfe61..000000000 --- a/tests/extensions/cassettes/test_storage/test_refs_apply.yaml +++ /dev/null @@ -1,332 +0,0 @@ -interactions: -- request: - body: null - headers: - Connection: - - close - Host: - - stac-extensions.github.io - User-Agent: - - Python-urllib/3.10 - method: GET - uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": - \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n - \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": - {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n - \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is - the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": - [\n \"type\",\n \"properties\"\n ],\n \"properties\": - {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": - {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": - {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n - \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n - \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n - \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n - \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n - \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n - \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n - \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n - \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": - \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n - \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n - \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": - \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n - \ \"properties\": {\n \"storage:schemes\": {\n \"type\": - \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": - {\n \"required\": [\n \"type\",\n \"platform\"\n - \ ],\n \"properties\": {\n \"type\": - {\n \"title\": \"Type identifier\",\n \"type\": - \"string\"\n },\n \"platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"format\": - \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n - \ },\n \"region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n - \ \"requester_pays\": {\n \"type\": \"boolean\",\n - \ \"title\": \"Requester pays\",\n \"default\": - false\n }\n },\n \"allOf\": [\n {\n - \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n - \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n - \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n - \ }\n ],\n \"additionalProperties\": - true\n }\n },\n \"additionalProperties\": false\n - \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": - {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": - {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": - {\n \"type\": \"array\",\n \"items\": {\n \"type\": - \"string\",\n \"minLength\": 1\n }\n }\n },\n - \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n - \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n - \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n - \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": - {\n \"$ref\": \"#/definitions/refs_field\"\n }\n - \ }\n }\n ]\n }\n },\n \"links\": {\n - \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n - \ }\n }\n }\n}\n" - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '0' - Cache-Control: - - max-age=600 - Connection: - - close - Content-Length: - - '4259' - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 01 Jun 2025 22:16:48 GMT - ETag: - - '"6718ce4f-10a3"' - Last-Modified: - - Wed, 23 Oct 2024 10:22:07 GMT - Server: - - GitHub.com - Strict-Transport-Security: - - max-age=31556952 - Vary: - - Accept-Encoding - Via: - - 1.1 varnish - X-Cache: - - HIT - X-Cache-Hits: - - '1' - X-Fastly-Request-ID: - - af64a991591f50737fadd2e89371cc12c8a2bb7d - X-GitHub-Request-Id: - - 591F:3B4DB0:4904A90:4C8204C:683CAF52 - X-Served-By: - - cache-iad-kcgs7200048-IAD - X-Timer: - - S1748816209.717952,VS0,VE1 - expires: - - Sun, 01 Jun 2025 20:01:47 GMT - x-proxy-cache: - - MISS - status: - code: 200 - message: OK -- request: - body: null - headers: - Connection: - - close - Host: - - stac-extensions.github.io - User-Agent: - - Python-urllib/3.10 - method: GET - uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n - \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": - {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n - \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": - \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": - {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n - \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n - \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": - \"^[a-z0-9-]+$\"\n }\n }\n }\n}" - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '0' - Cache-Control: - - max-age=600 - Connection: - - close - Content-Length: - - '706' - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 01 Jun 2025 22:16:48 GMT - ETag: - - '"6718ce4f-2c2"' - Last-Modified: - - Wed, 23 Oct 2024 10:22:07 GMT - Server: - - GitHub.com - Strict-Transport-Security: - - max-age=31556952 - Vary: - - Accept-Encoding - Via: - - 1.1 varnish - X-Cache: - - HIT - X-Cache-Hits: - - '1' - X-Fastly-Request-ID: - - 43d2b7092055b40e12b92793975aae512e94642f - X-GitHub-Request-Id: - - 87AB:3BCE73:4C312A6:4FAEA0D:683CAF53 - X-Served-By: - - cache-iad-kiad7000035-IAD - X-Timer: - - S1748816209.780679,VS0,VE3 - expires: - - Sun, 01 Jun 2025 20:01:47 GMT - x-proxy-cache: - - MISS - status: - code: 200 - message: OK -- request: - body: null - headers: - Connection: - - close - Host: - - stac-extensions.github.io - User-Agent: - - Python-urllib/3.10 - method: GET - uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n - \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": - {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n - \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n - \ }\n}" - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '0' - Cache-Control: - - max-age=600 - Connection: - - close - Content-Length: - - '353' - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 01 Jun 2025 22:16:48 GMT - ETag: - - '"6718ce4f-161"' - Last-Modified: - - Wed, 23 Oct 2024 10:22:07 GMT - Server: - - GitHub.com - Strict-Transport-Security: - - max-age=31556952 - Vary: - - Accept-Encoding - Via: - - 1.1 varnish - X-Cache: - - HIT - X-Cache-Hits: - - '1' - X-Fastly-Request-ID: - - c0e4c57aae853bd12f2127008255fb0d13a187fd - X-GitHub-Request-Id: - - 3461:2BF8CE:4BDFA81:4F5D0D4:683CAF53 - X-Served-By: - - cache-iad-kiad7000157-IAD - X-Timer: - - S1748816209.846702,VS0,VE4 - expires: - - Sun, 01 Jun 2025 20:01:47 GMT - x-origin-cache: - - HIT - x-proxy-cache: - - MISS - status: - code: 200 - message: OK -- request: - body: null - headers: - Connection: - - close - Host: - - stac-extensions.github.io - User-Agent: - - Python-urllib/3.10 - method: GET - uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n - \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n - \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n - \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": - {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n - \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '0' - Cache-Control: - - max-age=600 - Connection: - - close - Content-Length: - - '469' - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 01 Jun 2025 22:16:48 GMT - ETag: - - '"6718ce4f-1d5"' - Last-Modified: - - Wed, 23 Oct 2024 10:22:07 GMT - Server: - - GitHub.com - Strict-Transport-Security: - - max-age=31556952 - Vary: - - Accept-Encoding - Via: - - 1.1 varnish - X-Cache: - - HIT - X-Cache-Hits: - - '1' - X-Fastly-Request-ID: - - ea7c52a0de24dedf14c95d765216ad65b6bfa746 - X-GitHub-Request-Id: - - D40E:3B1629:4B5EDFA:4EDC767:683CAF53 - X-Served-By: - - cache-iad-kiad7000135-IAD - X-Timer: - - S1748816209.915699,VS0,VE2 - expires: - - Sun, 01 Jun 2025 20:01:47 GMT - x-origin-cache: - - HIT - x-proxy-cache: - - MISS - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml index 1e0f1a489..9b5201037 100644 --- a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml +++ b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml @@ -3,61 +3,131 @@ interactions: body: null headers: {} method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json response: body: string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" headers: {} status: code: 200 message: OK - request: body: null - headers: - Connection: - - close - Host: - - stac-extensions.github.io - User-Agent: - - Python-urllib/3.10 + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} method: GET uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json response: @@ -121,63 +191,13 @@ interactions: \ }\n }\n ]\n }\n },\n \"links\": {\n \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n \ }\n }\n }\n}\n" - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '0' - Cache-Control: - - max-age=600 - Connection: - - close - Content-Length: - - '4259' - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 01 Jun 2025 22:16:48 GMT - ETag: - - '"6718ce4f-10a3"' - Last-Modified: - - Wed, 23 Oct 2024 10:22:07 GMT - Server: - - GitHub.com - Strict-Transport-Security: - - max-age=31556952 - Vary: - - Accept-Encoding - Via: - - 1.1 varnish - X-Cache: - - HIT - X-Cache-Hits: - - '0' - X-Fastly-Request-ID: - - 855a50e2b2d2411c04eaca800019a8413b70b9a8 - X-GitHub-Request-Id: - - 591F:3B4DB0:4904A90:4C8204C:683CAF52 - X-Served-By: - - cache-iad-kiad7000152-IAD - X-Timer: - - S1748816208.373228,VS0,VE14 - expires: - - Sun, 01 Jun 2025 20:01:47 GMT - x-proxy-cache: - - MISS + headers: {} status: code: 200 message: OK - request: body: null - headers: - Connection: - - close - Host: - - stac-extensions.github.io - User-Agent: - - Python-urllib/3.10 + headers: {} method: GET uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json response: @@ -192,63 +212,13 @@ interactions: \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": \"^[a-z0-9-]+$\"\n }\n }\n }\n}" - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '0' - Cache-Control: - - max-age=600 - Connection: - - close - Content-Length: - - '706' - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 01 Jun 2025 22:16:48 GMT - ETag: - - '"6718ce4f-2c2"' - Last-Modified: - - Wed, 23 Oct 2024 10:22:07 GMT - Server: - - GitHub.com - Strict-Transport-Security: - - max-age=31556952 - Vary: - - Accept-Encoding - Via: - - 1.1 varnish - X-Cache: - - HIT - X-Cache-Hits: - - '0' - X-Fastly-Request-ID: - - 278480db61c132221d05f66f4d67dd50f9669b9a - X-GitHub-Request-Id: - - 87AB:3BCE73:4C312A6:4FAEA0D:683CAF53 - X-Served-By: - - cache-iad-kiad7000064-IAD - X-Timer: - - S1748816208.451944,VS0,VE18 - expires: - - Sun, 01 Jun 2025 20:01:47 GMT - x-proxy-cache: - - MISS + headers: {} status: code: 200 message: OK - request: body: null - headers: - Connection: - - close - Host: - - stac-extensions.github.io - User-Agent: - - Python-urllib/3.10 + headers: {} method: GET uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json response: @@ -259,65 +229,13 @@ interactions: {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n \ }\n}" - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '0' - Cache-Control: - - max-age=600 - Connection: - - close - Content-Length: - - '353' - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 01 Jun 2025 22:16:48 GMT - ETag: - - '"6718ce4f-161"' - Last-Modified: - - Wed, 23 Oct 2024 10:22:07 GMT - Server: - - GitHub.com - Strict-Transport-Security: - - max-age=31556952 - Vary: - - Accept-Encoding - Via: - - 1.1 varnish - X-Cache: - - HIT - X-Cache-Hits: - - '0' - X-Fastly-Request-ID: - - 17cf88621af9473909a54f09b7280d23a8d03f5b - X-GitHub-Request-Id: - - 3461:2BF8CE:4BDFA81:4F5D0D4:683CAF53 - X-Served-By: - - cache-iad-kiad7000129-IAD - X-Timer: - - S1748816209.531423,VS0,VE21 - expires: - - Sun, 01 Jun 2025 20:01:47 GMT - x-origin-cache: - - HIT - x-proxy-cache: - - MISS + headers: {} status: code: 200 message: OK - request: body: null - headers: - Connection: - - close - Host: - - stac-extensions.github.io - User-Agent: - - Python-urllib/3.10 + headers: {} method: GET uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json response: @@ -329,53 +247,7 @@ interactions: \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '0' - Cache-Control: - - max-age=600 - Connection: - - close - Content-Length: - - '469' - Content-Type: - - application/json; charset=utf-8 - Date: - - Sun, 01 Jun 2025 22:16:48 GMT - ETag: - - '"6718ce4f-1d5"' - Last-Modified: - - Wed, 23 Oct 2024 10:22:07 GMT - Server: - - GitHub.com - Strict-Transport-Security: - - max-age=31556952 - Vary: - - Accept-Encoding - Via: - - 1.1 varnish - X-Cache: - - HIT - X-Cache-Hits: - - '0' - X-Fastly-Request-ID: - - ff0d43db7da3291d7786a451b605ce5d2e96e1c4 - X-GitHub-Request-Id: - - D40E:3B1629:4B5EDFA:4EDC767:683CAF53 - X-Served-By: - - cache-iad-kiad7000060-IAD - X-Timer: - - S1748816209.609643,VS0,VE15 - expires: - - Sun, 01 Jun 2025 20:01:47 GMT - x-origin-cache: - - HIT - x-proxy-cache: - - MISS + headers: {} status: code: 200 message: OK From 6156235bd2c45177ab92dd303f6a2f1c3cc74eab Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Thu, 22 Jan 2026 09:49:10 -0500 Subject: [PATCH 16/18] parse account / bucket from href --- pystac/extensions/storage.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 669696daf..909e3fced 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -5,6 +5,7 @@ from __future__ import annotations +import re import warnings from typing import ( Any, @@ -610,6 +611,10 @@ class StorageExtensionHooks(ExtensionHooks): # Platforms that cannot be automatically migrated _UNSUPPORTED_PLATFORMS: set[str] = {"GCP", "IBM", "ALIBABA", "ORACLE", "OTHER"} + # Regex patterns for parsing cloud storage URLs + _S3_URL_PATTERN = re.compile(r"^s3://([^/]+)/") + _AZURE_BLOB_PATTERN = re.compile(r"^https://([^.]+)\.blob\.core\.windows\.net/") + def migrate( self, obj: dict[str, Any], version: STACVersionID, info: STACJSONDescription ) -> None: @@ -626,6 +631,7 @@ def migrate( schemes: dict[str, dict[str, Any]] = {} scheme_hash_to_key: dict[int, str] = {} assets_with_tier: list[str] = [] + assets_failed_parsing: list[str] = [] unsupported_platforms: set[str] = set() migrated_assets: list[str] = [] @@ -636,6 +642,7 @@ def migrate( PREFIX + "requester_pays", item_requester_pays ) tier = asset.get(PREFIX + "tier", item_tier) + href = asset.get("href", "") if tier is not None: assets_with_tier.append(asset_key) @@ -662,6 +669,20 @@ def migrate( if requester_pays is not None: scheme["requester_pays"] = requester_pays + # Parse bucket/account info from href + if platform_upper == "AWS": + if s3_match := self._S3_URL_PATTERN.match(href): + scheme["bucket"] = s3_match.group(1) + else: + assets_failed_parsing.append(asset_key) + continue + elif platform_upper == "AZURE": + if azure_match := self._AZURE_BLOB_PATTERN.match(href): + scheme["account"] = azure_match.group(1) + else: + assets_failed_parsing.append(asset_key) + continue + # Deduplicate schemes by content hash scheme_hash = hash(frozenset(scheme.items())) @@ -688,6 +709,7 @@ def migrate( asset.pop(PREFIX + "platform", None) asset.pop(PREFIX + "region", None) asset.pop(PREFIX + "requester_pays", None) + asset.pop(PREFIX + "tier", None) asset[REFS_PROP] = [scheme_key] migrated_assets.append(asset_key) @@ -698,6 +720,13 @@ def migrate( UserWarning, ) + if assets_failed_parsing: + warnings.warn( + "Could not parse bucket/account from href. " + f"The following assets were not migrated: {assets_failed_parsing}", + UserWarning, + ) + if unsupported_platforms: warnings.warn( "The following platforms cannot be automatically migrated to " @@ -706,7 +735,7 @@ def migrate( ) # Only remove item-level properties if all assets were migrated - if migrated_assets and not unsupported_platforms: + if migrated_assets and not unsupported_platforms and not assets_failed_parsing: props.pop(PREFIX + "platform", None) props.pop(PREFIX + "region", None) props.pop(PREFIX + "requester_pays", None) From ce300e5b102ecad1f85bdfb67e8edac3cbe0d3d1 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Thu, 22 Jan 2026 09:50:49 -0500 Subject: [PATCH 17/18] add migration tests --- tests/data-files/storage/item-v1.0.0.json | 90 +++++++++++++++++++++++ tests/extensions/test_storage.py | 59 +++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 tests/data-files/storage/item-v1.0.0.json diff --git a/tests/data-files/storage/item-v1.0.0.json b/tests/data-files/storage/item-v1.0.0.json new file mode 100644 index 000000000..d75c398ad --- /dev/null +++ b/tests/data-files/storage/item-v1.0.0.json @@ -0,0 +1,90 @@ +{ + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + ], + "id": "an_image", + "collection": "a_collection", + "type": "Feature", + "bbox": [ + -97.7466867683867, + 30.278398961994966, + -97.72990596574927, + 30.288621181865743 + ], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -97.7466867683867, + 30.28754662370266 + ], + [ + -97.74555747279238, + 30.278398961994966 + ], + [ + -97.72990596574927, + 30.27972380176124 + ], + [ + -97.73085242627444, + 30.288621181865743 + ], + [ + -97.7466867683867, + 30.28754662370266 + ] + ] + ] + }, + "properties": { + "datetime": "2019-08-22T18:35:18+00:00", + "mission": "SOME_MISSION", + "platform": "SOME_2", + "instrument": "IN_1", + "gsd": 1 + }, + "assets": { + "GCP": { + "href": "gs://bucket/data.tif", + "type": "image/vnd.stac.geotiff", + "storage:platform": "GCP", + "storage:region": "us-central1", + "storage:tier": "STANDARD" + }, + "AWS": { + "href": "s3://bucket/data.png", + "type": "image/png", + "storage:platform": "AWS", + "storage:region": "us-west-2", + "storage:requester_pays": true, + "storage:tier": "Glacier" + }, + "AZURE": { + "href": "https://project.blob.core.windows.net/images/data.tif", + "type": "image/vnd.stac.geotiff", + "storage:platform": "AZURE", + "storage:region": "westus2", + "storage:tier": "archive" + }, + "AWS_2": { + "href": "s3://bucket2/data.png", + "type": "image/png", + "storage:platform": "AWS", + "storage:region": "us-west-2", + "storage:tier": "Glacier" + } + }, + "links": [ + { + "href": "https://example.com/examples/item.json", + "rel": "self" + }, + { + "href": "https://example.com/examples/item.json", + "rel": "collection" + } + ] +} \ No newline at end of file diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index 43dd54f7e..70edf8afa 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -18,6 +18,7 @@ NAIP_EXAMPLE_URI = TestCases.get_path("data-files/storage/item-naip.json") NAIP_COLLECTION_URI = TestCases.get_path("data-files/storage/collection-naip.json") +V1_MIGRATION_ITEM_URI = TestCases.get_path("data-files/storage/item-v1.0.0.json") @pytest.fixture @@ -30,6 +31,13 @@ def naip_collection() -> Collection: return Collection.from_file(NAIP_COLLECTION_URI) +@pytest.fixture +def v1_item() -> Item: + with open(V1_MIGRATION_ITEM_URI) as f: + item_dict = json.load(f) + return Item.from_dict(item_dict, migrate=False) + + @pytest.fixture def sample_scheme() -> StorageScheme: return StorageScheme.create( @@ -345,3 +353,54 @@ def test_item_asset_accessor() -> None: title="title", description="desc", media_type="media", roles=["a_role"] ) assert isinstance(item_asset.ext.storage, StorageExtension) + + +@pytest.mark.filterwarnings("ignore") +def test_migrate(v1_item: Item) -> None: + item = Item.from_dict( + v1_item.to_dict(include_self_link=False, transform_hrefs=False), migrate=True + ) + + # Check schemes were created at item level + assert "storage:schemes" in item.properties + schemes = item.properties["storage:schemes"] + + # AWS asset should be migrated + assert "storage:refs" in item.assets["AWS"].to_dict() + aws_refs = item.assets["AWS"].to_dict()["storage:refs"] + assert len(aws_refs) == 1 + assert aws_refs[0] in schemes + assert schemes[aws_refs[0]]["type"] == "aws-s3" + assert schemes[aws_refs[0]]["region"] == "us-west-2" + assert schemes[aws_refs[0]]["requester_pays"] is True + assert schemes[aws_refs[0]]["bucket"] == "bucket" + + # AWS_2 should a different scheme than AWS (same region, no requester_pays) + assert "storage:refs" in item.assets["AWS_2"].to_dict() + aws2_refs = item.assets["AWS_2"].to_dict()["storage:refs"] + assert aws2_refs != aws_refs + assert schemes[aws2_refs[0]]["bucket"] == "bucket2" + + # AZURE asset should be migrated + assert "storage:refs" in item.assets["AZURE"].to_dict() + azure_refs = item.assets["AZURE"].to_dict()["storage:refs"] + assert len(azure_refs) == 1 + assert azure_refs[0] in schemes + assert schemes[azure_refs[0]]["type"] == "ms-azure" + assert schemes[azure_refs[0]]["region"] == "westus2" + assert schemes[azure_refs[0]]["account"] == "project" + + # GCP asset should NOT be migrated (unsupported platform) + assert "storage:refs" not in item.assets["GCP"].to_dict() + assert "storage:platform" in item.assets["GCP"].to_dict() + + # Old properties should be removed from migrated assets + assert "storage:platform" not in item.assets["AWS"].to_dict() + assert "storage:region" not in item.assets["AWS"].to_dict() + assert "storage:platform" not in item.assets["AZURE"].to_dict() + + # storage:tier should be removed from migrated assets + assert "storage:tier" not in item.assets["AWS"].to_dict() + assert "storage:tier" not in item.assets["AZURE"].to_dict() + # but preserved for unmigrated assets + assert item.assets["GCP"].to_dict().get("storage:tier") == "STANDARD" From 6a4f054a28e5c68f092a92f0e9a35e5f4f68edff Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Thu, 22 Jan 2026 10:47:37 -0500 Subject: [PATCH 18/18] fmt --- pystac/extensions/storage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 909e3fced..75664bf89 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -735,7 +735,11 @@ def migrate( ) # Only remove item-level properties if all assets were migrated - if migrated_assets and not unsupported_platforms and not assets_failed_parsing: + if ( + migrated_assets + and not unsupported_platforms + and not assets_failed_parsing + ): props.pop(PREFIX + "platform", None) props.pop(PREFIX + "region", None) props.pop(PREFIX + "requester_pays", None)