diff --git a/pystac/extensions/ext.py b/pystac/extensions/ext.py index 84f60c39d..0b72e9feb 100644 --- a/pystac/extensions/ext.py +++ b/pystac/extensions/ext.py @@ -172,6 +172,10 @@ def render(self) -> dict[str, Render]: def sci(self) -> ScientificExtension[Collection]: return ScientificExtension.ext(self.stac_object) + @property + def storage(self) -> StorageExtension[Collection]: + return StorageExtension.ext(self.stac_object) + @property def table(self) -> TableExtension[Collection]: return TableExtension.ext(self.stac_object) @@ -432,6 +436,10 @@ class ItemAssetExt(_AssetExt[ItemAssetDefinition]): def mlm(self) -> MLMExtension[ItemAssetDefinition]: return MLMExtension.ext(self.stac_object) + @property + def storage(self) -> StorageExtension[ItemAssetDefinition]: + return StorageExtension.ext(self.stac_object) + @dataclass class LinkExt(_AssetsExt[Link]): @@ -444,3 +452,7 @@ class LinkExt(_AssetsExt[Link]): @property def file(self) -> FileExtension[Link]: return FileExtension.ext(self.stac_object) + + @property + def storage(self) -> StorageExtension[Link]: + return StorageExtension.ext(self.stac_object) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 4270a9dc3..75664bf89 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -5,7 +5,8 @@ from __future__ import annotations -from collections.abc import Iterable +import re +import warnings from typing import ( Any, Generic, @@ -15,148 +16,316 @@ ) import pystac +from pystac.errors import RequiredPropertyMissing from pystac.extensions.base import ( ExtensionManagementMixin, PropertiesExtension, SummariesExtension, ) from pystac.extensions.hooks import ExtensionHooks -from pystac.utils import StringEnum +from pystac.serialization.identify import STACJSONDescription, STACVersionID +from pystac.utils import StringEnum, get_required, map_opt + +#: Generalized version of :class:`~pystac.Catalog`, :class:`~pystac.Collection`, +#: :class:`~pystac.Item`, :class:`~pystac.Asset`, :class:`~pystac.Link`, +#: or :class:`~pystac.ItemAssetDefinition` +T = TypeVar( + "T", + pystac.Catalog, + pystac.Collection, + pystac.Item, + pystac.Asset, + pystac.Link, + pystac.ItemAssetDefinition, +) -#: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Asset` or -#: :class:`~pystac.ItemAssetDefinition` -T = TypeVar("T", pystac.Item, pystac.Asset, pystac.ItemAssetDefinition) +SCHEMA_URI_PATTERN: str = ( + "https://stac-extensions.github.io/storage/v{version}/schema.json" +) +DEFAULT_VERSION: str = "2.0.0" +SUPPORTED_VERSIONS: list[str] = ["2.0.0", "1.0.0"] -SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v1.0.0/schema.json" PREFIX: str = "storage:" # Field names -PLATFORM_PROP: str = PREFIX + "platform" -REGION_PROP: str = PREFIX + "region" -REQUESTER_PAYS_PROP: str = PREFIX + "requester_pays" -TIER_PROP: str = PREFIX + "tier" - +REFS_PROP: str = PREFIX + "refs" +SCHEMES_PROP: str = PREFIX + "schemes" -class CloudPlatform(StringEnum): - ALIBABA = "ALIBABA" - AWS = "AWS" - AZURE = "AZURE" - GCP = "GCP" - IBM = "IBM" - ORACLE = "ORACLE" - OTHER = "OTHER" +# Storage scheme object names +TYPE_PROP: str = "type" +PLATFORM_PROP: str = "platform" +REGION_PROP: str = "region" +REQUESTER_PAYS_PROP: str = "requester_pays" -class StorageExtension( - Generic[T], - PropertiesExtension, - ExtensionManagementMixin[pystac.Item | pystac.Collection], -): - """An abstract class that can be used to extend the properties of an - :class:`~pystac.Item` or :class:`~pystac.Asset` with properties from the - :stac-ext:`Storage Extension `. This class is generic over the type of - STAC Object to be extended (e.g. :class:`~pystac.Item`, - :class:`~pystac.Asset`). +class StorageSchemeType(StringEnum): + AWS_S3 = "aws-s3" + CUSTOM_S3 = "custom-s3" + AZURE = "ms-azure" - To create a concrete instance of :class:`StorageExtension`, use the - :meth:`StorageExtension.ext` method. For example: - .. code-block:: python +class StorageScheme: + """ + Helper class for storage scheme objects. - >>> item: pystac.Item = ... - >>> storage_ext = StorageExtension.ext(item) + Can set well-defined properties, or if needed, + any arbitrary property. """ - name: Literal["storage"] = "storage" + _known_fields = {"type", "platform", "region", "requester_pays"} + _properties: dict[str, Any] + + def __init__(self, properties: dict[str, Any]): + super().__setattr__("_properties", properties) + + def __setattr__(self, name: str, value: Any) -> None: + if hasattr(type(self), name): + object.__setattr__(self, name, value) + return + + if name in self._known_fields: + prop = getattr(type(self), name) + prop.fset(self, value) + return + + props = object.__getattribute__(self, "_properties") + props[name] = value + + def __getattr__(self, name: str) -> Any: + props = object.__getattribute__(self, "_properties") + + if name in props: + return props[name] + + raise AttributeError(name) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, StorageScheme): + return NotImplemented + return bool(self._properties == other._properties) + + def __repr__(self) -> str: + return f"" def apply( self, - platform: CloudPlatform | None = None, + type: str, + platform: str, region: str | None = None, requester_pays: bool | None = None, - tier: str | None = None, + **kwargs: Any, ) -> None: - """Applies Storage Extension properties to the extended :class:`~pystac.Item` or - :class:`~pystac.Asset`. - - Args: - platform (str, CloudPlatform) : The cloud provider where data is stored. - region (str) : The region where the data is stored. Relevant to speed of - access and inter-region egress costs (as defined by PaaS provider). - requester_pays (bool) : Is the data requester pays or is it data - manager/cloud provider pays. - tier (str) : The title for the tier type (as defined by PaaS provider). - """ + self.type = type self.platform = platform self.region = region self.requester_pays = requester_pays - self.tier = tier + self._properties.update(kwargs) - @property - def platform(self) -> CloudPlatform | None: - """Get or sets the cloud provider where data is stored. + @classmethod + def create( + cls, + type: str, + platform: str, + region: str | None = None, + requester_pays: bool | None = None, + **kwargs: Any, + ) -> StorageScheme: + """Set the properties for a new StorageScheme object. + + Additional properties can be set through kwargs to fulfill + any additional variables in a templated uri. + + Args: + type (str): Type identifier for the platform. + platform (str): The cloud provider where data is stored as URI or URI + template to the API. + region (str | None): The region where the data is stored. + Defaults to None. + requester_pays (bool | None): requester pays or data manager/cloud + provider pays. Defaults to None. + kwargs (dict[str | Any]): Additional properties to set on scheme Returns: - str or None + StorageScheme: storage scheme + """ + c = cls({}) + c.apply( + type=type, + platform=platform, + region=region, + requester_pays=requester_pays, + **kwargs, + ) + return c + + @property + def type(self) -> str: + """ + Get or set the required type property + """ + return cast( + str, + get_required( + self._properties.get(TYPE_PROP), + self, + TYPE_PROP, + ), + ) + + @type.setter + def type(self, v: str) -> None: + self._properties[TYPE_PROP] = v + + @property + def platform(self) -> str: + """ + Get or set the required platform property """ - return self._get_property(PLATFORM_PROP, CloudPlatform) + return cast( + str, + get_required( + self._properties.get(PLATFORM_PROP), + self, + PLATFORM_PROP, + ), + ) @platform.setter - def platform(self, v: CloudPlatform | None) -> None: - self._set_property(PLATFORM_PROP, v) + def platform(self, v: str) -> None: + self._properties[PLATFORM_PROP] = v @property def region(self) -> str | None: - """Gets or sets the region where the data is stored. Relevant to speed of - access and inter-region egress costs (as defined by PaaS provider).""" - return self._get_property(REGION_PROP, str) + """ + Get or set the optional region property + """ + return self._properties.get(REGION_PROP) @region.setter def region(self, v: str | None) -> None: - self._set_property(REGION_PROP, v) + if v is not None: + self._properties[REGION_PROP] = v + else: + self._properties.pop(REGION_PROP, None) @property def requester_pays(self) -> bool | None: - # This value "defaults to false", according to the extension spec. - return self._get_property(REQUESTER_PAYS_PROP, bool) + """ + Get or set the optional requester_pays property + """ + return self._properties.get(REQUESTER_PAYS_PROP) @requester_pays.setter def requester_pays(self, v: bool | None) -> None: - self._set_property(REQUESTER_PAYS_PROP, v) + if v is not None: + self._properties[REQUESTER_PAYS_PROP] = v + else: + self._properties.pop(REQUESTER_PAYS_PROP, None) - @property - def tier(self) -> str | None: - return self._get_property(TIER_PROP, str) + def to_dict(self) -> dict[str, Any]: + """ + Returns the dictionary encoding of this object + + Returns: + dict[str, Any]: The dictionary encoding of this object + """ + return self._properties + + +class StorageExtension( + Generic[T], + PropertiesExtension, + ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog], +): + """An class that can be used to extend the properties of an + :class:`~pystac.Catalog`, :class:`~pystac.Collection`, :class:`~pystac.Item`, + :class:`~pystac.Asset`, :class:`~pystac.Link`, or + :class:`~pystac.ItemAssetDefinition` with properties from the + :stac-ext:`Storage Extension `. + This class is generic over the type of STAC Object to be extended (e.g. + :class:`~pystac.Item`, :class:`~pystac.Collection`). + To create a concrete instance of :class:`StorageExtension`, use the + :meth:`StorageExtension.ext` method. For example: + + .. code-block:: python + + >>> item: pystac.Item = ... + >>> storage_ext = StorageExtension.ext(item) + """ - @tier.setter - def tier(self, v: str | None) -> None: - self._set_property(TIER_PROP, v) + name: Literal["storage"] = "storage" @classmethod def get_schema_uri(cls) -> str: - return SCHEMA_URI + return SCHEMA_URI_PATTERN.format(version=DEFAULT_VERSION) + + # For type checking purposes only, these methods are overridden in mixins + def apply( + self, + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, + ) -> None: + raise NotImplementedError() + + @property + def schemes(self) -> dict[str, StorageScheme]: + raise NotImplementedError() + + @schemes.setter + def schemes(self, v: dict[str, StorageScheme]) -> None: + raise NotImplementedError() + + def add_scheme(self, key: str, scheme: StorageScheme) -> None: + raise NotImplementedError() + + @property + def refs(self) -> list[str]: + raise NotImplementedError() + + @refs.setter + def refs(self, v: list[str]) -> None: + raise NotImplementedError() + + def add_ref(self, ref: str) -> None: + raise NotImplementedError() @classmethod def ext(cls, obj: T, add_if_missing: bool = False) -> StorageExtension[T]: """Extends the given STAC Object with properties from the :stac-ext:`Storage Extension `. - This extension can be applied to instances of :class:`~pystac.Item` or - :class:`~pystac.Asset`. + This extension can be applied to instances of :class:`~pystac.Catalog`, + :class:`~pystac.Collection`, :class:`~pystac.Item`, :class:`~pystac.Asset`, + :class:`~pystac.Link`, or :class:`~pystac.ItemAssetDefinition`. Raises: - pystac.ExtensionTypeError : If an invalid object type is passed. """ if isinstance(obj, pystac.Item): cls.ensure_has_extension(obj, add_if_missing) return cast(StorageExtension[T], ItemStorageExtension(obj)) + + elif isinstance(obj, pystac.Collection): + cls.ensure_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], CollectionStorageExtension(obj)) + + elif isinstance(obj, pystac.Catalog): + cls.ensure_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], CatalogStorageExtension(obj)) + elif isinstance(obj, pystac.Asset): cls.ensure_owner_has_extension(obj, add_if_missing) return cast(StorageExtension[T], AssetStorageExtension(obj)) + + elif isinstance(obj, pystac.Link): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(StorageExtension[T], LinkStorageExtension(obj)) + elif isinstance(obj, pystac.ItemAssetDefinition): cls.ensure_owner_has_extension(obj, add_if_missing) return cast(StorageExtension[T], ItemAssetsStorageExtension(obj)) + else: raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) @@ -169,21 +338,85 @@ def summaries( return SummariesStorageExtension(obj) -class ItemStorageExtension(StorageExtension[pystac.Item]): - """A concrete implementation of :class:`StorageExtension` on an - :class:`~pystac.Item` that extends the properties of the Item to include - properties defined in the :stac-ext:`Storage Extension `. +class _SchemesMixin: + """Mixin for objects that support Storage Schemes (Items, Collections, Catalogs).""" - This class should generally not be instantiated directly. Instead, call - :meth:`StorageExtension.ext` on an :class:`~pystac.Item` to extend it. - """ + properties: dict[str, Any] + _set_property: Any - item: pystac.Item - """The :class:`~pystac.Item` being extended.""" + def apply( + self, + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, + ) -> None: + if refs is not None: + raise ValueError("'refs' cannot be applied with this STAC object type.") + if schemes is None: + raise RequiredPropertyMissing( + self, + SCHEMES_PROP, + "'schemes' property is required for this object type.", + ) + self.schemes = schemes + + @property + def schemes(self) -> dict[str, StorageScheme]: + schemes_dict: dict[str, Any] = get_required( + self.properties.get(SCHEMES_PROP), self, SCHEMES_PROP + ) + return {k: StorageScheme(v) for k, v in schemes_dict.items()} + + @schemes.setter + def schemes(self, v: dict[str, StorageScheme]) -> None: + v_trans = {k: c.to_dict() for k, c in v.items()} + self._set_property(SCHEMES_PROP, v_trans) + + def add_scheme(self, key: str, scheme: StorageScheme) -> None: + current = self.properties.get(SCHEMES_PROP, {}) + current[key] = scheme.to_dict() + self._set_property(SCHEMES_PROP, current) + + +class _RefsMixin: + """Mixin for objects that support Storage Refs (Assets, Links).""" properties: dict[str, Any] - """The :class:`~pystac.Item` properties, including extension properties.""" + _set_property: Any + + def apply( + self, + *, + schemes: dict[str, StorageScheme] | None = None, + refs: list[str] | None = None, + ) -> None: + if schemes is not None: + raise ValueError("'schemes' cannot be applied with this STAC object type.") + if refs is None: + raise RequiredPropertyMissing( + self, REFS_PROP, "'refs' property is required for this object type." + ) + self.refs = refs + @property + def refs(self) -> list[str]: + return get_required(self.properties.get(REFS_PROP), self, REFS_PROP) + + @refs.setter + def refs(self, v: list[str]) -> None: + self._set_property(REFS_PROP, v) + + def add_ref(self, ref: str) -> None: + try: + current = self.refs + if ref not in current: + current.append(ref) + self.refs = current + except RequiredPropertyMissing: + self.refs = [ref] + + +class ItemStorageExtension(_SchemesMixin, StorageExtension[pystac.Item]): def __init__(self, item: pystac.Item): self.item = item self.properties = item.properties @@ -192,43 +425,125 @@ def __repr__(self) -> str: return f"" -class AssetStorageExtension(StorageExtension[pystac.Asset]): +class CatalogStorageExtension(_SchemesMixin, StorageExtension[pystac.Catalog]): """A concrete implementation of :class:`StorageExtension` on an - :class:`~pystac.Asset` that extends the Asset fields to include properties defined - in the :stac-ext:`Storage Extension `. + :class:`~pystac.Catalog` that extends the properties of the Catalog to include + properties defined in the :stac-ext:`Storage Extension `. This class should generally not be instantiated directly. Instead, call - :meth:`StorageExtension.ext` on an :class:`~pystac.Asset` to extend it. + :meth:`StorageExtension.ext` on an :class:`~pystac.Catalog` to extend it. + """ + + catalog: pystac.Catalog + """The :class:`~pystac.Catalog` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Catalog` properties, including extension properties.""" + + def __init__(self, catalog: pystac.Catalog): + self.catalog = catalog + self.properties = catalog.extra_fields + + def __repr__(self) -> str: + return f"" + + +class CollectionStorageExtension(_SchemesMixin, StorageExtension[pystac.Collection]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Collection` that extends the properties of the Collection to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.Collection` to extend it. """ - asset_href: str - """The ``href`` value of the :class:`~pystac.Asset` being extended.""" + collection: pystac.Collection + """The :class:`~pystac.Collection` being extended.""" properties: dict[str, Any] - """The :class:`~pystac.Asset` fields, including extension properties.""" + """The :class:`~pystac.Collection` properties, including extension properties.""" + + def __init__(self, collection: pystac.Collection): + self.collection = collection + self.properties = collection.extra_fields + + def __repr__(self) -> str: + return f"" + - additional_read_properties: Iterable[dict[str, Any]] | None = None - """If present, this will be a list containing 1 dictionary representing the - properties of the owning :class:`~pystac.Item`.""" +class AssetStorageExtension(_RefsMixin, StorageExtension[pystac.Asset]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Asset` that extends the properties of the Asset to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.Asset` to extend it. + """ + + asset: pystac.Asset + """The :class:`~pystac.Asset` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Asset` properties, including extension properties.""" def __init__(self, asset: pystac.Asset): - self.asset_href = asset.href + self.asset = asset self.properties = asset.extra_fields - if asset.owner and isinstance(asset.owner, pystac.Item): - self.additional_read_properties = [asset.owner.properties] def __repr__(self) -> str: - return f"" + return f"" -class ItemAssetsStorageExtension(StorageExtension[pystac.ItemAssetDefinition]): +class LinkStorageExtension(_RefsMixin, StorageExtension[pystac.Link]): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.Link` that extends the properties of the Link to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.Link` to extend it. + """ + + link: pystac.Link + """The :class:`~pystac.Link` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Link` properties, including extension properties.""" + + def __init__(self, link: pystac.Link): + self.link = link + self.properties = link.extra_fields + + def __repr__(self) -> str: + return f"" + + +class ItemAssetsStorageExtension( + _RefsMixin, StorageExtension[pystac.ItemAssetDefinition] +): + """A concrete implementation of :class:`StorageExtension` on an + :class:`~pystac.ItemAssetDefinition` that extends the properties of the + ItemAssetDefinition to include properties defined in the + :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageExtension.ext` on an :class:`~pystac.ItemAssetDefinition` + to extend it. + """ + + item_asset: pystac.ItemAssetDefinition + """The :class:`~pystac.ItemAssetDefinition` being extended.""" + properties: dict[str, Any] - asset_defn: pystac.ItemAssetDefinition + """The :class:`~pystac.ItemAssetDefinition` properties, + including extension properties.""" def __init__(self, item_asset: pystac.ItemAssetDefinition): - self.asset_defn = item_asset + self.item_asset = item_asset self.properties = item_asset.properties + def __repr__(self) -> str: + return f"" + class SummariesStorageExtension(SummariesExtension): """A concrete implementation of :class:`~pystac.extensions.base.SummariesExtension` @@ -237,57 +552,202 @@ class SummariesStorageExtension(SummariesExtension): """ @property - def platform(self) -> list[CloudPlatform] | None: - """Get or sets the summary of :attr:`StorageExtension.platform` values + def schemes(self) -> list[dict[str, StorageScheme]] | None: + """Get or sets the summary of :attr:`StorageScheme.platform` values for this Collection. """ - return self.summaries.get_list(PLATFORM_PROP) - - @platform.setter - def platform(self, v: list[CloudPlatform] | None) -> None: - self._set_summary(PLATFORM_PROP, v) + return map_opt( + lambda schemes: [ + {k: StorageScheme(v) for k, v in x.items()} for x in schemes + ], + self.summaries.get_list(SCHEMES_PROP), + ) + + @schemes.setter + def schemes(self, v: list[dict[str, StorageScheme]] | None) -> None: + self._set_summary( + SCHEMES_PROP, + map_opt( + lambda schemes: [ + {k: c.to_dict() for k, c in x.items()} for x in schemes + ], + v, + ), + ) - @property - def region(self) -> list[str] | None: - """Get or sets the summary of :attr:`StorageExtension.region` values - for this Collection. - """ - return self.summaries.get_list(REGION_PROP) - @region.setter - def region(self, v: list[str] | None) -> None: - self._set_summary(REGION_PROP, v) +class StorageExtensionHooks(ExtensionHooks): + schema_uri: str = SCHEMA_URI_PATTERN.format(version=DEFAULT_VERSION) + prev_extension_ids = { + SCHEMA_URI_PATTERN.format(version=v) + for v in SUPPORTED_VERSIONS + if v != DEFAULT_VERSION + } + stac_object_types = { + pystac.STACObjectType.CATALOG, + pystac.STACObjectType.COLLECTION, + pystac.STACObjectType.ITEM, + } - @property - def requester_pays(self) -> list[bool] | None: - """Get or sets the summary of :attr:`StorageExtension.requester_pays` values - for this Collection. - """ - return self.summaries.get_list(REQUESTER_PAYS_PROP) + # Mapping from v1.0.0 platform enum values to v2.0.0 type identifiers + # Only AWS and Azure have defined v2.0.0 platform definitions + _PLATFORM_TYPE_MAP: dict[str, str] = { + "AWS": "aws-s3", + "AZURE": "ms-azure", + } - @requester_pays.setter - def requester_pays(self, v: list[bool] | None) -> None: - self._set_summary(REQUESTER_PAYS_PROP, v) + # Mapping from v1.0.0 platform enum values to v2.0.0 platform URI templates + _PLATFORM_URI_MAP: dict[str, str] = { + "AWS": "https://{bucket}.s3.{region}.amazonaws.com", + "AZURE": "https://{account}.blob.core.windows.net", + } - @property - def tier(self) -> list[str] | None: - """Get or sets the summary of :attr:`StorageExtension.tier` values - for this Collection. - """ - return self.summaries.get_list(TIER_PROP) + # Mapping from v1.0.0 platform enum values to scheme key prefixes + _PLATFORM_KEY_PREFIX: dict[str, str] = { + "AWS": "aws", + "AZURE": "azure", + } - @tier.setter - def tier(self, v: list[str] | None) -> None: - self._set_summary(TIER_PROP, v) + # Platforms that cannot be automatically migrated + _UNSUPPORTED_PLATFORMS: set[str] = {"GCP", "IBM", "ALIBABA", "ORACLE", "OTHER"} + # Regex patterns for parsing cloud storage URLs + _S3_URL_PATTERN = re.compile(r"^s3://([^/]+)/") + _AZURE_BLOB_PATTERN = re.compile(r"^https://([^.]+)\.blob\.core\.windows\.net/") -class StorageExtensionHooks(ExtensionHooks): - schema_uri: str = SCHEMA_URI - prev_extension_ids: set[str] = set() - stac_object_types = { - pystac.STACObjectType.COLLECTION, - pystac.STACObjectType.ITEM, - } + def migrate( + self, obj: dict[str, Any], version: STACVersionID, info: STACJSONDescription + ) -> None: + if SCHEMA_URI_PATTERN.format(version="1.0.0") in info.extensions: + props = obj.get("properties", obj) + + # v1 defined item level storage properties can + # be used across all assets + item_platform = props.get(PREFIX + "platform") + item_region = props.get(PREFIX + "region") + item_requester_pays = props.get(PREFIX + "requester_pays") + item_tier = props.get(PREFIX + "tier") + + schemes: dict[str, dict[str, Any]] = {} + scheme_hash_to_key: dict[int, str] = {} + assets_with_tier: list[str] = [] + assets_failed_parsing: list[str] = [] + unsupported_platforms: set[str] = set() + migrated_assets: list[str] = [] + + for asset_key, asset in obj.get("assets", {}).items(): + platform = asset.get(PREFIX + "platform", item_platform) + region = asset.get(PREFIX + "region", item_region) + requester_pays = asset.get( + PREFIX + "requester_pays", item_requester_pays + ) + tier = asset.get(PREFIX + "tier", item_tier) + href = asset.get("href", "") + + if tier is not None: + assets_with_tier.append(asset_key) + + # cannot migrate assets without a platform + if platform is None: + continue + + # cannot migrate assets with unsupported platforms + platform_upper = platform.upper() + if ( + platform_upper in self._UNSUPPORTED_PLATFORMS + or platform_upper not in self._PLATFORM_TYPE_MAP + ): + unsupported_platforms.add(platform_upper) + continue + + scheme: dict[str, Any] = { + "type": self._PLATFORM_TYPE_MAP[platform_upper], + "platform": self._PLATFORM_URI_MAP[platform_upper], + } + if region is not None: + scheme["region"] = region + if requester_pays is not None: + scheme["requester_pays"] = requester_pays + + # Parse bucket/account info from href + if platform_upper == "AWS": + if s3_match := self._S3_URL_PATTERN.match(href): + scheme["bucket"] = s3_match.group(1) + else: + assets_failed_parsing.append(asset_key) + continue + elif platform_upper == "AZURE": + if azure_match := self._AZURE_BLOB_PATTERN.match(href): + scheme["account"] = azure_match.group(1) + else: + assets_failed_parsing.append(asset_key) + continue + + # Deduplicate schemes by content hash + scheme_hash = hash(frozenset(scheme.items())) + + if scheme_hash in scheme_hash_to_key: + scheme_key = scheme_hash_to_key[scheme_hash] + else: + # Generate scheme key: provider-region or provider + # if key would collide, appends an int suffix + key_prefix = self._PLATFORM_KEY_PREFIX[platform_upper] + base_key = ( + f"{key_prefix}-{region.lower()}" if region else key_prefix + ) + + scheme_key = base_key + counter = 1 + + while scheme_key in schemes: + scheme_key = f"{base_key}-{counter}" + counter += 1 + + schemes[scheme_key] = scheme + scheme_hash_to_key[scheme_hash] = scheme_key + + asset.pop(PREFIX + "platform", None) + asset.pop(PREFIX + "region", None) + asset.pop(PREFIX + "requester_pays", None) + asset.pop(PREFIX + "tier", None) + asset[REFS_PROP] = [scheme_key] + migrated_assets.append(asset_key) + + if assets_with_tier: + warnings.warn( + "storage:tier was removed in storage extension v2.0.0 and cannot " + f"be migrated. Property left in place for: {assets_with_tier}", + UserWarning, + ) + + if assets_failed_parsing: + warnings.warn( + "Could not parse bucket/account from href. " + f"The following assets were not migrated: {assets_failed_parsing}", + UserWarning, + ) + + if unsupported_platforms: + warnings.warn( + "The following platforms cannot be automatically migrated to " + f"storage extension v2.0.0: {unsupported_platforms}", + UserWarning, + ) + + # Only remove item-level properties if all assets were migrated + if ( + migrated_assets + and not unsupported_platforms + and not assets_failed_parsing + ): + props.pop(PREFIX + "platform", None) + props.pop(PREFIX + "region", None) + props.pop(PREFIX + "requester_pays", None) + + if schemes: + props[SCHEMES_PROP] = schemes + + super().migrate(obj, version, info) STORAGE_EXTENSION_HOOKS: ExtensionHooks = StorageExtensionHooks() diff --git a/tests/data-files/storage/collection-naip.json b/tests/data-files/storage/collection-naip.json index c24f38276..c3afac3da 100644 --- a/tests/data-files/storage/collection-naip.json +++ b/tests/data-files/storage/collection-naip.json @@ -16,29 +16,50 @@ } ], "stac_extensions": [ - "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" ], "summaries": { - "storage:platform": [ - "AZURE", - "GCP", - "AWS" - ], - "storage:region": [ - "westus2", - "us-central1", - "us-west-2", - "eastus" - ], - "storage:requester_pays": [ - true, - false - ], - "storage:tier": [ - "archive", - "COLDLINE", - "Standard", - "hot" + "storage:schemes": [ + { + "naip-azure-nsl": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naip-nsl" + } + }, + { + "naip-azure-eu": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipeuwest", + "requester_pays": false + } + }, + { + "naip-azure-blobs": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipblobs", + "region": "eastus" + } + }, + { + "naip-gcs": { + "type": "gcp", + "platform": "https://storage.googleapis.com/{bucket}", + "bucket": "naip-nsl", + "requester_pays": true + } + }, + { + "naip-aws": { + "type": "gcp", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "naip-visualization", + "region": "us-west-2", + "requester_pays": true + } + } ] }, "extent": { diff --git a/tests/data-files/storage/item-naip.json b/tests/data-files/storage/item-naip.json index 22069111d..59ebefb14 100644 --- a/tests/data-files/storage/item-naip.json +++ b/tests/data-files/storage/item-naip.json @@ -1,7 +1,7 @@ { "stac_version": "1.1.0", "stac_extensions": [ - "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" ], "id": "m_3009743_sw_14_1_20160928_20161129", "collection": "NAIP_MOSAIC", @@ -43,55 +43,70 @@ "datetime": "2016-09-28T00:00:00+00:00", "mission": "NAIP", "platform": "UNKNOWN_PLATFORM", - "gsd": 1 + "gsd": 1, + "storage:schemes": { + "naip-azure-nsl": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naip-nsl" + }, + "naip-azure-eu": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipeuwest", + "requester_pays": false + }, + "naip-azure-blobs": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipblobs", + "region": "eastus" + }, + "naip-gcs": { + "type": "gcp", + "platform": "https://storage.googleapis.com/{bucket}", + "bucket": "naip-nsl", + "requester_pays": true + }, + "naip-aws": { + "type": "gcp", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "naip-visualization", + "region": "us-west-2", + "requester_pays": true + } + } }, "assets": { "GEOTIFF_AZURE_RGBIR": { "href": "https://naip-nsl.blob.core.windows.net/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff", - "storage:platform": "AZURE", - "storage:region": "westus2", - "storage:tier": "archive" + "storage:refs": ["naip-azure-nsl"] }, "CO_GEOTIFF_GCP_RGB": { "href": "gs://naip-data/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "GCP", - "storage:region": "us-central1", - "storage:requester_pays": true, - "storage:tier": "COLDLINE" + "storage:refs": ["naip-gcs"] }, "CO_GEOTIFF_AWS_RGB": { "href": "s3://naip-visualization/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AWS", - "storage:region": "us-west-2", - "storage:requester_pays": true, - "storage:tier": "Standard" + "storage:refs": ["naip-aws"] }, "CO_GEOTIFF_AZURE_RGB": { "href": "https://naipeuwest.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AZURE", - "storage:region": "westeurope", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-eu"] }, "CO_GEOTIFF_AZURE_RGB_DEPRECATED": { "href": "https://naipblobs.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AZURE", - "storage:region": "eastus", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-blobs"] }, "THUMBNAIL_AZURE_DEPRECATED": { "href": "https://naipblobs.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.200.jpg", "type": "image/jpeg", - "storage:platform": "AZURE", - "storage:region": "eastus", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-blobs"] } }, "links": [ diff --git a/tests/data-files/storage/item-v1.0.0.json b/tests/data-files/storage/item-v1.0.0.json new file mode 100644 index 000000000..d75c398ad --- /dev/null +++ b/tests/data-files/storage/item-v1.0.0.json @@ -0,0 +1,90 @@ +{ + "stac_version": "1.1.0", + "stac_extensions": [ + "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + ], + "id": "an_image", + "collection": "a_collection", + "type": "Feature", + "bbox": [ + -97.7466867683867, + 30.278398961994966, + -97.72990596574927, + 30.288621181865743 + ], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -97.7466867683867, + 30.28754662370266 + ], + [ + -97.74555747279238, + 30.278398961994966 + ], + [ + -97.72990596574927, + 30.27972380176124 + ], + [ + -97.73085242627444, + 30.288621181865743 + ], + [ + -97.7466867683867, + 30.28754662370266 + ] + ] + ] + }, + "properties": { + "datetime": "2019-08-22T18:35:18+00:00", + "mission": "SOME_MISSION", + "platform": "SOME_2", + "instrument": "IN_1", + "gsd": 1 + }, + "assets": { + "GCP": { + "href": "gs://bucket/data.tif", + "type": "image/vnd.stac.geotiff", + "storage:platform": "GCP", + "storage:region": "us-central1", + "storage:tier": "STANDARD" + }, + "AWS": { + "href": "s3://bucket/data.png", + "type": "image/png", + "storage:platform": "AWS", + "storage:region": "us-west-2", + "storage:requester_pays": true, + "storage:tier": "Glacier" + }, + "AZURE": { + "href": "https://project.blob.core.windows.net/images/data.tif", + "type": "image/vnd.stac.geotiff", + "storage:platform": "AZURE", + "storage:region": "westus2", + "storage:tier": "archive" + }, + "AWS_2": { + "href": "s3://bucket2/data.png", + "type": "image/png", + "storage:platform": "AWS", + "storage:region": "us-west-2", + "storage:tier": "Glacier" + } + }, + "links": [ + { + "href": "https://example.com/examples/item.json", + "rel": "self" + }, + { + "href": "https://example.com/examples/item.json", + "rel": "collection" + } + ] +} \ No newline at end of file diff --git a/tests/extensions/cassettes/test_storage/test_asset_platform.yaml b/tests/extensions/cassettes/test_storage/test_asset_platform.yaml deleted file mode 100644 index df56e24ae..000000000 --- a/tests/extensions/cassettes/test_storage/test_asset_platform.yaml +++ /dev/null @@ -1,52 +0,0 @@ -interactions: -- request: - body: null - headers: {} - method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_asset_region.yaml b/tests/extensions/cassettes/test_storage/test_asset_region.yaml deleted file mode 100644 index df56e24ae..000000000 --- a/tests/extensions/cassettes/test_storage/test_asset_region.yaml +++ /dev/null @@ -1,52 +0,0 @@ -interactions: -- request: - body: null - headers: {} - method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_asset_requester_pays.yaml b/tests/extensions/cassettes/test_storage/test_asset_requester_pays.yaml deleted file mode 100644 index df56e24ae..000000000 --- a/tests/extensions/cassettes/test_storage/test_asset_requester_pays.yaml +++ /dev/null @@ -1,52 +0,0 @@ -interactions: -- request: - body: null - headers: {} - method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_asset_tier.yaml b/tests/extensions/cassettes/test_storage/test_asset_tier.yaml deleted file mode 100644 index df56e24ae..000000000 --- a/tests/extensions/cassettes/test_storage/test_asset_tier.yaml +++ /dev/null @@ -1,52 +0,0 @@ -interactions: -- request: - body: null - headers: {} - method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" - headers: {} - status: - code: 200 - message: OK -version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml index df56e24ae..9b5201037 100644 --- a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml +++ b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml @@ -3,48 +3,250 @@ interactions: body: null headers: {} method: GET - uri: https://stac-extensions.github.io/storage/v1.0.0/schema.json - response: - body: - string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": - \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\",\n \"title\": - \"Storage Extension\",\n \"description\": \"STAC Storage Extension to a STAC - Item and STAC Assets.\",\n \"oneOf\": [\n {\n \"$comment\": \"This - is the schema for STAC Items.\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\",\n \"properties\",\n - \ \"assets\"\n ],\n \"properties\": {\n \"type\": - {\n \"const\": \"Feature\"\n },\n \"properties\": - {\n \"allOf\": [\n {\n \"$comment\": - \"Require fields here for item properties.\",\n \"required\": - []\n },\n {\n \"$ref\": \"#/definitions/fields\"\n - \ }\n ]\n },\n \"assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n \ }\n ]\n },\n {\n \"$comment\": \"This is the schema - for STAC Collections\",\n \"allOf\": [\n {\n \"type\": - \"object\",\n \"required\": [\n \"type\"\n ],\n - \ \"properties\": {\n \"type\": {\n \"const\": - \"Collection\"\n },\n \"assets\": {\n \"type\": - \"object\",\n \"additionalProperties\": {\n \"$ref\": - \"#/definitions/fields\"\n }\n },\n \"item_assets\": - {\n \"type\": \"object\",\n \"additionalProperties\": - {\n \"$ref\": \"#/definitions/fields\"\n }\n }\n - \ }\n },\n {\n \"$ref\": \"#/definitions/stac_extensions\"\n - \ }\n ]\n }\n ], \n \"definitions\": {\n \"stac_extensions\": - {\n \"type\": \"object\",\n \"required\": [\n \"stac_extensions\"\n - \ ],\n \"properties\": {\n \"stac_extensions\": {\n \"type\": - \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v1.0.0/schema.json\"\n - \ }\n }\n }\n },\n \"fields\": {\n \"type\": - \"object\",\n \"properties\": {\n \"storage:platform\": {\n \"title\": - \"Platform\",\n \"type\": \"string\",\n \"enum\": [\n \"OTHER\",\n - \ \"AWS\",\n \"GCP\",\n \"AZURE\",\n \"IBM\",\n - \ \"ALIBABA\",\n \"ORACLE\"\n ],\n \"default\": - \"OTHER\"\n },\n \"storage:region\": {\n \"title\": - \"Region\",\n \"type\": \"string\"\n },\n \"storage:requester_pays\": - {\n \"type\": \"boolean\",\n \"title\": \"Requester pays\",\n - \ \"default\": false\n },\n \"storage:tier\": {\n \"title\": - \"Tier\",\n \"type\": \"string\"\n }\n }\n }\n }\n}\n" + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n + \ }\n ]\n },\n {\n \"$comment\": \"This is the schema + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: {} + status: + code: 200 + message: OK +- request: + body: null + headers: {} + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" headers: {} status: code: 200 diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index 4c1ba2322..70edf8afa 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -1,17 +1,24 @@ import json import random +from copy import deepcopy from string import ascii_letters import pytest import pystac -from pystac import ExtensionTypeError, Item +from pystac import ExtensionTypeError, Item, ItemAssetDefinition from pystac.collection import Collection -from pystac.extensions.storage import CloudPlatform, StorageExtension +from pystac.errors import RequiredPropertyMissing +from pystac.extensions.storage import ( + StorageExtension, + StorageScheme, + StorageSchemeType, +) from tests.utils import TestCases, assert_to_from_dict NAIP_EXAMPLE_URI = TestCases.get_path("data-files/storage/item-naip.json") NAIP_COLLECTION_URI = TestCases.get_path("data-files/storage/collection-naip.json") +V1_MIGRATION_ITEM_URI = TestCases.get_path("data-files/storage/item-v1.0.0.json") @pytest.fixture @@ -24,6 +31,35 @@ def naip_collection() -> Collection: return Collection.from_file(NAIP_COLLECTION_URI) +@pytest.fixture +def v1_item() -> Item: + with open(V1_MIGRATION_ITEM_URI) as f: + item_dict = json.load(f) + return Item.from_dict(item_dict, migrate=False) + + +@pytest.fixture +def sample_scheme() -> StorageScheme: + return StorageScheme.create( + type=StorageSchemeType.AWS_S3, + platform="https://{bucket}.s3.{region}.amazonaws.com", + region="us-west-2", + requester_pays=True, + ) + + +@pytest.fixture +def naip_asset(naip_item: Item) -> pystac.Asset: + # Grab a random asset with the platform property + return random.choice( + [ + _asset + for _asset in naip_item.assets.values() + if "storage:refs" in _asset.to_dict() + ] + ) + + def test_to_from_dict() -> None: with open(NAIP_EXAMPLE_URI) as f: item_dict = json.load(f) @@ -41,12 +77,12 @@ def test_add_to(sample_item: Item) -> None: StorageExtension.add_to(sample_item) StorageExtension.add_to(sample_item) - eo_uris = [ + uris = [ uri for uri in sample_item.stac_extensions if uri == StorageExtension.get_schema_uri() ] - assert len(eo_uris) == 1 + assert len(uris) == 1 @pytest.mark.vcr() @@ -54,13 +90,6 @@ def test_validate_storage(naip_item: Item) -> None: naip_item.validate() -def test_extend_invalid_object() -> None: - link = pystac.Link("child", "https://some-domain.com/some/path/to.json") - - with pytest.raises(pystac.ExtensionTypeError): - StorageExtension.ext(link) # type: ignore - - def test_extension_not_implemented(sample_item: Item) -> None: # Should raise exception if Item does not include extension URI with pytest.raises(pystac.ExtensionNotImplemented): @@ -77,6 +106,15 @@ def test_extension_not_implemented(sample_item: Item) -> None: _ = StorageExtension.ext(ownerless_asset) +def test_collection_ext_add_to(naip_collection: Collection) -> None: + naip_collection.stac_extensions = [] + assert StorageExtension.get_schema_uri() not in naip_collection.stac_extensions + + _ = StorageExtension.ext(naip_collection, add_if_missing=True) + + assert StorageExtension.get_schema_uri() in naip_collection.stac_extensions + + def test_item_ext_add_to(sample_item: Item) -> None: assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions @@ -85,6 +123,16 @@ def test_item_ext_add_to(sample_item: Item) -> None: assert StorageExtension.get_schema_uri() in sample_item.stac_extensions +def test_catalog_ext_add_to() -> None: + catalog = pystac.Catalog("stac", "a catalog") + + assert StorageExtension.get_schema_uri() not in catalog.stac_extensions + + _ = StorageExtension.ext(catalog, add_if_missing=True) + + assert StorageExtension.get_schema_uri() in catalog.stac_extensions + + def test_asset_ext_add_to(sample_item: Item) -> None: assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions asset = sample_item.assets["thumbnail"] @@ -94,6 +142,15 @@ def test_asset_ext_add_to(sample_item: Item) -> None: assert StorageExtension.get_schema_uri() in sample_item.stac_extensions +def test_link_ext_add_to(sample_item: Item) -> None: + assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions + asset = sample_item.links[0] + + _ = StorageExtension.ext(asset, add_if_missing=True) + + assert StorageExtension.get_schema_uri() in sample_item.stac_extensions + + def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: asset_dict = sample_item.assets["thumbnail"].to_dict() asset = pystac.Asset.from_dict(asset_dict) @@ -104,79 +161,38 @@ def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: def test_should_raise_exception_when_passing_invalid_extension_object() -> None: with pytest.raises( - ExtensionTypeError, match=r"^StorageExtension does not apply to type 'object'$" + ExtensionTypeError, + match=r"^StorageExtension does not apply to type 'object'$", ): # calling it wrong purposely so ---------v StorageExtension.ext(object()) # type: ignore -def test_summaries_platform(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.platform == col_dict["summaries"]["storage:platform"] - # Set - new_platform_summary = [random.choice([v for v in CloudPlatform])] - assert storage_summaries.platform != new_platform_summary - storage_summaries.platform = new_platform_summary - assert storage_summaries.platform == new_platform_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:platform"] == new_platform_summary - - -def test_summaries_region(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.region == col_dict["summaries"]["storage:region"] - # Set - new_region_summary = [random.choice(ascii_letters)] - assert storage_summaries.region != new_region_summary - storage_summaries.region = new_region_summary - assert storage_summaries.region == new_region_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:region"] == new_region_summary - - -def test_summaries_requester_pays(naip_collection: Collection) -> None: +def test_summaries_schemes(naip_collection: Collection) -> None: col_dict = naip_collection.to_dict() storage_summaries = StorageExtension.summaries(naip_collection) - # Get assert ( - storage_summaries.requester_pays - == col_dict["summaries"]["storage:requester_pays"] + list( + map( + lambda x: {k: c.to_dict() for k, c in x.items()}, + storage_summaries.schemes or [], + ) + ) + == col_dict["summaries"]["storage:schemes"] ) - - # Set - new_requester_pays_summary = [True] - assert storage_summaries.requester_pays != new_requester_pays_summary - storage_summaries.requester_pays = new_requester_pays_summary - assert storage_summaries.requester_pays == new_requester_pays_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:requester_pays"] == new_requester_pays_summary - - -def test_summaries_tier(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.tier == col_dict["summaries"]["storage:tier"] - # Set - new_tier_summary = [random.choice(ascii_letters)] - assert storage_summaries.tier != new_tier_summary - storage_summaries.tier = new_tier_summary - assert storage_summaries.tier == new_tier_summary + new_schemes_summary = [ + {"key": StorageScheme.create("aws-s3", "https://a.platform.example.com")} + ] + assert storage_summaries.schemes != new_schemes_summary + storage_summaries.schemes = new_schemes_summary + assert storage_summaries.schemes == new_schemes_summary col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:tier"] == new_tier_summary + assert col_dict["summaries"]["storage:schemes"] == [ + {k: c.to_dict() for k, c in x.items()} for x in new_schemes_summary + ] def test_summaries_adds_uri(naip_collection: Collection) -> None: @@ -195,142 +211,196 @@ def test_summaries_adds_uri(naip_collection: Collection) -> None: assert StorageExtension.get_schema_uri() not in naip_collection.stac_extensions -def test_item_apply(naip_item: Item) -> None: - asset = random.choice(list(naip_item.assets.values())) - - storage_ext = StorageExtension.ext(asset) - - new_platform = random.choice( - [v for v in CloudPlatform if v != storage_ext.platform] - ) +def test_schemes_apply(naip_item: Item) -> None: + storage_ext = StorageExtension.ext(naip_item) + new_key = random.choice(ascii_letters) + new_type = random.choice(ascii_letters) + new_platform = random.choice(ascii_letters) new_region = random.choice(ascii_letters) - new_requestor_pays = random.choice( - [v for v in {True, False} if v != storage_ext.requester_pays] - ) - new_tier = random.choice(ascii_letters) + new_requestor_pays = random.choice([v for v in {True, False}]) storage_ext.apply( - platform=new_platform, - region=new_region, - requester_pays=new_requestor_pays, - tier=new_tier, + schemes={ + new_key: StorageScheme.create( + new_type, new_platform, new_region, new_requestor_pays + ), + } ) - assert storage_ext.platform == new_platform - assert storage_ext.region == new_region - assert storage_ext.requester_pays == new_requestor_pays - assert storage_ext.tier == new_tier + applied_schemes = storage_ext.schemes + assert list(applied_schemes) == [new_key] + assert applied_schemes[new_key].type == new_type + assert applied_schemes[new_key].platform == new_platform + assert applied_schemes[new_key].region == new_region + assert applied_schemes[new_key].requester_pays == new_requestor_pays @pytest.mark.vcr() -def test_asset_platform(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:platform" in _asset.to_dict() - ] - ) +def test_refs_apply(naip_asset: pystac.Asset) -> None: + test_refs = ["a_ref", "b_ref"] - storage_ext = StorageExtension.ext(asset) + storage_ext = StorageExtension.ext(naip_asset) + storage_ext.apply(refs=test_refs) # Get - assert storage_ext.platform == asset.extra_fields.get("storage:platform") + assert storage_ext.refs == test_refs # Set - new_platform = random.choice( - [val for val in CloudPlatform if val != storage_ext.platform] - ) - storage_ext.platform = new_platform - assert storage_ext.platform == new_platform + new_refs = [random.choice(ascii_letters)] + storage_ext.refs = new_refs + assert storage_ext.refs == new_refs - naip_item.validate() +def test_schemes_apply_raises(naip_item: Item) -> None: + storage_ext = StorageExtension.ext(naip_item) -@pytest.mark.vcr() -def test_asset_region(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:region" in _asset.to_dict() - ] - ) + with pytest.raises( + ValueError, + match="'refs' cannot be applied with this STAC object type.", + ): + storage_ext.apply( + schemes={ + "a_key": StorageScheme.create("a_type", "a_platform"), + }, + refs=["a_ref"], + ) + with pytest.raises( + RequiredPropertyMissing, + match="'schemes' property is required for this object type.", + ): + storage_ext.apply(refs=None) - storage_ext = StorageExtension.ext(asset) - # Get - assert storage_ext.region == asset.extra_fields.get("storage:region") +def test_refs_apply_raises(naip_asset: Item) -> None: + storage_ext = StorageExtension.ext(naip_asset) - # Set - new_region = random.choice( - [val for val in CloudPlatform if val != storage_ext.region] - ) - storage_ext.region = new_region - assert storage_ext.region == new_region + with pytest.raises( + ValueError, + match="'schemes' cannot be applied with this STAC object type.", + ): + storage_ext.apply( + schemes={ + "a_key": StorageScheme.create("a_type", "a_platform"), + }, + refs=["a_ref"], + ) - naip_item.validate() + with pytest.raises( + RequiredPropertyMissing, + match="'refs' property is required for this object type.", + ): + storage_ext.apply(schemes=None) - # Set to None - storage_ext.region = None - assert "storage:region" not in asset.extra_fields +def test_add_storage_scheme(naip_item: Item) -> None: + storage_ext = naip_item.ext.storage + storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) + assert "new_scheme" in storage_ext.schemes -@pytest.mark.vcr() -def test_asset_requester_pays(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:requester_pays" in _asset.to_dict() - ] - ) + storage_ext.properties.pop("storage:schemes") + storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) + assert len(storage_ext.schemes) == 1 + assert "new_scheme" in storage_ext.schemes - storage_ext = StorageExtension.ext(asset) - # Get - assert storage_ext.requester_pays == asset.extra_fields.get( - "storage:requester_pays" - ) +def test_add_refs(naip_item: Item) -> None: + scheme_name = random.choice(ascii_letters) + asset = naip_item.assets["GEOTIFF_AZURE_RGBIR"] + storage_ext = asset.ext.storage + assert isinstance(storage_ext, StorageExtension) - # Set - new_requester_pays = True if not storage_ext.requester_pays else False - storage_ext.requester_pays = new_requester_pays - assert storage_ext.requester_pays == new_requester_pays + storage_ext.add_ref(scheme_name) + assert scheme_name in storage_ext.refs - naip_item.validate() + storage_ext.properties.pop("storage:refs") + scheme_name_2 = random.choice(ascii_letters) + storage_ext.add_ref(scheme_name_2) + assert len(storage_ext.refs) == 1 + assert scheme_name_2 in storage_ext.refs - # Set to None - storage_ext.requester_pays = None - assert "storage:requester_pays" not in asset.extra_fields +def test_storage_scheme_create(sample_scheme: StorageScheme) -> None: + assert sample_scheme.type == StorageSchemeType.AWS_S3 + assert sample_scheme.platform == "https://{bucket}.s3.{region}.amazonaws.com" + assert sample_scheme.region == "us-west-2" + assert sample_scheme.requester_pays is True -@pytest.mark.vcr() -def test_asset_tier(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:tier" in _asset.to_dict() - ] - ) + sample_scheme.type = StorageSchemeType.AZURE + sample_scheme.platform = "https://{account}.blob.core.windows.net" + sample_scheme.region = "eastus" + sample_scheme.account = "account" + sample_scheme.requester_pays = False - storage_ext = StorageExtension.ext(asset) + assert sample_scheme.type == StorageSchemeType.AZURE + assert sample_scheme.platform == "https://{account}.blob.core.windows.net" + assert sample_scheme.region == "eastus" + assert sample_scheme.account == "account" + assert sample_scheme.requester_pays is False - # Get - assert storage_ext.tier == asset.extra_fields.get("storage:tier") - # Set - new_tier = random.choice(ascii_letters) - storage_ext.tier = new_tier - assert storage_ext.tier == new_tier +def test_storage_scheme_equality(sample_scheme: StorageScheme) -> None: + other = deepcopy(sample_scheme) + assert sample_scheme == other - naip_item.validate() + other.requester_pays = False + assert sample_scheme != other + + assert sample_scheme != object() + + +def test_item_asset_accessor() -> None: + item_asset = ItemAssetDefinition.create( + title="title", description="desc", media_type="media", roles=["a_role"] + ) + assert isinstance(item_asset.ext.storage, StorageExtension) + + +@pytest.mark.filterwarnings("ignore") +def test_migrate(v1_item: Item) -> None: + item = Item.from_dict( + v1_item.to_dict(include_self_link=False, transform_hrefs=False), migrate=True + ) - # Set to None - storage_ext.tier = None - assert "storage:tier" not in asset.extra_fields + # Check schemes were created at item level + assert "storage:schemes" in item.properties + schemes = item.properties["storage:schemes"] + + # AWS asset should be migrated + assert "storage:refs" in item.assets["AWS"].to_dict() + aws_refs = item.assets["AWS"].to_dict()["storage:refs"] + assert len(aws_refs) == 1 + assert aws_refs[0] in schemes + assert schemes[aws_refs[0]]["type"] == "aws-s3" + assert schemes[aws_refs[0]]["region"] == "us-west-2" + assert schemes[aws_refs[0]]["requester_pays"] is True + assert schemes[aws_refs[0]]["bucket"] == "bucket" + + # AWS_2 should a different scheme than AWS (same region, no requester_pays) + assert "storage:refs" in item.assets["AWS_2"].to_dict() + aws2_refs = item.assets["AWS_2"].to_dict()["storage:refs"] + assert aws2_refs != aws_refs + assert schemes[aws2_refs[0]]["bucket"] == "bucket2" + + # AZURE asset should be migrated + assert "storage:refs" in item.assets["AZURE"].to_dict() + azure_refs = item.assets["AZURE"].to_dict()["storage:refs"] + assert len(azure_refs) == 1 + assert azure_refs[0] in schemes + assert schemes[azure_refs[0]]["type"] == "ms-azure" + assert schemes[azure_refs[0]]["region"] == "westus2" + assert schemes[azure_refs[0]]["account"] == "project" + + # GCP asset should NOT be migrated (unsupported platform) + assert "storage:refs" not in item.assets["GCP"].to_dict() + assert "storage:platform" in item.assets["GCP"].to_dict() + + # Old properties should be removed from migrated assets + assert "storage:platform" not in item.assets["AWS"].to_dict() + assert "storage:region" not in item.assets["AWS"].to_dict() + assert "storage:platform" not in item.assets["AZURE"].to_dict() + + # storage:tier should be removed from migrated assets + assert "storage:tier" not in item.assets["AWS"].to_dict() + assert "storage:tier" not in item.assets["AZURE"].to_dict() + # but preserved for unmigrated assets + assert item.assets["GCP"].to_dict().get("storage:tier") == "STANDARD"