diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 22c6fbc..f32290e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,6 @@ repos: - id: black args: ["--target-version=py38"] - - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: diff --git a/README.md b/README.md index 2117b06..bad3a97 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,14 @@ except Exception as exc: Plus d'exemple dans la documentation développeur. + ## Contribuer * Installer les dépendances de développement : ```sh python3 -m pip install -e .[dev] + pre-commit install ``` * Consulter les [directives de contribution](./CONTRIBUTING.md) @@ -46,7 +48,7 @@ apt install python3-venv python3-rados python3-gdal python3 -m venv .venv source .venv/bin/activate python3 -m pip install --upgrade build bump2version -bump2version --allow-dirty --current-version 0.0.0 --new-version x.y.z patch pyproject.toml src/rok4/__init__.py +bump2version --current-version 0.0.0 --new-version x.y.z patch # Run unit tests python3 -m pip install -e .[test] diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index f55433f..09ea0e4 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -331,6 +331,10 @@ def slab_width(self) -> int: def slab_height(self) -> int: return self.__slab_size[1] + @property + def tile_limits(self) -> Dict[str, int]: + return self.__tile_limits + def is_in_limits(self, column: int, row: int) -> bool: """Is the tile indices in limits ? @@ -475,13 +479,14 @@ def from_descriptor(cls, descriptor: str) -> "Pyramid": return pyramid @classmethod - def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid": + def from_other(cls, other: "Pyramid", name: str, storage: Dict, **kwargs) -> "Pyramid": """Create a pyramid from another one Args: other (Pyramid): pyramid to clone name (str): new pyramid's name storage (Dict[str, Union[str, int]]): new pyramid's storage informations + **mask (bool) : Presence or not of mask (only for RASTER) Raises: FormatError: Provided path or the TMS is not a well formed JSON @@ -493,7 +498,8 @@ def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid": """ try: # On convertit le type de stockage selon l'énumération - storage["type"] = StorageType[storage["type"]] + if type(storage["type"]) is str: + storage["type"] = StorageType[storage["type"]] if storage["type"] == StorageType.FILE and name.find("/") != -1: raise Exception(f"A FILE stored pyramid's name cannot contain '/' : '{name}'") @@ -519,7 +525,9 @@ def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid": # Attributs d'une pyramide raster if pyramid.type == PyramidType.RASTER: - if other.own_masks: + if "mask" in kwargs: + pyramid.__masks = kwargs["mask"] + elif other.own_masks: pyramid.__masks = True else: pyramid.__masks = False @@ -668,6 +676,10 @@ def own_masks(self) -> bool: def format(self) -> str: return self.__format + @property + def channels(self) -> str: + return self.raster_specifications["channels"] + @property def tile_extension(self) -> str: if self.__format in [ @@ -735,10 +747,14 @@ def load_list(self) -> None: self.__content["loaded"] = True - def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]]: + def list_generator( + self, level_id: str = None + ) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]]: """Get list content List is copied as temporary file, roots are read and informations about each slab is returned. If list is already loaded, we yield the cached content + Args : + level_id (str) : id of the level for load only one level Examples: @@ -776,7 +792,11 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] """ if self.__content["loaded"]: for slab, infos in self.__content["cache"].items(): - yield slab, infos + if level_id is not None: + if slab[1] == level_id: + yield slab, infos + else: + yield slab, infos else: # Copie de la liste dans un fichier temporaire (cette liste peut être un objet) list_obj = tempfile.NamedTemporaryFile(mode="r", delete=False) @@ -824,7 +844,11 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] "md5": slab_md5, } - yield ((slab_type, level, column, row), infos) + if level_id is not None: + if level == level_id: + yield ((slab_type, level, column, row), infos) + else: + yield ((slab_type, level, column, row), infos) remove(f"file://{list_file}") @@ -1372,6 +1396,58 @@ def get_tile_indices( return (level_object.id,) + level_object.tile_matrix.point_to_indices(x, y) + def delete_level(self, level_id: str) -> None: + """Delete the given level in the description of the pyramid + + Args: + level_id: Level identifier + + Raises: + Exception: Cannot find level + """ + + try: + del self.__levels[level_id] + except Exception: + raise Exception(f"The level {level_id} don't exist in the pyramid") + + def add_level( + self, + level_id: str, + tiles_per_width: int, + tiles_per_height: int, + tile_limits: Dict[str, int], + ) -> None: + """Add a level in the description of the pyramid + + Args: + level_id: Level identifier + tiles_per_width : Number of tiles in width by slab + tiles_per_height : Number of tiles in height by slab + tile_limits : Minimum and maximum tiles' columns and rows of pyramid's content + """ + + data = { + "id": level_id, + "tile_limits": tile_limits, + "tiles_per_width": tiles_per_width, + "tiles_per_height": tiles_per_height, + "storage": {"type": self.storage_type.name}, + } + if self.own_masks: + data["storage"]["mask_prefix"] = True + if self.storage_type == StorageType.FILE: + data["storage"]["path_depth"] = self.storage_depth + + lev = Level.from_descriptor(data, self) + + if self.__tms.get_level(lev.id) is None: + raise Exception( + f"Pyramid {self.name} owns a level with the ID '{lev.id}', not defined in the TMS '{self.tms.name}'" + ) + else: + self.__levels[lev.id] = lev + @property def size(self) -> int: """Get the size of the pyramid diff --git a/src/rok4/storage.py b/src/rok4/storage.py index a3c6975..1cf0f6f 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -11,9 +11,9 @@ Readings uses a LRU cache system with a TTL. It's possible to configure it with environment variables : - ROK4_READING_LRU_CACHE_SIZE : Number of cached element. Default 64. Set 0 or a negative integer to configure a cache without bound. A power of two make cache more efficient. -- ROK4_READING_LRU_CACHE_TTL : Validity duration of cached element, in seconds. Default 300. 0 or negative integer to disable time validity. +- ROK4_READING_LRU_CACHE_TTL : Validity duration of cached element, in seconds. Default 300. 0 or negative integer to get cache without expiration date. -To disable cache, set ROK4_READING_LRU_CACHE_SIZE to 1 and ROK4_READING_LRU_CACHE_TTL to 0. +To disable cache (always read data on storage), set ROK4_READING_LRU_CACHE_SIZE to 1 and ROK4_READING_LRU_CACHE_TTL to 1. Using CEPH storage requires environment variables : - ROK4_CEPH_CONFFILE @@ -24,7 +24,6 @@ - ROK4_S3_KEY - ROK4_S3_SECRETKEY - ROK4_S3_URL -- ROK4_SSL_NO_VERIFY (optionnal) with a non empty value disables certificate check.. Define PYTHONWARNINGS to "ignore:Unverified HTTPS request" to disable warnings logs To use several S3 clusters, each environment variable have to contain a list (comma-separated), with the same number of elements @@ -74,7 +73,6 @@ __OBJECT_SYMLINK_SIGNATURE = "SYMLINK#" __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None - __LRU_SIZE = 64 __LRU_TTL = 300 @@ -97,10 +95,10 @@ pass -def __get_ttl_hash(): +def __get_ttl_hash() -> int: """Return the time string rounded according to time-to-live value""" if __LRU_TTL == 0: - return time.time() + return 0 else: return round(time.time() / __LRU_TTL) @@ -127,7 +125,6 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s verify = True if "ROK4_SSL_NO_VERIFY" in os.environ and os.environ["ROK4_SSL_NO_VERIFY"] != "": verify = False - # C'est la première fois qu'on cherche à utiliser le stockage S3, chargeons les informations depuis les variables d'environnement try: keys = os.environ["ROK4_S3_KEY"].split(",") @@ -1033,6 +1030,12 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: elif target_type == StorageType.FILE: try: + to_tray = get_infos_from_path(link_path)[2] + if to_tray != "": + os.makedirs(to_tray, exist_ok=True) + + if exists(link_path): + remove(link_path) if hard: os.link(target_path, link_path) else: @@ -1081,7 +1084,7 @@ def get_osgeo_path(path: str) -> str: def size_path(path: str) -> int: - """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list) + """Return the size of the given path (or, for the CEPH, the sum of the size of each object of the .list) Args: path (str): Source path diff --git a/src/rok4/tile_matrix_set.py b/src/rok4/tile_matrix_set.py index 84a3d22..a51d717 100644 --- a/src/rok4/tile_matrix_set.py +++ b/src/rok4/tile_matrix_set.py @@ -177,6 +177,14 @@ def point_to_indices(self, x: float, y: float) -> Tuple[int, int, int, int]: absolute_pixel_row % self.tile_size[1], ) + @property + def tile_width(self) -> int: + return self.tile_size[0] + + @property + def tile_heigth(self) -> int: + return self.tile_size[1] + class TileMatrixSet: """A tile matrix set is multi levels grid definition diff --git a/src/rok4/utils.py b/src/rok4/utils.py index 40c7884..050d679 100644 --- a/src/rok4/utils.py +++ b/src/rok4/utils.py @@ -1,5 +1,6 @@ """Provide functions to manipulate OGR / OSR entities """ + # -- IMPORTS -- # standard library diff --git a/tests/test_layer.py b/tests/test_layer.py index e91939d..a4df1b3 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -1,8 +1,9 @@ import os from unittest import mock -from unittest.mock import MagicMock +from unittest.mock import * from rok4.enums import PyramidType +from rok4.exceptions import * from rok4.layer import Layer diff --git a/tests/test_storage.py b/tests/test_storage.py index 1aac9f0..dd7dddd 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -603,9 +603,11 @@ def test_link_hard_nok(): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.symlink", return_value=None) -def test_link_file_ok(mock_link): +@mock.patch("os.makedirs", return_value=None) +def test_link_file_ok(mock_makedirs, mock_link): try: link("file:///path/to/target.ext", "file:///path/to/link.ext") + mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) mock_link.assert_called_once_with("/path/to/target.ext", "/path/to/link.ext") except Exception as exc: assert False, f"FILE link raises an exception: {exc}" @@ -613,9 +615,11 @@ def test_link_file_ok(mock_link): @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.link", return_value=None) -def test_hlink_file_ok(mock_link): +@mock.patch("os.makedirs", return_value=None) +def test_hlink_file_ok(mock_makedirs, mock_link): try: link("file:///path/to/target.ext", "file:///path/to/link.ext", True) + mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) mock_link.assert_called_once_with("/path/to/target.ext", "/path/to/link.ext") except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" diff --git a/tests/test_tile_matrix_set.py b/tests/test_tile_matrix_set.py index cab1d19..4209daa 100644 --- a/tests/test_tile_matrix_set.py +++ b/tests/test_tile_matrix_set.py @@ -1,5 +1,6 @@ import os from unittest import mock +from unittest.mock import * import pytest