Ajouts de fonctions nécessaire pour le joincache de pytools

.pre-commit-config.yaml

@@ -33,7 +33,6 @@ repos:
       - id: black
         args: ["--target-version=py38"]
 
-
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:

README.md

@@ -29,12 +29,14 @@ except Exception as exc:
 
 Plus d'exemple dans la documentation développeur.
 
+
 ## Contribuer
 
 * Installer les dépendances de développement :
 
     ```sh
     python3 -m pip install -e .[dev]
+    pre-commit install
     ```
 
 * Consulter les [directives de contribution](./CONTRIBUTING.md)
@@ -46,7 +48,7 @@ apt install python3-venv python3-rados python3-gdal
 python3 -m venv .venv
 source .venv/bin/activate
 python3 -m pip install --upgrade build bump2version
-bump2version --allow-dirty --current-version 0.0.0 --new-version x.y.z patch pyproject.toml src/rok4/__init__.py
+bump2version --current-version 0.0.0 --new-version x.y.z patch
 
 # Run unit tests
 python3 -m pip install -e .[test]

src/rok4/pyramid.py

@@ -331,6 +331,10 @@ def slab_width(self) -> int:
     def slab_height(self) -> int:
         return self.__slab_size[1]
 
+    @property
+    def tile_limits(self) -> Dict[str, int]:
+        return self.__tile_limits
+
     def is_in_limits(self, column: int, row: int) -> bool:
         """Is the tile indices in limits ?
 
@@ -475,13 +479,14 @@ def from_descriptor(cls, descriptor: str) -> "Pyramid":
         return pyramid
 
     @classmethod
-    def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid":
+    def from_other(cls, other: "Pyramid", name: str, storage: Dict, **kwargs) -> "Pyramid":
         """Create a pyramid from another one
 
         Args:
             other (Pyramid): pyramid to clone
             name (str): new pyramid's name
             storage (Dict[str, Union[str, int]]): new pyramid's storage informations
+            **mask (bool) : Presence or not of mask (only for RASTER)
 
         Raises:
             FormatError: Provided path or the TMS is not a well formed JSON
@@ -493,7 +498,8 @@ def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid":
         """
         try:
             # On convertit le type de stockage selon l'énumération
-            storage["type"] = StorageType[storage["type"]]
+            if type(storage["type"]) is str:
+                storage["type"] = StorageType[storage["type"]]
 
             if storage["type"] == StorageType.FILE and name.find("/") != -1:
                 raise Exception(f"A FILE stored pyramid's name cannot contain '/' : '{name}'")
@@ -519,7 +525,9 @@ def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid":
 
             # Attributs d'une pyramide raster
             if pyramid.type == PyramidType.RASTER:
-                if other.own_masks:
+                if "mask" in kwargs:
+                    pyramid.__masks = kwargs["mask"]
+                elif other.own_masks:
                     pyramid.__masks = True
                 else:
                     pyramid.__masks = False
@@ -668,6 +676,10 @@ def own_masks(self) -> bool:
     def format(self) -> str:
         return self.__format
 
+    @property
+    def channels(self) -> str:
+        return self.raster_specifications["channels"]
+
     @property
     def tile_extension(self) -> str:
         if self.__format in [
@@ -735,10 +747,14 @@ def load_list(self) -> None:
 
         self.__content["loaded"] = True
 
-    def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]]:
+    def list_generator(
+        self, level_id: str = None
+    ) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]]:
         """Get list content
 
         List is copied as temporary file, roots are read and informations about each slab is returned. If list is already loaded, we yield the cached content
+        Args :
+            level_id (str) : id of the level for load only one level
 
         Examples:
 
@@ -776,7 +792,11 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]
         """
         if self.__content["loaded"]:
             for slab, infos in self.__content["cache"].items():
-                yield slab, infos
+                if level_id is not None:
+                    if slab[1] == level_id:
+                        yield slab, infos
+                else:
+                    yield slab, infos
         else:
             # Copie de la liste dans un fichier temporaire (cette liste peut être un objet)
             list_obj = tempfile.NamedTemporaryFile(mode="r", delete=False)
@@ -824,7 +844,11 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]
                         "md5": slab_md5,
                     }
 
-                    yield ((slab_type, level, column, row), infos)
+                    if level_id is not None:
+                        if level == level_id:
+                            yield ((slab_type, level, column, row), infos)
+                    else:
+                        yield ((slab_type, level, column, row), infos)
 
             remove(f"file://{list_file}")
 
@@ -1372,6 +1396,58 @@ def get_tile_indices(
 
         return (level_object.id,) + level_object.tile_matrix.point_to_indices(x, y)
 
+    def delete_level(self, level_id: str) -> None:
+        """Delete the given level in the description of the pyramid
+
+        Args:
+            level_id: Level identifier
+
+        Raises:
+            Exception: Cannot find level
+        """
+
+        try:
+            del self.__levels[level_id]
+        except Exception:
+            raise Exception(f"The level {level_id} don't exist in the pyramid")
+
+    def add_level(
+        self,
+        level_id: str,
+        tiles_per_width: int,
+        tiles_per_height: int,
+        tile_limits: Dict[str, int],
+    ) -> None:
+        """Add a level in the description of the pyramid
+
+        Args:
+            level_id: Level identifier
+            tiles_per_width : Number of tiles in width by slab
+            tiles_per_height : Number of tiles in height by slab
+            tile_limits : Minimum and maximum tiles' columns and rows of pyramid's content
+        """
+
+        data = {
+            "id": level_id,
+            "tile_limits": tile_limits,
+            "tiles_per_width": tiles_per_width,
+            "tiles_per_height": tiles_per_height,
+            "storage": {"type": self.storage_type.name},
+        }
+        if self.own_masks:
+            data["storage"]["mask_prefix"] = True
+        if self.storage_type == StorageType.FILE:
+            data["storage"]["path_depth"] = self.storage_depth
+
+        lev = Level.from_descriptor(data, self)
+
+        if self.__tms.get_level(lev.id) is None:
+            raise Exception(
+                f"Pyramid {self.name} owns a level with the ID '{lev.id}', not defined in the TMS '{self.tms.name}'"
+            )
+        else:
+            self.__levels[lev.id] = lev
+
     @property
     def size(self) -> int:
         """Get the size of the pyramid

src/rok4/storage.py

@@ -11,9 +11,9 @@
 
 Readings uses a LRU cache system with a TTL. It's possible to configure it with environment variables :
 - ROK4_READING_LRU_CACHE_SIZE : Number of cached element. Default 64. Set 0 or a negative integer to configure a cache without bound. A power of two make cache more efficient.
-- ROK4_READING_LRU_CACHE_TTL : Validity duration of cached element, in seconds. Default 300. 0 or negative integer to disable time validity.
+- ROK4_READING_LRU_CACHE_TTL : Validity duration of cached element, in seconds. Default 300. 0 or negative integer to get cache without expiration date.
 
-To disable cache, set ROK4_READING_LRU_CACHE_SIZE to 1 and ROK4_READING_LRU_CACHE_TTL to 0.
+To disable cache (always read data on storage), set ROK4_READING_LRU_CACHE_SIZE to 1 and ROK4_READING_LRU_CACHE_TTL to 1.
 
 Using CEPH storage requires environment variables :
 - ROK4_CEPH_CONFFILE
@@ -24,7 +24,6 @@
 - ROK4_S3_KEY
 - ROK4_S3_SECRETKEY
 - ROK4_S3_URL
-- ROK4_SSL_NO_VERIFY (optionnal) with a non empty value disables certificate check.. Define PYTHONWARNINGS to "ignore:Unverified HTTPS request" to disable warnings logs
 
 To use several S3 clusters, each environment variable have to contain a list (comma-separated), with the same number of elements
 
@@ -74,7 +73,6 @@
 __OBJECT_SYMLINK_SIGNATURE = "SYMLINK#"
 __S3_CLIENTS = {}
 __S3_DEFAULT_CLIENT = None
-
 __LRU_SIZE = 64
 __LRU_TTL = 300
 
@@ -97,10 +95,10 @@
     pass
 
 
-def __get_ttl_hash():
+def __get_ttl_hash() -> int:
     """Return the time string rounded according to time-to-live value"""
     if __LRU_TTL == 0:
-        return time.time()
+        return 0
     else:
         return round(time.time() / __LRU_TTL)
 
@@ -127,7 +125,6 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s
         verify = True
         if "ROK4_SSL_NO_VERIFY" in os.environ and os.environ["ROK4_SSL_NO_VERIFY"] != "":
             verify = False
-
         # C'est la première fois qu'on cherche à utiliser le stockage S3, chargeons les informations depuis les variables d'environnement
         try:
             keys = os.environ["ROK4_S3_KEY"].split(",")
@@ -1033,6 +1030,12 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None:
 
     elif target_type == StorageType.FILE:
         try:
+            to_tray = get_infos_from_path(link_path)[2]
+            if to_tray != "":
+                os.makedirs(to_tray, exist_ok=True)
+
+            if exists(link_path):
+                remove(link_path)
             if hard:
                 os.link(target_path, link_path)
             else:
@@ -1081,7 +1084,7 @@ def get_osgeo_path(path: str) -> str:
 
 
 def size_path(path: str) -> int:
-    """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list)
+    """Return the size of the given path (or, for the CEPH, the sum of the size of each object of the .list)
 
     Args:
         path (str): Source path

src/rok4/tile_matrix_set.py

@@ -177,6 +177,14 @@ def point_to_indices(self, x: float, y: float) -> Tuple[int, int, int, int]:
             absolute_pixel_row % self.tile_size[1],
         )
 
+    @property
+    def tile_width(self) -> int:
+        return self.tile_size[0]
+
+    @property
+    def tile_heigth(self) -> int:
+        return self.tile_size[1]
+
 
 class TileMatrixSet:
     """A tile matrix set is multi levels grid definition

src/rok4/utils.py

@@ -1,5 +1,6 @@
 """Provide functions to manipulate OGR / OSR entities
 """
+
 # -- IMPORTS --
 
 # standard library

tests/test_layer.py

@@ -1,8 +1,9 @@
 import os
 from unittest import mock
-from unittest.mock import MagicMock
+from unittest.mock import *
 
 from rok4.enums import PyramidType
+from rok4.exceptions import *
 from rok4.layer import Layer
 
 

tests/test_storage.py

@@ -603,19 +603,23 @@ def test_link_hard_nok():
 
 @mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch("os.symlink", return_value=None)
-def test_link_file_ok(mock_link):
+@mock.patch("os.makedirs", return_value=None)
+def test_link_file_ok(mock_makedirs, mock_link):
     try:
         link("file:///path/to/target.ext", "file:///path/to/link.ext")
+        mock_makedirs.assert_called_once_with("/path/to", exist_ok=True)
         mock_link.assert_called_once_with("/path/to/target.ext", "/path/to/link.ext")
     except Exception as exc:
         assert False, f"FILE link raises an exception: {exc}"
 
 
 @mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch("os.link", return_value=None)
-def test_hlink_file_ok(mock_link):
+@mock.patch("os.makedirs", return_value=None)
+def test_hlink_file_ok(mock_makedirs, mock_link):
     try:
         link("file:///path/to/target.ext", "file:///path/to/link.ext", True)
+        mock_makedirs.assert_called_once_with("/path/to", exist_ok=True)
         mock_link.assert_called_once_with("/path/to/target.ext", "/path/to/link.ext")
     except Exception as exc:
         assert False, f"FILE hard link raises an exception: {exc}"

tests/test_tile_matrix_set.py

@@ -1,5 +1,6 @@
 import os
 from unittest import mock
+from unittest.mock import *
 
 import pytest