Ajout de la fonction de calcul de la taille d'une pyramide

CHANGELOG.md

@@ -1,43 +1,5 @@
 ## Summary
 
-Lecture facilitée de la liste d'une pyramide. Lecture d'informations sur une donnée raster unique depuis un fichier ou une liste de paramètres. Modification de la gestion des vecteurs.
-
-## Changelog
-
-### [Added]
-
-* Raster
-    * Chargement des informations sur un fichier raster (chemin du fichier, chemin du fichier de masque si applicable, nombre de canaux, boundingbox de l'emprise géographique)
-        * depuis le fichier raster
-        * depuis une liste de paramètres provenant d'une utilisation précédente
-    * Tests unitaires
-    * Documentation interne des fonctions et classes
-
-* Pyramid
-    * Fonctions de gestion de la liste : chargement et lecture (via un generator)
-    * Taille du header d'une dalle stockée dans la variable `ROK4_IMAGE_HEADER_SIZE`
-    * La proriété `tile_extension` : retourne l'extension d'une tuile de la pyramide en fonction du format
-    * Des exemples d'utilisation des fonctions principales
-
-### [Changed]
-
-* Vector
-    * Utilisation de kwargs pour les paramètres du csv
-    * Gestion des CSV par OGR
-    * Passage par get_osgeo_path pour la lecture virtuelle
-    * 2 constructeurs pour les vecteurs : from_file et from_parameters
-
-* README.md
-    * Modification du bloc code de compilation pour utiliser explicitement python3, et installer certaines dépendances.
-* Utils
-    * Fonction de calcul de la boundix box d'une donnée
-    * Fonction de détermination du format de variable des couleurs dans une donéne raster
-
-### [Fixed]
-
-* Storage
-    * Lecture de la taille d'un objet S3 : pas besoin d'enlever des quotes dans le header `Content-Length`
-
 <!--
 ### [Added]
 

src/rok4/Pyramid.py

@@ -302,6 +302,7 @@ def bbox(self) -> Tuple[float, float, float, float]:
         Returns:
             Tuple[float, float, float, float]: level terrain extent (xmin, ymin, xmax, ymax)
         """
+
         min_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox(
             self.__tile_limits["min_col"], self.__tile_limits["max_row"]
         )
@@ -548,7 +549,11 @@ def serializable(self) -> Dict:
         Returns:
             Dict: descriptor structured object description
         """
-        serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format}
+
+        serialization = {
+            "tile_matrix_set": self.__tms.name,
+            "format": self.__format
+        }
 
         serialization["levels"] = []
         sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True)
@@ -615,6 +620,7 @@ def storage_root(self) -> str:
         Returns:
             str: Pyramid's storage root
         """
+
         return self.__storage["root"].split("@", 1)[
             0
         ]  # Suppression de l'éventuel hôte de spécification du cluster S3
@@ -664,6 +670,7 @@ def format(self) -> str:
 
     @property
     def tile_extension(self) -> str:
+
         if self.__format in [
             "TIFF_RAW_UINT8",
             "TIFF_LZW_UINT8",
@@ -828,7 +835,7 @@ def get_level(self, level_id: str) -> "Level":
         Returns:
             The corresponding pyramid's level, None if not present
         """
-
+        
         return self.__levels.get(level_id, None)
 
     def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]:
@@ -913,6 +920,7 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]:
 
     def write_descriptor(self) -> None:
         """Write the pyramid's descriptor to the final location (in the pyramid's storage root)"""
+
         content = json.dumps(self.serializable)
         put_data_str(content, self.__descriptor)
 
@@ -1011,6 +1019,7 @@ def get_slab_path_from_infos(
         else:
             return slab_path
 
+
     def get_tile_data_binary(self, level: str, column: int, row: int) -> str:
         """Get a pyramid's tile as binary string
 
@@ -1173,6 +1182,7 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr
         level_object = self.get_level(level)
 
         if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8":
+
             try:
                 img = Image.open(io.BytesIO(binary_tile))
             except Exception as e:
@@ -1350,3 +1360,25 @@ def get_tile_indices(
             x, y = reproject_point((x, y), sr, self.__tms.sr)
 
         return (level_object.id,) + level_object.tile_matrix.point_to_indices(x, y)
+
+    @property
+    def size(self) -> int:
+        """Get the size of the pyramid
+
+        Examples:
+
+                from rok4.Pyramid import Pyramid
+
+                try:
+                    pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json")
+                    size = pyramid.size()
+
+                except Exception as e:
+                    print("Cannot load the pyramid from its descriptor and get his size")
+
+        Returns:
+            int: size of the pyramid
+        """
+        if not hasattr(self,"_Pyramid__size") :
+            self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name))
+        return self.__size

src/rok4/Storage.py

@@ -128,6 +128,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s
 
 def disconnect_s3_clients() -> None:
     """Clean S3 clients"""
+
     global __S3_CLIENTS, __S3_DEFAULT_CLIENT
     __S3_CLIENTS = {}
     __S3_DEFAULT_CLIENT = None
@@ -180,6 +181,7 @@ def __get_ceph_ioctx(pool: str) -> "rados.Ioctx":
 
 def disconnect_ceph_clients() -> None:
     """Clean CEPH clients"""
+
     global __CEPH_CLIENT, __CEPH_IOCTXS
     __CEPH_CLIENT = None
     __CEPH_IOCTXS = {}
@@ -907,3 +909,59 @@ def get_osgeo_path(path: str) -> str:
 
     else:
         raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}")
+
+def size_path(path: str) -> int :
+    """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list)
+
+    Args:
+        path (str): Source path
+
+    Raises:
+        StorageError: Unhandled link or link issue
+        MissingEnvironmentError: Missing object storage informations
+
+    Returns:
+        int: size of the path
+    """
+    storage_type, unprefixed_path, tray_name, base_name  = get_infos_from_path(path)
+
+    if storage_type == StorageType.FILE:
+        try :
+            total = 0
+            with os.scandir(unprefixed_path) as it:
+                for entry in it:
+                    if entry.is_file():
+                        total += entry.stat().st_size
+                    elif entry.is_dir():
+                        total += size_path(entry.path)
+
+        except Exception as e:
+            raise StorageError("FILE", e)
+
+    elif storage_type == StorageType.S3:
+        s3_client, bucket_name = __get_s3_client(tray_name)
+
+        try :
+            paginator = s3_client["client"].get_paginator('list_objects_v2')
+            pages = paginator.paginate(
+                Bucket=bucket_name,
+                Prefix=base_name+"/",
+                PaginationConfig={
+                    'PageSize': 10000,
+                }
+            )
+            total = 0
+            for page in pages:
+                for key in page['Contents']:
+                    total += key['Size']
+
+        except Exception as e:
+            raise StorageError("S3", e)
+
+
+    elif storage_type == StorageType.CEPH:
+        raise NotImplementedError
+    else:
+        raise StorageError("UNKNOWN", "Unhandled storage type to calculate size")
+
+    return total

tests/test_Storage.py

@@ -21,7 +21,6 @@ def test_hash_file_ok(mock_file):
     except Exception as exc:
         assert False, f"FILE md5 sum raises an exception: {exc}"
 
-
 @mock.patch.dict(os.environ, {}, clear=True)
 def test_get_infos_from_path():
     assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi")
@@ -104,7 +103,6 @@ def test_file_read_ok(mock_file):
     except Exception as exc:
         assert False, f"FILE read raises an exception: {exc}"
 
-
 @mock.patch.dict(
     os.environ,
     {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"},
@@ -119,7 +117,6 @@ def test_s3_read_nok(mocked_s3_client):
     with pytest.raises(StorageError):
         data = get_data_str("s3://bucket/path/to/object")
 
-
 @mock.patch.dict(
     os.environ,
     {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"},
@@ -165,7 +162,6 @@ def test_ceph_read_ok(mocked_rados_client):
 
 ############ put_data_str
 
-
 @mock.patch.dict(
     os.environ,
     {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"},
@@ -377,7 +373,6 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client):
     except Exception as exc:
         assert False, f"CEPH -> FILE copy raises an exception: {exc}"
 
-
 @mock.patch.dict(
     os.environ,
     {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"},
@@ -402,7 +397,6 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client):
     except Exception as exc:
         assert False, f"FILE -> CEPH copy raises an exception: {exc}"
 
-
 @mock.patch.dict(
     os.environ,
     {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"},
@@ -480,7 +474,6 @@ def test_link_hard_nok():
     with pytest.raises(StorageError):
         link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True)
 
-
 @mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch("os.symlink", return_value=None)
 def test_link_file_ok(mock_link):
@@ -500,7 +493,6 @@ def test_hlink_file_ok(mock_link):
     except Exception as exc:
         assert False, f"FILE hard link raises an exception: {exc}"
 
-
 @mock.patch.dict(
     os.environ,
     {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"},
@@ -557,7 +549,6 @@ def test_link_s3_nok(mocked_s3_client):
 
 ############ get_size
 
-
 @mock.patch.dict(os.environ, {}, clear=True)
 @mock.patch("os.stat")
 def test_size_file_ok(mock_stat):
@@ -697,7 +688,6 @@ def test_remove_file_ok(mock_remove):
     except Exception as exc:
         assert False, f"FILE deletion (not found) raises an exception: {exc}"
 
-
 @mock.patch.dict(
     os.environ,
     {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"},
@@ -753,7 +743,6 @@ def test_get_osgeo_path_file_ok():
     except Exception as exc:
         assert False, f"FILE osgeo path raises an exception: {exc}"
 
-
 @mock.patch.dict(
     os.environ,
     {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"},
@@ -773,3 +762,41 @@ def test_get_osgeo_path_s3_ok():
 def test_get_osgeo_path_nok():
     with pytest.raises(NotImplementedError):
         get_osgeo_path("ceph://pool/data.ext")
+
+
+############ size_path
+def test_size_path_file_ok():
+    try:
+        size = size_path("file://tests/fixtures/TIFF_PBF_MVT")
+        assert size == 124838
+    except Exception as exc:
+        assert False, f"FILE size of the path raises an exception: {exc}"
+
+def test_size_file_nok():
+    with pytest.raises(StorageError) :
+        size = size_path("file://tests/fixtures/TIFF_PBF_M")
+
+@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True)
+def test_size_path_ceph_nok():
+
+    with pytest.raises(NotImplementedError):
+        size = size_path("ceph://pool/path")
+
+@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True)
+@mock.patch('rok4.Storage.boto3.client')
+def test_size_path_s3_ok(mocked_s3_client):
+
+    disconnect_s3_clients()
+    pages = [{"Contents" : [{"Size" : 10},{"Size" : 20}]}, {"Contents" : [{"Size" : 50}]}]
+    paginator = MagicMock()
+    paginator.paginate.return_value = pages
+    client = MagicMock()
+    client.get_paginator.return_value = paginator
+    mocked_s3_client.return_value = client
+
+    try:
+        size = size_path("s3://bucket/path")
+        assert size == 80
+    except Exception as exc:
+        assert False, f"S3 size of the path raises an exception: {exc}"
+