From c8a51a0158bfc6bb737d67af77b178c10168f9cd Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 14:50:00 +0200 Subject: [PATCH 1/4] Move StorageType to enums module --- src/rok4/Pyramid.py | 21 ++++---- src/rok4/Storage.py | 92 +++++++++++++++++----------------- src/rok4/enums.py | 14 ++++++ tests/test_Pyramid.py | 2 +- tests/test_Storage.py | 114 +++++++++++++++++++++++++++++------------- 5 files changed, 149 insertions(+), 94 deletions(-) create mode 100644 src/rok4/enums.py diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index 7667244..fd01f1d 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -21,6 +21,7 @@ from rok4.TileMatrixSet import TileMatrixSet, TileMatrix from rok4.Storage import * from rok4.Utils import * +from rok4.enums import StorageType class PyramidType(Enum): @@ -549,11 +550,8 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - - serialization = { - "tile_matrix_set": self.__tms.name, - "format": self.__format - } + + serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) @@ -620,7 +618,7 @@ def storage_root(self) -> str: Returns: str: Pyramid's storage root """ - + return self.__storage["root"].split("@", 1)[ 0 ] # Suppression de l'éventuel hôte de spécification du cluster S3 @@ -670,7 +668,6 @@ def format(self) -> str: @property def tile_extension(self) -> str: - if self.__format in [ "TIFF_RAW_UINT8", "TIFF_LZW_UINT8", @@ -835,7 +832,7 @@ def get_level(self, level_id: str) -> "Level": Returns: The corresponding pyramid's level, None if not present """ - + return self.__levels.get(level_id, None) def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: @@ -1019,7 +1016,6 @@ def get_slab_path_from_infos( else: return slab_path - def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """Get a pyramid's tile as binary string @@ -1182,7 +1178,6 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr level_object = self.get_level(level) if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: @@ -1379,6 +1374,8 @@ def size(self) -> int: Returns: int: size of the pyramid """ - if not hasattr(self,"_Pyramid__size") : - self.__size = size_path(get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name)) + if not hasattr(self, "_Pyramid__size"): + self.__size = size_path( + get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name) + ) return self.__size diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py index bcbee52..406e431 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/Storage.py @@ -45,17 +45,10 @@ gdal.UseExceptions() +from rok4.enums import StorageType from rok4.Exceptions import * -class StorageType(Enum): - FILE = "file://" - S3 = "s3://" - CEPH = "ceph://" - HTTP = "http://" - HTTPS = "https://" - - __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -75,7 +68,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s Returns: Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name """ - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT if not __S3_CLIENTS: @@ -134,7 +127,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", s def disconnect_s3_clients() -> None: """Clean S3 clients""" - + global __S3_CLIENTS, __S3_DEFAULT_CLIENT __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -363,16 +356,15 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - - if range is None : + if range is None: try: reponse = requests.get(f"{storage_type.value}{path}", stream=True) data = reponse.content - if reponse.status_code == 404 : + if reponse.status_code == 404: raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: raise StorageError(storage_type.name, e) - else : + else: raise NotImplementedError else: @@ -471,7 +463,6 @@ def get_size(path: str) -> int: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: # Le stream=True permet de ne télécharger que le header initialement reponse = requests.get(storage_type.value + path, stream=True).headers["content-length"] @@ -526,12 +517,11 @@ def exists(path: str) -> bool: return os.path.exists(path) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: response = requests.get(storage_type.value + path, stream=True) - if response.status_code == 200 : + if response.status_code == 200: return True - else : + else: return False except Exception as e: raise StorageError(storage_type.name, e) @@ -839,43 +829,52 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" ) - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.FILE: try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) with open(to_path, "wb") as f: - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) except Exception as e: - raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH : + raise StorageError( + f"HTTP(S) and FILE", + f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}", + ) + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.CEPH: to_ioctx = __get_ceph_ioctx(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) offset = 0 - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: size = len(chunk) to_ioctx.write(to_base_name, chunk, offset) offset += size except Exception as e: - raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 : + raise StorageError( + f"HTTP(S) and CEPH", + f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}", + ) + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.S3: to_s3_client, to_bucket = __get_s3_client(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) - with tempfile.NamedTemporaryFile("w+b",delete=False) as f: + response = requests.get(from_type.value + from_path, stream=True) + with tempfile.NamedTemporaryFile("w+b", delete=False) as f: name_fich = f.name - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) @@ -884,7 +883,10 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: os.remove(name_fich) except Exception as e: - raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}") + raise StorageError( + f"HTTP(S) and S3", + f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}", + ) else: raise StorageError( @@ -998,7 +1000,8 @@ def get_osgeo_path(path: str) -> str: else: raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}") -def size_path(path: str) -> int : + +def size_path(path: str) -> int: """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list) Args: @@ -1011,10 +1014,10 @@ def size_path(path: str) -> int : Returns: int: size of the path """ - storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) + storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.FILE: - try : + try: total = 0 with os.scandir(unprefixed_path) as it: for entry in it: @@ -1029,24 +1032,23 @@ def size_path(path: str) -> int : elif storage_type == StorageType.S3: s3_client, bucket_name = __get_s3_client(tray_name) - try : - paginator = s3_client["client"].get_paginator('list_objects_v2') + try: + paginator = s3_client["client"].get_paginator("list_objects_v2") pages = paginator.paginate( Bucket=bucket_name, - Prefix=base_name+"/", + Prefix=base_name + "/", PaginationConfig={ - 'PageSize': 10000, - } + "PageSize": 10000, + }, ) total = 0 for page in pages: - for key in page['Contents']: - total += key['Size'] + for key in page["Contents"]: + total += key["Size"] except Exception as e: raise StorageError("S3", e) - elif storage_type == StorageType.CEPH: raise NotImplementedError else: diff --git a/src/rok4/enums.py b/src/rok4/enums.py new file mode 100644 index 0000000..2fca210 --- /dev/null +++ b/src/rok4/enums.py @@ -0,0 +1,14 @@ +#! python3 # noqa: E265 + +# standard lib +from enum import Enum + + +class StorageType(Enum): + """Matrice de correspondance entre type de stockage et protocole.""" + + CEPH = "ceph://" + FILE = "file://" + HTTP = "http://" + HTTPS = "https://" + S3 = "s3://" diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index 6eebfd7..d72ce2f 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -1,6 +1,6 @@ from rok4.Pyramid import * from rok4.TileMatrixSet import TileMatrixSet -from rok4.Storage import StorageType +from rok4.enums import StorageType from rok4.Utils import * from rok4.Exceptions import * diff --git a/tests/test_Storage.py b/tests/test_Storage.py index 9779954..540632c 100644 --- a/tests/test_Storage.py +++ b/tests/test_Storage.py @@ -1,5 +1,6 @@ from rok4.Storage import * from rok4.Exceptions import * +from rok4.enums import StorageType import pytest import os @@ -21,6 +22,7 @@ def test_hash_file_ok(mock_file): except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") @@ -103,6 +105,7 @@ def test_file_read_ok(mock_file): except Exception as exc: assert False, f"FILE read raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -117,6 +120,7 @@ def test_s3_read_nok(mocked_s3_client): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -159,8 +163,9 @@ def test_ceph_read_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH read raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get", side_effect={"status_code":404}) +@mock.patch("requests.get", side_effect={"status_code": 404}) def test_http_read_error(mock_http): with pytest.raises(StorageError): requests_instance = MagicMock() @@ -171,28 +176,31 @@ def test_http_read_error(mock_http): mock_http.assert_called_with("http://path/to/file.ext", stream=True) + @mock.patch.dict(os.environ, {}, clear=True) def test_http_read_range_error(): with pytest.raises(NotImplementedError): - data = get_data_binary("http://path/to/file.ext", (0,100)) + data = get_data_binary("http://path/to/file.ext", (0, 100)) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("requests.get") def test_http_read_ok(mock_http): - try : + try: requests_instance = MagicMock() - requests_instance.content = b'data' + requests_instance.content = b"data" mock_http.return_value = requests_instance data = get_data_str("http://path/to/file.ext") mock_http.assert_called_with("http://path/to/file.ext", stream=True) - assert data == 'data' + assert data == "data" except Exception as exc: assert False, f"HTTP read raises an exception: {exc}" ############ put_data_str + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -225,6 +233,7 @@ def test_s3_write_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 write raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -302,6 +311,7 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): copy("s3://bucket/source.ext", "file:///path/to/destination.ext", "toto") mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -320,6 +330,7 @@ def test_copy_file_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"FILE -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -338,6 +349,7 @@ def test_copy_s3_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -373,6 +385,7 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client): with pytest.raises(StorageError): copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto") + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -399,6 +412,7 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -423,6 +437,7 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client): except Exception as exc: assert False, f"FILE -> CEPH copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -487,14 +502,14 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> S3 copy raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') -@patch('builtins.open', new_callable=mock_open) +@mock.patch("requests.get") +@patch("builtins.open", new_callable=mock_open) def test_copy_http_file_ok(mock_open, mock_requests): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance copy("http://path/to/source.ext", "file:///path/to/destination.ext") @@ -503,17 +518,20 @@ def test_copy_http_file_ok(mock_open, mock_requests): except Exception as exc: assert False, f"HTTP -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -@mock.patch('requests.get') + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("requests.get") def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write.return_value = None @@ -526,16 +544,20 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -@mock.patch('requests.get') -@patch('tempfile.NamedTemporaryFile', new_callable=mock_open) -@mock.patch('os.remove') + +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +@mock.patch("requests.get") +@patch("tempfile.NamedTemporaryFile", new_callable=mock_open) +@mock.patch("os.remove") def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance disconnect_s3_clients() @@ -546,7 +568,7 @@ def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_cl copy("http://path/to/source.ext", "s3://bucket/destination.ext") mock_requests.assert_called_once_with("http://path/to/source.ext", stream=True) - mock_tempfile.assert_called_once_with("w+b",delete=False) + mock_tempfile.assert_called_once_with("w+b", delete=False) except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" @@ -563,6 +585,7 @@ def test_link_hard_nok(): with pytest.raises(StorageError): link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.symlink", return_value=None) def test_link_file_ok(mock_link): @@ -582,6 +605,7 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -619,6 +643,7 @@ def test_link_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -637,6 +662,7 @@ def test_link_s3_nok(mocked_s3_client): ############ get_size + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.stat") def test_size_file_ok(mock_stat): @@ -647,6 +673,7 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -667,6 +694,7 @@ def test_size_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -685,12 +713,12 @@ def test_size_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 size raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_size_http_ok(mock_requests): - http_instance = MagicMock() - http_instance.headers = {"content-length":12} + http_instance.headers = {"content-length": 12} mock_requests.return_value = http_instance try: @@ -717,6 +745,7 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -768,10 +797,10 @@ def test_exists_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"CEPH not exists raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_exists_http_ok(mock_requests): - http_instance = MagicMock() http_instance.status_code = 200 mock_requests.return_value = http_instance @@ -792,6 +821,7 @@ def test_exists_http_ok(mock_requests): ############ remove + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.remove") def test_remove_file_ok(mock_remove): @@ -807,6 +837,7 @@ def test_remove_file_ok(mock_remove): except Exception as exc: assert False, f"FILE deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -832,6 +863,7 @@ def test_remove_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -861,6 +893,7 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -881,6 +914,7 @@ def test_get_osgeo_path_nok(): with pytest.raises(NotImplementedError): get_osgeo_path("ceph://pool/data.ext") + ############ size_path def test_size_path_file_ok(): try: @@ -889,22 +923,31 @@ def test_size_path_file_ok(): except Exception as exc: assert False, f"FILE size of the path raises an exception: {exc}" + def test_size_file_nok(): - with pytest.raises(StorageError) : + with pytest.raises(StorageError): size = size_path("file://tests/fixtures/TIFF_PBF_M") -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -def test_size_path_ceph_nok(): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +def test_size_path_ceph_nok(): with pytest.raises(NotImplementedError): size = size_path("ceph://pool/path") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -def test_size_path_s3_ok(mocked_s3_client): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +def test_size_path_s3_ok(mocked_s3_client): disconnect_s3_clients() - pages = [{"Contents" : [{"Size" : 10},{"Size" : 20}]}, {"Contents" : [{"Size" : 50}]}] + pages = [{"Contents": [{"Size": 10}, {"Size": 20}]}, {"Contents": [{"Size": 50}]}] paginator = MagicMock() paginator.paginate.return_value = pages client = MagicMock() @@ -916,4 +959,3 @@ def test_size_path_s3_ok(mocked_s3_client): assert size == 80 except Exception as exc: assert False, f"S3 size of the path raises an exception: {exc}" - From 175152bf6cf1f17fc71b0a8ba14d2d7895829c31 Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 14:55:22 +0200 Subject: [PATCH 2/4] Move Pyramidtype to enums module --- src/rok4/Layer.py | 3 ++- src/rok4/Pyramid.py | 9 +-------- src/rok4/enums.py | 7 +++++++ tests/test_Layer.py | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/rok4/Layer.py b/src/rok4/Layer.py index 316a9ca..e57e3ab 100644 --- a/src/rok4/Layer.py +++ b/src/rok4/Layer.py @@ -12,10 +12,11 @@ import re from rok4.Exceptions import * -from rok4.Pyramid import Pyramid, PyramidType +from rok4.Pyramid import Pyramid from rok4.TileMatrixSet import TileMatrixSet from rok4.Storage import * from rok4.Utils import * +from rok4.enums import PyramidType class Layer: diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index fd01f1d..0d176d4 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -21,14 +21,7 @@ from rok4.TileMatrixSet import TileMatrixSet, TileMatrix from rok4.Storage import * from rok4.Utils import * -from rok4.enums import StorageType - - -class PyramidType(Enum): - """Pyramid's data type""" - - RASTER = "RASTER" - VECTOR = "VECTOR" +from rok4.enums import PyramidType, StorageType class SlabType(Enum): diff --git a/src/rok4/enums.py b/src/rok4/enums.py index 2fca210..9a0a486 100644 --- a/src/rok4/enums.py +++ b/src/rok4/enums.py @@ -4,6 +4,13 @@ from enum import Enum +class PyramidType(Enum): + """Pyramid's data type""" + + RASTER = "RASTER" + VECTOR = "VECTOR" + + class StorageType(Enum): """Matrice de correspondance entre type de stockage et protocole.""" diff --git a/tests/test_Layer.py b/tests/test_Layer.py index 0a60d5c..7a3788e 100644 --- a/tests/test_Layer.py +++ b/tests/test_Layer.py @@ -1,5 +1,5 @@ from rok4.Layer import Layer -from rok4.Pyramid import PyramidType +from rok4.enums import PyramidType from rok4.Exceptions import * import pytest From 5672c878a517804a33eff3ad7abc9d924581219f Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 15:02:21 +0200 Subject: [PATCH 3/4] Move SlabType to enums module --- src/rok4/Pyramid.py | 9 +-------- src/rok4/enums.py | 7 +++++++ tests/test_Pyramid.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index 0d176d4..0d7e382 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -21,14 +21,7 @@ from rok4.TileMatrixSet import TileMatrixSet, TileMatrix from rok4.Storage import * from rok4.Utils import * -from rok4.enums import PyramidType, StorageType - - -class SlabType(Enum): - """Slab's type""" - - DATA = "DATA" # Slab of data, raster or vector - MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data +from rok4.enums import PyramidType, SlabType, StorageType ROK4_IMAGE_HEADER_SIZE = 2048 diff --git a/src/rok4/enums.py b/src/rok4/enums.py index 9a0a486..76300ce 100644 --- a/src/rok4/enums.py +++ b/src/rok4/enums.py @@ -11,6 +11,13 @@ class PyramidType(Enum): VECTOR = "VECTOR" +class SlabType(Enum): + """Slab's type""" + + DATA = "DATA" # Slab of data, raster or vector + MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data + + class StorageType(Enum): """Matrice de correspondance entre type de stockage et protocole.""" diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index d72ce2f..60576f3 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -1,6 +1,6 @@ from rok4.Pyramid import * from rok4.TileMatrixSet import TileMatrixSet -from rok4.enums import StorageType +from rok4.enums import SlabType, StorageType from rok4.Utils import * from rok4.Exceptions import * From 726a18dfb4bacbfb919e9c7dbbc182e13b66afff Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Tue, 18 Jul 2023 15:12:11 +0200 Subject: [PATCH 4/4] Rm unused ColorFormat --- src/rok4/Utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/rok4/Utils.py b/src/rok4/Utils.py index da15aef..627686c 100644 --- a/src/rok4/Utils.py +++ b/src/rok4/Utils.py @@ -6,25 +6,12 @@ from typing import Dict, List, Tuple, Union from osgeo import ogr, osr, gdal -from enum import Enum ogr.UseExceptions() osr.UseExceptions() gdal.UseExceptions() -class ColorFormat(Enum): - """A color format enumeration. - Except from "BIT", the member's name matches - a common variable format name. The member's value is - the allocated bit size associated to this format. - """ - - BIT = 1 - UINT8 = 8 - FLOAT32 = 32 - - __SR_BOOK = {}