diff --git a/src/rok4/enums.py b/src/rok4/enums.py index 861464f..8ca58ba 100644 --- a/src/rok4/enums.py +++ b/src/rok4/enums.py @@ -27,6 +27,7 @@ class StorageType(Enum): HTTPS = "https://" S3 = "s3://" + class ColorFormat(Enum): """A color format enumeration. Except from "BIT", the member's name matches @@ -36,4 +37,4 @@ class ColorFormat(Enum): BIT = 1 UINT8 = 8 - FLOAT32 = 32 \ No newline at end of file + FLOAT32 = 32 diff --git a/src/rok4/layer.py b/src/rok4/layer.py index 631a870..6a72d0c 100644 --- a/src/rok4/layer.py +++ b/src/rok4/layer.py @@ -5,18 +5,18 @@ - `Layer` - Descriptor to broadcast pyramids' data """ -from typing import Dict, List, Tuple, Union import json -from json.decoder import JSONDecodeError import os import re +from json.decoder import JSONDecodeError +from typing import Dict, List, Tuple, Union +from rok4.enums import PyramidType from rok4.exceptions import * from rok4.pyramid import Pyramid -from rok4.tile_matrix_set import TileMatrixSet from rok4.storage import * +from rok4.tile_matrix_set import TileMatrixSet from rok4.utils import * -from rok4.enums import PyramidType class Layer: diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py index ebb63b0..a2ae4a9 100644 --- a/src/rok4/pyramid.py +++ b/src/rok4/pyramid.py @@ -6,22 +6,23 @@ - `Level` - Level of a pyramid """ -from typing import Dict, List, Tuple, Union, Iterator +import io import json -from json.decoder import JSONDecodeError import os import re -import numpy import zlib -import io +from json.decoder import JSONDecodeError +from typing import Dict, Iterator, List, Tuple, Union + import mapbox_vector_tile +import numpy from PIL import Image +from rok4.enums import PyramidType, SlabType, StorageType from rok4.exceptions import * -from rok4.tile_matrix_set import TileMatrixSet, TileMatrix from rok4.storage import * +from rok4.tile_matrix_set import TileMatrix, TileMatrixSet from rok4.utils import * -from rok4.enums import PyramidType, SlabType, StorageType ROK4_IMAGE_HEADER_SIZE = 2048 """Slab's header size, 2048 bytes""" @@ -535,11 +536,8 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - - serialization = { - "tile_matrix_set": self.__tms.name, - "format": self.__format - } + + serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) @@ -656,7 +654,6 @@ def format(self) -> str: @property def tile_extension(self) -> str: - if self.__format in [ "TIFF_RAW_UINT8", "TIFF_LZW_UINT8", @@ -732,7 +729,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] S3 stored descriptor from rok4.pyramid import Pyramid - + try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/descriptor.json") @@ -756,7 +753,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] 'slab': 'DATA_18_5424_7526' } ) - + Raises: StorageError: Unhandled pyramid storage to copy list MissingEnvironmentError: Missing object storage informations @@ -774,7 +771,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict] roots = {} s3_cluster = self.storage_s3_cluster - with open(list_file, "r") as listin: + with open(list_file) as listin: # Lecture des racines for line in listin: line = line.rstrip() @@ -1170,7 +1167,6 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr level_object = self.get_level(level) if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: @@ -1368,10 +1364,10 @@ def size(self) -> int: Returns: int: size of the pyramid """ - + if not hasattr(self, "_Pyramid__size"): self.__size = size_path( get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name) ) - + return self.__size diff --git a/src/rok4/raster.py b/src/rok4/raster.py index 95ccf68..3d2d813 100644 --- a/src/rok4/raster.py +++ b/src/rok4/raster.py @@ -9,14 +9,13 @@ import copy import json import re - from typing import Dict, Tuple from osgeo import gdal, ogr +from rok4.enums import ColorFormat from rok4.storage import exists, get_osgeo_path, put_data_str from rok4.utils import compute_bbox, compute_format -from rok4.enums import ColorFormat # Enable GDAL/OGR exceptions ogr.UseExceptions() @@ -239,7 +238,7 @@ def from_list(cls, path: str, srs: str) -> "RasterSet": local_list_path = get_osgeo_path(path) image_list = [] - with open(file=local_list_path, mode="r") as list_file: + with open(file=local_list_path) as list_file: for line in list_file: image_path = line.strip(" \t\n\r") image_list.append(image_path) @@ -298,7 +297,7 @@ def from_descriptor(cls, path: str) -> "RasterSet": """ self = cls() descriptor_path = get_osgeo_path(path) - with open(file=descriptor_path, mode="r") as file_handle: + with open(file=descriptor_path) as file_handle: raw_content = file_handle.read() serialization = json.loads(raw_content) self.srs = serialization["srs"] diff --git a/src/rok4/storage.py b/src/rok4/storage.py index 6e2eca8..f0a5ffa 100644 --- a/src/rok4/storage.py +++ b/src/rok4/storage.py @@ -30,23 +30,23 @@ To precise the cluster to use, bucket name should be bucket_name@s3.storage.fr or bucket_name@s4.storage.fr. If no host is defined (no @) in the bucket name, first S3 cluster is used """ +import hashlib +import os +import re +import tempfile +from shutil import copyfile +from typing import Dict, List, Tuple, Union + import boto3 import botocore.exceptions -import tempfile -import re -import os import rados -import hashlib import requests -from typing import Dict, List, Tuple, Union -from shutil import copyfile from osgeo import gdal gdal.UseExceptions() -from rok4.exceptions import * from rok4.enums import StorageType - +from rok4.exceptions import * __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None @@ -355,16 +355,15 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - - if range is None : + if range is None: try: reponse = requests.get(f"{storage_type.value}{path}", stream=True) data = reponse.content - if reponse.status_code == 404 : + if reponse.status_code == 404: raise FileNotFoundError(f"{storage_type.value}{path}") except Exception as e: raise StorageError(storage_type.name, e) - else : + else: raise NotImplementedError else: @@ -463,7 +462,6 @@ def get_size(path: str) -> int: raise StorageError("FILE", e) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: # Le stream=True permet de ne télécharger que le header initialement reponse = requests.get(storage_type.value + path, stream=True).headers["content-length"] @@ -518,12 +516,11 @@ def exists(path: str) -> bool: return os.path.exists(path) elif storage_type == StorageType.HTTP or storage_type == StorageType.HTTPS: - try: response = requests.get(storage_type.value + path, stream=True) - if response.status_code == 200 : + if response.status_code == 200: return True - else : + else: return False except Exception as e: raise StorageError(storage_type.name, e) @@ -831,47 +828,52 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" ) - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.FILE : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.FILE: try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) with open(to_path, "wb") as f: - for chunk in response.iter_content(chunk_size=65536) : - + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) except Exception as e: + raise StorageError( + f"HTTP(S) and FILE", + f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}", + ) - raise StorageError(f"HTTP(S) and FILE", f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.CEPH : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.CEPH: to_ioctx = __get_ceph_ioctx(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) + response = requests.get(from_type.value + from_path, stream=True) offset = 0 - for chunk in response.iter_content(chunk_size=65536) : + for chunk in response.iter_content(chunk_size=65536): if chunk: size = len(chunk) to_ioctx.write(to_base_name, chunk, offset) offset += size except Exception as e: + raise StorageError( + f"HTTP(S) and CEPH", + f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}", + ) - raise StorageError(f"HTTP(S) and CEPH", f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}") - - elif (from_type == StorageType.HTTP or from_type == StorageType.HTTPS) and to_type == StorageType.S3 : - + elif ( + from_type == StorageType.HTTP or from_type == StorageType.HTTPS + ) and to_type == StorageType.S3: to_s3_client, to_bucket = __get_s3_client(to_tray) try: - response = requests.get(from_type.value + from_path, stream = True) - with tempfile.NamedTemporaryFile("w+b",delete=False) as f: + response = requests.get(from_type.value + from_path, stream=True) + with tempfile.NamedTemporaryFile("w+b", delete=False) as f: name_fich = f.name - for chunk in response.iter_content(chunk_size=65536) : - + for chunk in response.iter_content(chunk_size=65536): if chunk: f.write(chunk) @@ -880,8 +882,10 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: os.remove(name_fich) except Exception as e: - raise StorageError(f"HTTP(S) and S3", f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}") - + raise StorageError( + f"HTTP(S) and S3", + f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}", + ) else: raise StorageError( @@ -928,9 +932,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: try: target_s3_client["client"].put_object( - Body=f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode( - "utf-8" - ), + Body=f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode(), Bucket=link_bucket, Key=link_base_name, ) @@ -941,9 +943,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: ioctx = __get_ceph_ioctx(link_tray) try: - ioctx.write_full( - link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode("utf-8") - ) + ioctx.write_full(link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode()) except Exception as e: raise StorageError("CEPH", e) @@ -995,7 +995,8 @@ def get_osgeo_path(path: str) -> str: else: raise NotImplementedError(f"Cannot get a GDAL/OGR compliant path from {path}") -def size_path(path: str) -> int : + +def size_path(path: str) -> int: """Return the size of the path given (or, for the CEPH, the sum of the size of each object of the .list) Args: @@ -1008,10 +1009,10 @@ def size_path(path: str) -> int : Returns: int: size of the path """ - storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) + storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.FILE: - try : + try: total = 0 with os.scandir(unprefixed_path) as it: for entry in it: @@ -1026,20 +1027,19 @@ def size_path(path: str) -> int : elif storage_type == StorageType.S3: s3_client, bucket_name = __get_s3_client(tray_name) - - try : - paginator = s3_client["client"].get_paginator('list_objects_v2') + try: + paginator = s3_client["client"].get_paginator("list_objects_v2") pages = paginator.paginate( Bucket=bucket_name, - Prefix=base_name+"/", + Prefix=base_name + "/", PaginationConfig={ - 'PageSize': 10000, - } + "PageSize": 10000, + }, ) total = 0 for page in pages: - for key in page['Contents']: - total += key['Size'] + for key in page["Contents"]: + total += key["Size"] except Exception as e: raise StorageError("S3", e) diff --git a/src/rok4/tile_matrix_set.py b/src/rok4/tile_matrix_set.py index b6685bd..976a63e 100644 --- a/src/rok4/tile_matrix_set.py +++ b/src/rok4/tile_matrix_set.py @@ -9,15 +9,16 @@ - ROK4_TMS_DIRECTORY """ -from typing import Dict, List, Tuple -from json.decoder import JSONDecodeError import json import os +from json.decoder import JSONDecodeError +from typing import Dict, List, Tuple from rok4.exceptions import * from rok4.storage import get_data_str from rok4.utils import * + class TileMatrix: """A tile matrix is a tile matrix set's level. diff --git a/src/rok4/utils.py b/src/rok4/utils.py index 4db30d4..004852b 100644 --- a/src/rok4/utils.py +++ b/src/rok4/utils.py @@ -3,7 +3,6 @@ import os import re - from typing import Dict, List, Tuple, Union from osgeo import gdal, ogr, osr diff --git a/src/rok4/vector.py b/src/rok4/vector.py index c3c045b..2cdb0a2 100644 --- a/src/rok4/vector.py +++ b/src/rok4/vector.py @@ -6,15 +6,17 @@ """ -from osgeo import ogr import os import tempfile +from osgeo import ogr + # Enable GDAL/OGR exceptions ogr.UseExceptions() -from rok4.storage import get_osgeo_path, copy from rok4.exceptions import * +from rok4.storage import copy, get_osgeo_path + class Vector: """A data vector diff --git a/tests/test_layer.py b/tests/test_layer.py index a9a79a2..9cd314b 100644 --- a/tests/test_layer.py +++ b/tests/test_layer.py @@ -1,12 +1,13 @@ -import pytest import os - -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest -from rok4.layer import Layer from rok4.enums import PyramidType from rok4.exceptions import * +from rok4.layer import Layer + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch( diff --git a/tests/test_pyramid.py b/tests/test_pyramid.py index fa04a01..e71dd15 100644 --- a/tests/test_pyramid.py +++ b/tests/test_pyramid.py @@ -1,13 +1,15 @@ -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest +from rok4.enums import SlabType, StorageType +from rok4.exceptions import * from rok4.pyramid import * from rok4.tile_matrix_set import TileMatrixSet -from rok4.enums import SlabType, StorageType from rok4.utils import * -from rok4.exceptions import * + @mock.patch("rok4.pyramid.get_data_str", side_effect=StorageError("FILE", "Not found")) def test_wrong_file(mocked_get_data_str): diff --git a/tests/test_raster.py b/tests/test_raster.py index c91f527..8bf80f0 100644 --- a/tests/test_raster.py +++ b/tests/test_raster.py @@ -14,8 +14,8 @@ import pytest -from rok4.raster import Raster, RasterSet from rok4.enums import ColorFormat +from rok4.raster import Raster, RasterSet # rok4.raster.Raster class tests diff --git a/tests/test_storage.py b/tests/test_storage.py index d60e8c0..f2649e5 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,15 +1,15 @@ -import pytest import os +from unittest import mock +from unittest.mock import * import botocore.exceptions +import pytest from rados import ObjectNotFound -from unittest import mock -from unittest.mock import * - -from rok4.storage import * -from rok4.exceptions import * from rok4.enums import StorageType +from rok4.exceptions import * +from rok4.storage import * + @mock.patch.dict(os.environ, {}, clear=True) @patch("builtins.open", new_callable=mock_open, read_data=b"data") @@ -21,6 +21,7 @@ def test_hash_file_ok(mock_file): except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") @@ -118,6 +119,7 @@ def test_s3_read_nok(mocked_s3_client): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -160,8 +162,9 @@ def test_ceph_read_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH read raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch("requests.get", side_effect={"status_code":404}) +@mock.patch("requests.get", side_effect={"status_code": 404}) def test_http_read_error(mock_http): with pytest.raises(StorageError): requests_instance = MagicMock() @@ -172,18 +175,19 @@ def test_http_read_error(mock_http): mock_http.assert_called_with("http://path/to/file.ext", stream=True) + @mock.patch.dict(os.environ, {}, clear=True) def test_http_read_range_error(): with pytest.raises(NotImplementedError): - data = get_data_binary("http://path/to/file.ext", (0,100)) + data = get_data_binary("http://path/to/file.ext", (0, 100)) + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("requests.get") def test_http_read_ok(mock_http): - - try : + try: requests_instance = MagicMock() - requests_instance.content = b'data' + requests_instance.content = b"data" mock_http.return_value = requests_instance data = get_data_str("http://path/to/file.ext") @@ -195,6 +199,7 @@ def test_http_read_ok(mock_http): ############ put_data_str + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -227,6 +232,7 @@ def test_s3_write_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 write raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -304,6 +310,7 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): copy("s3://bucket/source.ext", "file:///path/to/destination.ext", "toto") mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -322,6 +329,7 @@ def test_copy_file_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"FILE -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -340,6 +348,7 @@ def test_copy_s3_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 -> S3 copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -402,6 +411,7 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -491,14 +501,14 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> S3 copy raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') -@patch('builtins.open', new_callable=mock_open) +@mock.patch("requests.get") +@patch("builtins.open", new_callable=mock_open) def test_copy_http_file_ok(mock_open, mock_requests): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance copy("http://path/to/source.ext", "file:///path/to/destination.ext") @@ -507,14 +517,18 @@ def test_copy_http_file_ok(mock_open, mock_requests): except Exception as exc: assert False, f"HTTP -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.storage.rados.Rados') -@mock.patch('requests.get') + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.storage.rados.Rados") +@mock.patch("requests.get") def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance disconnect_ceph_clients() @@ -529,16 +543,20 @@ def test_copy_http_ceph_ok(mock_requests, mocked_rados_client): except Exception as exc: assert False, f"HTTP -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.storage.boto3.client') -@mock.patch('requests.get') -@patch('tempfile.NamedTemporaryFile', new_callable=mock_open) -@mock.patch('os.remove') + +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.storage.boto3.client") +@mock.patch("requests.get") +@patch("tempfile.NamedTemporaryFile", new_callable=mock_open) +@mock.patch("os.remove") def test_copy_http_s3_ok(mock_remove, mock_tempfile, mock_requests, mocked_s3_client): try: - http_instance = MagicMock() - http_instance.iter_content.return_value = ["data","data2"] + http_instance.iter_content.return_value = ["data", "data2"] mock_requests.return_value = http_instance disconnect_s3_clients() @@ -586,6 +604,7 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -642,6 +661,7 @@ def test_link_s3_nok(mocked_s3_client): ############ get_size + @mock.patch.dict(os.environ, {}, clear=True) @mock.patch("os.stat") def test_size_file_ok(mock_stat): @@ -652,6 +672,7 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -672,6 +693,7 @@ def test_size_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH size raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -690,12 +712,12 @@ def test_size_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 size raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_size_http_ok(mock_requests): - http_instance = MagicMock() - http_instance.headers = {"content-length":12} + http_instance.headers = {"content-length": 12} mock_requests.return_value = http_instance try: @@ -722,6 +744,7 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, @@ -773,10 +796,10 @@ def test_exists_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"CEPH not exists raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('requests.get') +@mock.patch("requests.get") def test_exists_http_ok(mock_requests): - http_instance = MagicMock() http_instance.status_code = 200 mock_requests.return_value = http_instance @@ -839,6 +862,7 @@ def test_remove_ceph_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH deletion (not found) raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -868,6 +892,7 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" + @mock.patch.dict( os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, @@ -888,6 +913,7 @@ def test_get_osgeo_path_nok(): with pytest.raises(NotImplementedError): get_osgeo_path("ceph://pool/data.ext") + ############ size_path def test_size_path_file_ok(): try: @@ -898,21 +924,29 @@ def test_size_path_file_ok(): def test_size_file_nok(): - with pytest.raises(StorageError) : + with pytest.raises(StorageError): size = size_path("file://tests/fixtures/TIFF_PBF_M") -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -def test_size_path_ceph_nok(): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +def test_size_path_ceph_nok(): with pytest.raises(NotImplementedError): size = size_path("ceph://pool/path") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.storage.boto3.client') -def test_size_path_s3_ok(mocked_s3_client): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.storage.boto3.client") +def test_size_path_s3_ok(mocked_s3_client): disconnect_s3_clients() - pages = [{"Contents" : [{"Size" : 10},{"Size" : 20}]}, {"Contents" : [{"Size" : 50}]}] + pages = [{"Contents": [{"Size": 10}, {"Size": 20}]}, {"Contents": [{"Size": 50}]}] paginator = MagicMock() paginator.paginate.return_value = pages client = MagicMock() @@ -924,4 +958,3 @@ def test_size_path_s3_ok(mocked_s3_client): assert size == 80 except Exception as exc: assert False, f"S3 size of the path raises an exception: {exc}" - diff --git a/tests/test_tile_matrix_set.py b/tests/test_tile_matrix_set.py index 98bccd6..faf03f1 100644 --- a/tests/test_tile_matrix_set.py +++ b/tests/test_tile_matrix_set.py @@ -1,11 +1,12 @@ - -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest -from rok4.tile_matrix_set import TileMatrixSet from rok4.exceptions import * +from rok4.tile_matrix_set import TileMatrixSet + @mock.patch.dict(os.environ, {}, clear=True) def test_missing_env(): diff --git a/tests/test_utils.py b/tests/test_utils.py index bfcb636..0b73b9e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,14 +1,15 @@ -import pytest -import os -from osgeo import gdal, osr import math +import os import random - -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest +from osgeo import gdal, osr -from rok4.utils import * from rok4.exceptions import * +from rok4.utils import * + def test_srs_to_spatialreference_ignf_ok(): try: @@ -75,6 +76,7 @@ def test_reproject_point_ok(): # Tests for the rok4.utils.compute_bbox function. + def test_compute_bbox_epsg_3857_ok(): try: mocked_datasource = MagicMock(gdal.Dataset) @@ -215,6 +217,7 @@ def test_compute_bbox_no_srs_ok(): # Tests for the rok4.utils.compute_format function. + @mock.patch("rok4.utils.gdal.Info") @mock.patch("rok4.utils.gdal.GetColorInterpretationName", return_value="Palette") @mock.patch("rok4.utils.gdal.GetDataTypeSize", return_value=8) diff --git a/tests/test_vector.py b/tests/test_vector.py index 513c559..8f75618 100644 --- a/tests/test_vector.py +++ b/tests/test_vector.py @@ -1,11 +1,13 @@ -import pytest import os -from unittest.mock import * from unittest import mock +from unittest.mock import * + +import pytest -from rok4.vector import * from rok4.exceptions import * from rok4.storage import disconnect_ceph_clients +from rok4.vector import * + @mock.patch.dict(os.environ, {}, clear=True) def test_missing_env():