From 3d239d2cc715fa9c59290c3423828933b030c7ea Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Thu, 11 May 2023 15:53:14 +0200 Subject: [PATCH] Apply basic git hooks --- .coveragerc | 2 +- .github/workflows/build-and-release.yaml | 11 +- LICENSE.txt | 4 +- README.pypi.md | 2 +- setup.py | 2 +- src/rok4/Layer.py | 19 ++-- src/rok4/Pyramid.py | 134 +++++++++++------------ src/rok4/Raster.py | 4 +- src/rok4/Storage.py | 60 +++++----- src/rok4/TileMatrixSet.py | 13 +-- src/rok4/Utils.py | 18 +-- src/rok4/__init__.py | 2 +- tests/test_Layer.py | 4 +- tests/test_Pyramid.py | 12 +- tests/test_Raster.py | 3 +- tests/test_Storage.py | 72 ++++++------ tests/test_TileMatrixSet.py | 2 +- tests/test_Utils.py | 7 +- tests/test_Vector.py | 1 - 19 files changed, 183 insertions(+), 189 deletions(-) diff --git a/.coveragerc b/.coveragerc index 7f85b15..5877884 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,5 @@ [run] -include = +include = */src/* omit = */__init__.py diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 40fb13e..b5cc6e6 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -23,8 +23,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ github.ref_name }} - release_name: Release ${{ github.ref_name }} + tag_name: ${{ github.ref_name }} + release_name: Release ${{ github.ref_name }} body_path: CHANGELOG.md draft: false prerelease: false @@ -135,7 +135,7 @@ jobs: asset_content_type: application/zip - name: Add tarball package to release - id: upload-release-targz + id: upload-release-targz uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -159,7 +159,7 @@ jobs: commit_documentation: name: Add documentation and unit tests results into gh-pages branch needs: build_and_test - if: "always()&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" + if: "always()&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status=='success')" runs-on: ubuntu-latest steps: @@ -208,7 +208,7 @@ jobs: delete_version: name: Remove release and tag if error occured needs: build_and_test - if: "always()&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status!='success')" + if: "always()&&(needs.create_release.outputs.job_status=='success')&&(needs.build_and_test.outputs.job_status!='success')" runs-on: ubuntu-latest steps: @@ -219,4 +219,3 @@ jobs: delete_release: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/LICENSE.txt b/LICENSE.txt index d6eb151..2bb09b4 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -122,7 +122,7 @@ These expressions may be used both in singular and plural form. The purpose of the Agreement is the grant by the Licensor to the Licensee of a non-exclusive, transferable and worldwide license for the Software as set forth in Article 5 hereinafter for the whole term of the -protection granted by the rights over said Software. +protection granted by the rights over said Software. Article 3 - ACCEPTANCE @@ -268,7 +268,7 @@ When the Licensee creates Derivative Software, this Derivative Software may be distributed under a license agreement other than this Agreement, subject to compliance with the requirement to include a notice concerning the rights over the Software as defined in Article 6.4. -In the event the creation of the Derivative Software required modification +In the event the creation of the Derivative Software required modification of the Source Code, the Licensee undertakes that: 1. the resulting Modified Software will be governed by this Agreement, diff --git a/README.pypi.md b/README.pypi.md index 1f0bf75..fc3417b 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -24,4 +24,4 @@ except Exception as exc: print(exc) ``` -More examples in the developer documentation \ No newline at end of file +More examples in the developer documentation diff --git a/setup.py b/setup.py index fc1f76c..6068493 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,3 @@ from setuptools import setup -setup() \ No newline at end of file +setup() diff --git a/src/rok4/Layer.py b/src/rok4/Layer.py index 1c9f7b3..3f421f3 100644 --- a/src/rok4/Layer.py +++ b/src/rok4/Layer.py @@ -48,7 +48,7 @@ def from_descriptor(cls, descriptor: str) -> 'Layer': Returns: Layer: a Layer instance - """ + """ try: data = json.loads(get_data_str(descriptor)) @@ -72,7 +72,7 @@ def from_descriptor(cls, descriptor: str) -> 'Layer': layer.__keywords.append(k) - if layer.type == PyramidType.RASTER: + if layer.type == PyramidType.RASTER: if "resampling" in data: layer.__resampling = data["resampling"] @@ -145,7 +145,7 @@ def from_parameters(cls, pyramids: List[Dict[str, str]], name: str, **kwargs) -> else: layer.__abstract = name - if layer.type == PyramidType.RASTER: + if layer.type == PyramidType.RASTER: if "styles" in kwargs and kwargs["styles"] is not None and len(kwargs["styles"]) > 0: layer.__styles = kwargs["styles"] else: @@ -161,7 +161,7 @@ def __init__(self) -> None: self.__format = None self.__tms = None self.__best_level = None - self.__levels = dict() + self.__levels = {} self.__keywords = [] self.__pyramids = [] @@ -176,7 +176,7 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None: Exception: Pyramids' do not all own the same TMS Exception: Pyramids' do not all own the same channels number Exception: Overlapping in usage pyramids' levels - """ + """ ## Toutes les pyramides doivent avoir les même caractéristiques channels = None @@ -227,12 +227,12 @@ def __str__(self) -> str: return f"{self.type.name} layer '{self.__name}'" @property - def serializable(self) -> Dict: + def serializable(self) -> Dict: """Get the dict version of the layer object, descriptor compliant Returns: Dict: descriptor structured object description - """ + """ serialization = { "title": self.__title, "abstract": self.__abstract, @@ -273,7 +273,7 @@ def serializable(self) -> Dict: if self.__tms.srs.upper() not in serialization["wms"]["crs"]: serialization["wms"]["crs"].append(self.__tms.srs.upper()) - + serialization["styles"] = self.__styles serialization["resampling"] = self.__resampling @@ -284,7 +284,7 @@ def write_descriptor(self, directory: str = None) -> None: Args: directory (str, optional): Directory (file or object) where to print the layer's descriptor, called .json. Defaults to None, JSON is printed to standard output. - """ + """ content = json.dumps(self.serializable) if directory is None: @@ -306,4 +306,3 @@ def bbox(self) -> Tuple[float, float, float, float]: @property def geobbox(self) -> Tuple[float, float, float, float]: return self.__geobbox - diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index ab0a885..75b433c 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -81,7 +81,7 @@ def b36_path_decode(path: str) -> Tuple[int, int]: Returns: Tuple[int, int]: slab's column and row - """ + """ path = path.replace('/', '') path = re.sub(r'(\.TIFF?)', "", path.upper()) @@ -107,7 +107,7 @@ def b36_path_encode(column: int, row: int, slashs: int) -> str: Returns: str: base-36 based path - """ + """ b36_column = b36_number_encode(column) b36_row = b36_number_encode(row) @@ -156,7 +156,7 @@ def from_descriptor(cls, data: Dict, pyramid: 'Pyramid') -> 'Level': Returns: Pyramid: a Level instance - """ + """ level = cls() level.__pyramid = pyramid @@ -171,9 +171,9 @@ def from_descriptor(cls, data: Dict, pyramid: 'Pyramid') -> 'Level': if pyramid.storage_type.name != data["storage"]["type"]: raise Exception(f"Pyramid {pyramid.descriptor} owns levels using different storage types ({ data['storage']['type'] }) than its one ({pyramid.storage_type.name})") - if pyramid.storage_type == StorageType.FILE: - pyramid.storage_depth = data["storage"]["path_depth"] - + if pyramid.storage_type == StorageType.FILE: + pyramid.storage_depth = data["storage"]["path_depth"] + if "mask_directory" in data["storage"] or "mask_prefix" in data["storage"]: if not pyramid.own_masks: raise Exception(f"Pyramid {pyramid.__descriptor} does not define a mask format but level {level.__id} define mask storage informations") @@ -229,12 +229,12 @@ def __str__(self) -> str: @property - def serializable(self) -> Dict: + def serializable(self) -> Dict: """Get the dict version of the pyramid object, pyramid's descriptor compliant Returns: Dict: pyramid's descriptor structured object description - """ + """ serialization = { "id": self.__id, "tiles_per_width": self.__slab_size[0], @@ -275,35 +275,35 @@ def serializable(self) -> Dict: return serialization @property - def id(self) -> str: + def id(self) -> str: return self.__id @property - def bbox(self) -> Tuple[float, float, float, float]: + def bbox(self) -> Tuple[float, float, float, float]: """Return level extent, based on tile limits Returns: Tuple[float, float, float, float]: level terrain extent (xmin, ymin, xmax, ymax) - """ + """ min_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox(self.__tile_limits["min_col"], self.__tile_limits["max_row"]) max_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox(self.__tile_limits["max_col"], self.__tile_limits["min_row"]) return (min_bbox[0], min_bbox[1], max_bbox[2], max_bbox[3]) @property - def resolution(self) -> str: + def resolution(self) -> str: return self.__pyramid.tms.get_level(self.__id).resolution @property - def tile_matrix(self) -> TileMatrix: + def tile_matrix(self) -> TileMatrix: return self.__pyramid.tms.get_level(self.__id) @property - def slab_width(self) -> int: + def slab_width(self) -> int: return self.__slab_size[0] @property - def slab_height(self) -> int: + def slab_height(self) -> int: return self.__slab_size[1] def is_in_limits(self, column: int, row: int) -> bool: @@ -325,7 +325,7 @@ def set_limits_from_bbox(self, bbox: Tuple[float, float, float, float]) -> None: bbox (Tuple[float, float, float, float]): terrain extent (xmin, ymin, xmax, ymax), in TMS coordinates system """ - + col_min, row_min, col_max, row_max = self.__pyramid.tms.get_level(self.__id).bbox_to_tiles(bbox) self.__tile_limits = { "min_row": row_min, @@ -348,9 +348,9 @@ class Pyramid: __format (str): Data format __storage (Dict[str, Union[rok4.Storage.StorageType,str,int]]): Pyramid's storage informations (type, root and depth if FILE storage) __raster_specifications (Dict): If raster pyramid, raster specifications - __content (Dict): Loading status (loaded) and list content (cache). + __content (Dict): Loading status (loaded) and list content (cache). - Example (S3 storage): + Example (S3 storage): { 'cache': { @@ -392,7 +392,7 @@ def from_descriptor(cls, descriptor: str) -> 'Pyramid': Returns: Pyramid: a Pyramid instance - """ + """ try: data = json.loads(get_data_str(descriptor)) @@ -467,7 +467,7 @@ def from_other(cls, other: 'Pyramid', name: str, storage: Dict) -> 'Pyramid': pyramid = cls() # Attributs communs - pyramid.__name = name + pyramid.__name = name pyramid.__storage = storage pyramid.__masks = other.__masks @@ -496,25 +496,25 @@ def from_other(cls, other: 'Pyramid', name: str, storage: Dict) -> 'Pyramid': return pyramid def __init__(self) -> None: - self.__storage = dict() - self.__levels = dict() + self.__storage = {} + self.__levels = {} self.__masks = None self.__content = { "loaded": False, - "cache": dict() + "cache": {} } def __str__(self) -> str: return f"{self.type.name} pyramid '{self.__name}' ({self.__storage['type'].name} storage)" @property - def serializable(self) -> Dict: + def serializable(self) -> Dict: """Get the dict version of the pyramid object, descriptor compliant Returns: Dict: descriptor structured object description - """ + """ serialization = { "tile_matrix_set": self.__tms.name, "format": self.__format @@ -549,7 +549,7 @@ def name(self) -> str: @property def tms(self) -> TileMatrixSet: return self.__tms - + @property def raster_specifications(self) -> Dict: """Get raster specifications for a RASTER pyramid @@ -561,34 +561,34 @@ def raster_specifications(self) -> Dict: "photometric": "rgb", "interpolation": "bicubic" } - + Returns: Dict: Raster specifications, None if VECTOR pyramid - """ + """ return self.__raster_specifications @property - def storage_type(self) -> StorageType: + def storage_type(self) -> StorageType: """Get the storage type Returns: StorageType: FILE, S3 or CEPH - """ + """ return self.__storage["type"] @property - def storage_root(self) -> str: + def storage_root(self) -> str: """Get the pyramid's storage root. If storage is S3, the used cluster is removed. Returns: str: Pyramid's storage root - """ + """ return self.__storage["root"].split("@", 1)[0] # Suppression de l'éventuel hôte de spécification du cluster S3 @property - def storage_depth(self) -> int: + def storage_depth(self) -> int: return self.__storage.get("depth", None) @@ -598,7 +598,7 @@ def storage_s3_cluster(self) -> str: Returns: str: the host if known, None if the default one have to be used or if storage is not S3 - """ + """ if self.__storage["type"] == StorageType.S3: try: return self.__storage["root"].split("@")[1] @@ -617,7 +617,7 @@ def storage_depth(self, d: int) -> None: Raises: Exception: the depth is not equal to the already known depth - """ + """ if "depth" in self.__storage and self.__storage["depth"] != d: raise Exception(f"Pyramid {pyramid.__descriptor} owns levels with different path depths") self.__storage["depth"] = d @@ -627,11 +627,11 @@ def own_masks(self) -> bool: return self.__masks @property - def format(self) -> str: + def format(self) -> str: return self.__format @property - def tile_extension(self) -> str: + def tile_extension(self) -> str: if self.__format in ["TIFF_RAW_UINT8", "TIFF_LZW_UINT8", "TIFF_ZIP_UINT8", "TIFF_PKB_UINT8", "TIFF_RAW_FLOAT32", "TIFF_LZW_FLOAT32", "TIFF_ZIP_FLOAT32", "TIFF_PKB_FLOAT32"]: return "tif" elif self.__format in ["TIFF_JPG_UINT8", "TIFF_JPG90_UINT8"]: @@ -644,12 +644,12 @@ def tile_extension(self) -> str: raise Exception(f"Unknown pyramid's format ({self.__format}), cannot return the tile extension") @property - def bottom_level(self) -> 'Level': + def bottom_level(self) -> 'Level': """Get the best resolution level in the pyramid Returns: Level: the bottom level - """ + """ return sorted(self.__levels.values(), key=lambda l: l.resolution)[0] @property @@ -658,7 +658,7 @@ def top_level(self) -> 'Level': Returns: Level: the top level - """ + """ return sorted(self.__levels.values(), key=lambda l: l.resolution)[-1] @property @@ -667,7 +667,7 @@ def type(self) -> PyramidType: Returns: PyramidType: RASTER or VECTOR - """ + """ if self.__format == "TIFF_PBF_MVT": return PyramidType.VECTOR else: @@ -677,7 +677,7 @@ def load_list(self) -> None: """Load list content and cache it If list is already loaded, nothing done - """ + """ if self.__content["loaded"]: return @@ -686,7 +686,7 @@ def load_list(self) -> None: self.__content["loaded"] = True - def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: + def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: """Get list content List is copied as temporary file, roots are read and informations about each slab is returned. If list is already loaded, we yield the cached content @@ -711,7 +711,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: Value example: - ( + ( (, '18', 5424, 7526), { 'link': False, @@ -721,7 +721,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: } ) - """ + """ if self.__content["loaded"]: for slab, infos in self.__content["cache"].items(): yield slab, infos @@ -733,7 +733,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: copy(self.__list, f"file://{list_file}") list_obj.close() - roots = dict() + roots = {} s3_cluster = self.storage_s3_cluster with open(list_file, "r") as listin: @@ -741,7 +741,7 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: # Lecture des racines for line in listin: line = line.rstrip() - + if line == "#": break @@ -787,7 +787,7 @@ def get_level(self, level_id: str) -> 'Level': Returns: The corresponding pyramid's level, None if not present """ - + return self.__levels.get(level_id, None) @@ -830,7 +830,7 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: """ sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution) - + levels = [] begin = False @@ -857,19 +857,19 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: break else: continue - + if top_id is None: # Pas de niveau du haut fourni, on a été jusqu'en haut et c'est normal end = True if not begin or not end: raise Exception(f"Provided levels ids are not consistent to extract levels from the pyramid {self.name}") - + return levels def write_descriptor(self) -> None: """Write the pyramid's descriptor to the final location (in the pyramid's storage root) - """ + """ content = json.dumps(self.serializable) put_data_str(content, self.__descriptor) @@ -895,7 +895,7 @@ def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: S3 stored pyramid from rok4.Pyramid import Pyramid - + try: pyramid = Pyramid.from_descriptor("s3://bucket_name/path/to/pyramid.json") slab_type, level, column, row = self.get_infos_from_slab_path("s3://bucket_name/path/to/pyramid/MASK_15_9164_5846") @@ -905,7 +905,7 @@ def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: Returns: Tuple[SlabType, str, int, int]: Slab's type (DATA or MASK), level identifier, slab's column and slab's row - """ + """ if self.__storage["type"] == StorageType.FILE: parts = path.split("/") @@ -951,17 +951,17 @@ def get_slab_path_from_infos(self, slab_type: SlabType, level: str, column: int, Returns: str: Absolute or relative slab's storage path - """ + """ if self.__storage["type"] == StorageType.FILE: slab_path = os.path.join(slab_type.value, level, b36_path_encode(column, row, self.__storage["depth"])) else: slab_path = f"{slab_type.value}_{level}_{column}_{row}" - + if full: return get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name, slab_path ) else: return slab_path - + def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """Get a pyramid's tile as binary string @@ -1012,7 +1012,7 @@ def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """ level_object = self.get_level(level) - + if level_object is None: raise Exception(f"No level {level} in the pyramid") @@ -1037,7 +1037,7 @@ def get_tile_data_binary(self, level: str, column: int, row: int) -> str: slab_path = self.get_slab_path_from_infos(SlabType.DATA, level, slab_column, slab_row) # Récupération des offset et tailles des tuiles dans la dalle - # Une dalle ROK4 a une en-tête fixe de 2048 octets, + # Une dalle ROK4 a une en-tête fixe de 2048 octets, # puis sont stockés les offsets (chacun sur 4 octets) # puis les tailles (chacune sur 4 octets) try: @@ -1120,12 +1120,12 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - + try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: raise FormatError("JPEG", "binary tile", e) - + data = numpy.asarray(img) elif self.__format == "TIFF_RAW_UINT8": @@ -1133,14 +1133,14 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr binary_tile, dtype = numpy.dtype('uint8') ) - data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) + data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) elif self.__format == "TIFF_PNG_UINT8": try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: raise FormatError("PNG", "binary tile", e) - + data = numpy.asarray(img) elif self.__format == "TIFF_ZIP_UINT8": @@ -1152,7 +1152,7 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr except Exception as e: raise FormatError("ZIP", "binary tile", e) - data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) + data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) elif self.__format == "TIFF_ZIP_FLOAT32": try: @@ -1163,14 +1163,14 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr except Exception as e: raise FormatError("ZIP", "binary tile", e) - data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) + data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) elif self.__format == "TIFF_RAW_FLOAT32": data = numpy.frombuffer( binary_tile, dtype = numpy.dtype('float32') ) - data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) + data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) else: raise NotImplementedError(f"Cannot get tile as raster data for format {self.__format}") @@ -1279,7 +1279,7 @@ def get_tile_indices(self, x: float, y: float, level: str = None, **kwargs) -> T level_object = self.bottom_level if level is not None: level_object = self.get_level(level) - + if level_object is None: raise Exception(f"Cannot found the level to calculate indices") diff --git a/src/rok4/Raster.py b/src/rok4/Raster.py index 2a06ee6..80c058e 100644 --- a/src/rok4/Raster.py +++ b/src/rok4/Raster.py @@ -79,7 +79,7 @@ def from_file(cls, path: str) -> 'Raster': path_pattern = re.compile('(/[^/]+?)[.][a-zA-Z0-9_-]+$') mask_path = path_pattern.sub('\\1.msk', path) - if exists(mask_path): + if exists(mask_path): work_mask_path = get_osgeo_path(mask_path) mask_driver = gdal.IdentifyDriver(work_mask_path).ShortName if 'GTiff' != mask_driver: @@ -128,7 +128,7 @@ def from_parameters(cls, path: str, bands: int, bbox: Tuple[float, float, float, """ self = cls() - + self.path = path self.bands = bands self.bbox = bbox diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py index dc19b80..3e22fd3 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/Storage.py @@ -49,7 +49,7 @@ class StorageType(Enum): S3 = "s3://" CEPH = "ceph://" -__S3_CLIENTS = dict() +__S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',str]], str, str]: """Get the S3 client @@ -65,7 +65,7 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',st Returns: Tuple[Dict[str, Union['boto3.client',str]], str, str]: the S3 informations (client, host, key, secret) and the simple bucket name - """ + """ global __S3_CLIENTS, __S3_DEFAULT_CLIENT if not __S3_CLIENTS: @@ -122,13 +122,13 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',st def disconnect_s3_clients() -> None: """Clean S3 clients - """ + """ global __S3_CLIENTS, __S3_DEFAULT_CLIENT - __S3_CLIENTS = dict() + __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None __CEPH_CLIENT = None -__CEPH_IOCTXS = dict() +__CEPH_IOCTXS = {} def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx': """Get the CEPH IO context @@ -143,7 +143,7 @@ def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx': Returns: rados.Ioctx: IO ceph context - """ + """ global __CEPH_CLIENT, __CEPH_IOCTXS if __CEPH_CLIENT is None: @@ -155,7 +155,7 @@ def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx': ) __CEPH_CLIENT.connect() - + except KeyError as e: raise MissingEnvironmentError(e) except Exception as e: @@ -166,15 +166,15 @@ def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx': __CEPH_IOCTXS[pool] = __CEPH_CLIENT.open_ioctx(pool) except Exception as e: raise StorageError("CEPH", e) - + return __CEPH_IOCTXS[pool] def disconnect_ceph_clients() -> None: """Clean CEPH clients - """ + """ global __CEPH_CLIENT, __CEPH_IOCTXS __CEPH_CLIENT = None - __CEPH_IOCTXS = dict() + __CEPH_IOCTXS = {} __OBJECT_SYMLINK_SIGNATURE = "SYMLINK#" @@ -182,8 +182,8 @@ def get_infos_from_path(path: str) -> Tuple[StorageType, str, str, str]: """Extract storage type, the unprefixed path, the container and the basename from path (Default: FILE storage) For a FILE storage, the tray is the directory and the basename is the file name. - - For an object storage (CEPH or S3), the tray is the bucket or the pool and the basename is the object name. + + For an object storage (CEPH or S3), the tray is the bucket or the pool and the basename is the object name. For a S3 bucket, format can be @ to use several clusters. Cluster name is the host (without protocol) Args: @@ -215,7 +215,7 @@ def get_path_from_infos(storage_type: StorageType, *args) -> str: Returns: str: Full path - """ + """ return f"{storage_type.value}{os.path.join(*args)}" @@ -257,7 +257,7 @@ def get_data_str(path: str) -> str: return get_data_binary(path).decode('utf-8') -def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: +def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: """Load data into a binary string Args: @@ -273,10 +273,10 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: str: Data binary content """ - storage_type, path, tray_name, base_name = get_infos_from_path(path) + storage_type, path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.S3: - + s3_client, bucket_name = __get_s3_client(tray_name) try: @@ -302,7 +302,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("S3", e) elif storage_type == StorageType.CEPH: - + ioctx = __get_ceph_ioctx(tray_name) try: @@ -327,7 +327,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: else: f.seek(range[0]) data = f.read(range[1]) - + f.close() except FileNotFoundError as e: @@ -358,7 +358,7 @@ def put_data_str(data: str, path: str) -> None: storage_type, path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.S3: - + s3_client, bucket_name = __get_s3_client(tray_name) try: @@ -371,7 +371,7 @@ def put_data_str(data: str, path: str) -> None: raise StorageError("S3", e) elif storage_type == StorageType.CEPH: - + ioctx = __get_ceph_ioctx(tray_name) try: @@ -553,7 +553,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: # Réalisation de la copie, selon les types de stockage if from_type == StorageType.FILE and to_type == StorageType.FILE : - + try: if to_tray != "": os.makedirs(to_tray, exist_ok=True) @@ -569,13 +569,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: raise StorageError(f"FILE", f"Cannot copy file {from_path} to {to_path} : {e}") elif from_type == StorageType.S3 and to_type == StorageType.FILE : - + s3_client, from_bucket = __get_s3_client(from_tray) try: if to_tray != "": os.makedirs(to_tray, exist_ok=True) - + s3_client["client"].download_file(from_bucket, from_base_name, to_path) if from_md5 is not None : @@ -589,7 +589,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: elif from_type == StorageType.FILE and to_type == StorageType.S3 : s3_client, to_bucket = __get_s3_client(to_tray) - + try: s3_client["client"].upload_file(from_path, to_bucket, to_base_name) @@ -611,7 +611,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: { 'Bucket': from_bucket, 'Key': from_base_name - }, + }, to_bucket, to_base_name ) else: @@ -626,7 +626,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError(f"S3", f"Cannot copy S3 object {from_path} to {to_path} : {e}") - + elif from_type == StorageType.CEPH and to_type == StorageType.FILE : @@ -668,7 +668,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: elif from_type == StorageType.FILE and to_type == StorageType.CEPH : ioctx = __get_ceph_ioctx(to_tray) - + if from_md5 is not None: checker = hashlib.md5() @@ -730,7 +730,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: except Exception as e: raise StorageError(f"CEPH", f"Cannot copy CEPH object {from_path} to {to_path} : {e}") - + elif from_type == StorageType.CEPH and to_type == StorageType.S3 : from_ioctx = __get_ceph_ioctx(from_tray) @@ -814,7 +814,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: raise StorageError("S3", e) elif target_type == StorageType.CEPH: - + ioctx = __get_ceph_ioctx(link_tray) try: @@ -850,7 +850,7 @@ def get_osgeo_path(path: str) -> str: Returns: str: GDAL/OGR Open compliant path - """ + """ storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) diff --git a/src/rok4/TileMatrixSet.py b/src/rok4/TileMatrixSet.py index 502e74b..80bd570 100644 --- a/src/rok4/TileMatrixSet.py +++ b/src/rok4/TileMatrixSet.py @@ -76,7 +76,7 @@ def y_to_row(self, y: float) -> int: """ return int((self.origin[1] - y) / (self.resolution * self.tile_size[1])) - def tile_to_bbox(self, tile_col: int, tile_row: int) -> Tuple[float, float, float, float]: + def tile_to_bbox(self, tile_col: int, tile_row: int) -> Tuple[float, float, float, float]: """Get tile terrain extent (xmin, ymin, xmax, ymax), in TMS coordinates system TMS spatial reference is Lat / Lon case is handled. @@ -103,7 +103,7 @@ def tile_to_bbox(self, tile_col: int, tile_row: int) -> Tuple[float, float, floa self.origin[1] - self.resolution * tile_row * self.tile_size[1] ) - def bbox_to_tiles(self, bbox: Tuple[float, float, float, float]) -> Tuple[int, int, int, int]: + def bbox_to_tiles(self, bbox: Tuple[float, float, float, float]) -> Tuple[int, int, int, int]: """Get extrems tile columns and rows corresponding to provided bounding box TMS spatial reference is Lat / Lon case is handled. @@ -141,8 +141,8 @@ def point_to_indices(self, x: float, y: float) -> Tuple[int, int, int, int]: Returns: Tuple[int, int, int, int]: tile's column, tile's row, pixel's (in the tile) column, pixel's row - """ - + """ + if self.__latlon: absolute_pixel_column = int((y - self.origin[0]) / self.resolution) absolute_pixel_row = int((self.origin[1] - x) / self.resolution) @@ -220,10 +220,9 @@ def get_level(self, level_id: str) -> 'TileMatrix': Returns: The corresponding tile matrix, None if not present """ - + return self.levels.get(level_id, None) @property - def sorted_levels(self) -> List[TileMatrix]: + def sorted_levels(self) -> List[TileMatrix]: return sorted(self.levels.values(), key=lambda l: l.resolution) - diff --git a/src/rok4/Utils.py b/src/rok4/Utils.py index 47e1bcd..0e4c480 100644 --- a/src/rok4/Utils.py +++ b/src/rok4/Utils.py @@ -22,7 +22,7 @@ class ColorFormat(Enum): FLOAT32 = 32 -__SR_BOOK = dict() +__SR_BOOK = {} def srs_to_spatialreference(srs: str) -> 'osgeo.osr.SpatialReference': """Convert coordinates system as string to OSR spatial reference @@ -55,7 +55,7 @@ def srs_to_spatialreference(srs: str) -> 'osgeo.osr.SpatialReference': return __SR_BOOK[srs.upper()] -def bbox_to_geometry(bbox: Tuple[float, float, float, float], densification: int = 0) -> 'osgeo.ogr.Geometry': +def bbox_to_geometry(bbox: Tuple[float, float, float, float], densification: int = 0) -> 'osgeo.ogr.Geometry': """Convert bbox coordinates to OGR geometry Args: @@ -64,10 +64,10 @@ def bbox_to_geometry(bbox: Tuple[float, float, float, float], densification: int Raises: RuntimeError: Provided SRS is invalid for OSR - + Returns: osgeo.ogr.Geometry: Corresponding OGR geometry, with spatial reference if provided - """ + """ ring = ogr.Geometry(ogr.wkbLinearRing) @@ -91,18 +91,18 @@ def bbox_to_geometry(bbox: Tuple[float, float, float, float], densification: int ring.AddPoint(bbox[2], bbox[3]) ring.AddPoint(bbox[0], bbox[3]) ring.AddPoint(bbox[0], bbox[1]) - + geom = ogr.Geometry(ogr.wkbPolygon) geom.AddGeometry(ring) geom.SetCoordinateDimension(2) - + return geom def reproject_bbox(bbox: Tuple[float, float, float, float], srs_src: str, srs_dst: str, densification: int = 5) -> Tuple[float, float, float, float]: - """Return bounding box in other coordinates system + """Return bounding box in other coordinates system Points are added to be sure output bounding box contains input bounding box @@ -176,7 +176,7 @@ def compute_bbox(source_dataset: gdal.Dataset) -> tuple: Args: source_dataset (gdal.Dataset): Dataset object created from the raster image - + Limitations: Image's axis must be parallel to SRS' axis @@ -244,7 +244,7 @@ def compute_format(dataset: gdal.Dataset, path: str = None) -> 'ColorFormat': if dataset.RasterCount < 1: raise Exception(f"Image {path} contains no color band.") - + band_1_datatype = dataset.GetRasterBand(1).DataType data_type_name = gdal.GetDataTypeName(band_1_datatype) data_type_size = gdal.GetDataTypeSize(band_1_datatype) diff --git a/src/rok4/__init__.py b/src/rok4/__init__.py index 3aa0d7b..6c8e6b9 100644 --- a/src/rok4/__init__.py +++ b/src/rok4/__init__.py @@ -1 +1 @@ -__version__ = "0.0.0" \ No newline at end of file +__version__ = "0.0.0" diff --git a/tests/test_Layer.py b/tests/test_Layer.py index b92f84b..26b0963 100644 --- a/tests/test_Layer.py +++ b/tests/test_Layer.py @@ -126,7 +126,7 @@ def test_parameters_raster_ok(mocked_put_data_str, mocked_utils_reproject_bbox, assert layer.geobbox == (0, 0, 100, 100) layer.write_descriptor("file:///home/ign/layers/") mocked_put_data_str.assert_called_once_with('{"title": "title", "abstract": "abstract", "keywords": ["RASTER", "layername"], "wmts": {"authorized": true}, "tms": {"authorized": true}, "bbox": {"south": 0, "west": 0, "north": 100, "east": 100}, "pyramids": [{"bottom_level": "10", "top_level": "10", "path": "file:///home/ign/pyramids/RGEALTI.json"}], "wms": {"authorized": true, "crs": ["CRS:84", "IGNF:WGS84G", "EPSG:3857", "EPSG:4258", "EPSG:4326"]}, "styles": ["normal"], "resampling": "nn"}', 'file:///home/ign/layers/layername.json') - + except Exception as exc: - assert False, f"Layer creation from parameters raises an exception: {exc}" \ No newline at end of file + assert False, f"Layer creation from parameters raises an exception: {exc}" diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index 065e0b8..c1936dd 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -58,7 +58,7 @@ def test_raster_missing_raster_specifications(mocked_tms_class, mocked_get_data_ @mock.patch('rok4.Pyramid.get_data_str', return_value='{"raster_specifications":{"channels":3,"nodata":"255,0,0","photometric":"rgb","interpolation":"bicubic"}, "format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"unknown"}], "tile_matrix_set": "PM"}') @mock.patch('rok4.Pyramid.TileMatrixSet') def test_wrong_level(mocked_tms_class, mocked_get_data_str): - + tms_instance = MagicMock() tms_instance.get_level.return_value = None tms_instance.name = "PM" @@ -66,7 +66,7 @@ def test_wrong_level(mocked_tms_class, mocked_get_data_str): with pytest.raises(Exception) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") - + mocked_tms_class.assert_called_once_with('PM') mocked_get_data_str.assert_called_once_with('file:///pyramid.json') tms_instance.get_level.assert_called_once_with('unknown') @@ -165,7 +165,7 @@ def test_tile_read_raster(mocked_tms_class): tms_instance = MagicMock() tms_instance.name = "UTM20W84MART_1M_MNT" tms_instance.srs = "IGNF:UTM20W84MART" - + tm_instance = MagicMock() tm_instance.id = "8" tm_instance.tile_size = (256,256) @@ -192,7 +192,7 @@ def test_tile_read_vector(mocked_tms_class): tms_instance = MagicMock() tms_instance.name = "PM" tms_instance.srs = "EPSG:3857" - + tm_instance = MagicMock() tm_instance.id = "4" tm_instance.tile_size = (256,256) @@ -221,7 +221,7 @@ def test_list_read(mocked_tms_class): tms_instance = MagicMock() tms_instance.name = "PM" tms_instance.srs = "EPSG:3857" - + tm_instance = MagicMock() tm_instance.id = "4" tm_instance.tile_size = (256,256) @@ -252,4 +252,4 @@ def test_b36_path_decode(): def test_b36_path_encode(): assert b36_path_encode(4032, 18217, 2) == "3E/42/01.tif" - assert b36_path_encode(14, 18217, 1) == "0E02/E1.tif" \ No newline at end of file + assert b36_path_encode(14, 18217, 1) == "0E02/E1.tif" diff --git a/tests/test_Raster.py b/tests/test_Raster.py index d7b2b24..6cc526e 100644 --- a/tests/test_Raster.py +++ b/tests/test_Raster.py @@ -128,7 +128,7 @@ def test_unsupported_mask_format(self, mocked_exists, mocked_gdal_open, mocked_i mocked_identifydriver.return_value = type('', (object,), {'ShortName': 'JPG'}) with pytest.raises(Exception): - Raster.from_file(self.source_image_path) + Raster.from_file(self.source_image_path) mocked_exists.assert_has_calls([ call(self.source_image_path), call(self.source_mask_path) ]) mocked_get_osgeo_path.assert_has_calls([ call(self.source_image_path), call(self.source_mask_path) ]) @@ -177,4 +177,3 @@ def test_image_and_mask(self): assert result.format == i_format assert result.dimensions == i_dimensions assert result.mask == i_mask - diff --git a/tests/test_Storage.py b/tests/test_Storage.py index 34a5c37..69d2ece 100644 --- a/tests/test_Storage.py +++ b/tests/test_Storage.py @@ -20,7 +20,7 @@ def test_hash_file_ok(mock_file): except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" -@mock.patch.dict(os.environ, {}, clear=True) +@mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") assert (StorageType.FILE, "/toto/titi/tutu.json", "/toto/titi", "tutu.json") == get_infos_from_path("file:///toto/titi/tutu.json") @@ -28,7 +28,7 @@ def test_get_infos_from_path(): assert (StorageType.FILE, "wrong://toto/titi", "wrong://toto", "titi") == get_infos_from_path("wrong://toto/titi") -@mock.patch.dict(os.environ, {}, clear=True) +@mock.patch.dict(os.environ, {}, clear=True) def test_get_path_from_infos(): assert get_path_from_infos(StorageType.S3, "toto", "toto/titi") == "s3://toto/toto/titi" assert get_path_from_infos(StorageType.FILE, "/toto/titi", "tutu.json") == "file:///toto/titi/tutu.json" @@ -66,7 +66,7 @@ def test_s3_invalid_endpoint(mocked_s3_client): def test_file_read_error(mock_file): with pytest.raises(FileNotFoundError): data = get_data_str("file:///path/to/file.ext") - + mock_file.assert_called_with("/path/to/file.ext", "rb") @@ -80,7 +80,7 @@ def test_file_read_ok(mock_file): except Exception as exc: assert False, f"FILE read raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_s3_read_nok(mocked_s3_client): disconnect_s3_clients() @@ -90,7 +90,7 @@ def test_s3_read_nok(mocked_s3_client): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_s3_read_ok(mocked_s3_client): @@ -110,7 +110,7 @@ def test_s3_read_ok(mocked_s3_client): assert False, f"S3 read raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') def test_ceph_read_ok(mocked_rados_client): @@ -131,7 +131,7 @@ def test_ceph_read_ok(mocked_rados_client): ############ put_data_str -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_s3_write_nok(mocked_s3_client): @@ -144,7 +144,7 @@ def test_s3_write_nok(mocked_s3_client): put_data_str("data", "s3://bucket/path/to/object") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_s3_write_ok(mocked_s3_client): @@ -158,7 +158,7 @@ def test_s3_write_ok(mocked_s3_client): assert False, f"S3 write raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') def test_ceph_write_ok(mocked_rados_client): @@ -190,7 +190,7 @@ def test_copy_file_file_ok(mock_hash_file, mock_copyfile, mock_makedirs): assert False, f"FILE -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') @mock.patch('os.makedirs', return_value=None) @mock.patch('rok4.Storage.hash_file', return_value="toto") @@ -210,7 +210,7 @@ def test_copy_s3_file_ok(mock_hash_file, mock_makedirs, mocked_s3_client): assert False, f"S3 -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') @mock.patch('os.makedirs', return_value=None) @mock.patch('rok4.Storage.hash_file', return_value="toto") @@ -226,7 +226,7 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_copy_file_s3_ok(mocked_s3_client): @@ -242,7 +242,7 @@ def test_copy_file_s3_ok(mocked_s3_client): assert False, f"FILE -> S3 copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_copy_s3_s3_ok(mocked_s3_client): @@ -258,7 +258,7 @@ def test_copy_s3_s3_ok(mocked_s3_client): assert False, f"S3 -> S3 copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_copy_s3_s3_intercluster_ok(mocked_s3_client): @@ -274,7 +274,7 @@ def test_copy_s3_s3_intercluster_ok(mocked_s3_client): assert False, f"S3 -> S3 inter cluster copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_copy_s3_s3_intercluster_nok(mocked_s3_client): @@ -287,7 +287,7 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client): with pytest.raises(StorageError): copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto") -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') @mock.patch('os.makedirs', return_value=None) @patch("builtins.open", new_callable=mock_open) @@ -306,7 +306,7 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_copy_file_ceph_ok(mock_file, mocked_rados_client): @@ -323,7 +323,7 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client): except Exception as exc: assert False, f"FILE -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_copy_ceph_ceph_ok(mock_file, mocked_rados_client): @@ -342,7 +342,7 @@ def test_copy_ceph_ceph_ok(mock_file, mocked_rados_client): assert False, f"CEPH -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c", "ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c", "ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') @mock.patch('rok4.Storage.boto3.client') @patch("builtins.open", new_callable=mock_open, read_data=b"data") @@ -379,7 +379,7 @@ def test_link_hard_nok(): with pytest.raises(StorageError): link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True) -@mock.patch.dict(os.environ, {}, clear=True) +@mock.patch.dict(os.environ, {}, clear=True) @mock.patch('os.symlink', return_value=None) def test_link_file_ok(mock_link): try: @@ -389,7 +389,7 @@ def test_link_file_ok(mock_link): assert False, f"FILE link raises an exception: {exc}" -@mock.patch.dict(os.environ, {}, clear=True) +@mock.patch.dict(os.environ, {}, clear=True) @mock.patch('os.link', return_value=None) def test_hlink_file_ok(mock_link): try: @@ -398,7 +398,7 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') def test_link_ceph_ok(mocked_rados_client): @@ -415,7 +415,7 @@ def test_link_ceph_ok(mocked_rados_client): assert False, f"CEPH link raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_link_s3_ok(mocked_s3_client): @@ -430,7 +430,7 @@ def test_link_s3_ok(mocked_s3_client): assert False, f"S3 link raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_link_s3_nok(mocked_s3_client): @@ -444,7 +444,7 @@ def test_link_s3_nok(mocked_s3_client): ############ get_size -@mock.patch.dict(os.environ, {}, clear=True) +@mock.patch.dict(os.environ, {}, clear=True) @mock.patch('os.stat') def test_size_file_ok(mock_stat): mock_stat.return_value.st_size = 12 @@ -454,7 +454,7 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') def test_size_ceph_ok(mocked_rados_client): @@ -472,7 +472,7 @@ def test_size_ceph_ok(mocked_rados_client): assert False, f"CEPH size raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_size_s3_ok(mocked_s3_client): @@ -490,7 +490,7 @@ def test_size_s3_ok(mocked_s3_client): ############ exists -@mock.patch.dict(os.environ, {}, clear=True) +@mock.patch.dict(os.environ, {}, clear=True) @mock.patch('os.path.exists', return_value=True) def test_exists_file_ok(mock_exists): try: @@ -504,7 +504,7 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') def test_exists_ceph_ok(mocked_rados_client): @@ -527,7 +527,7 @@ def test_exists_ceph_ok(mocked_rados_client): assert False, f"CEPH not exists raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_exists_s3_ok(mocked_s3_client): @@ -550,7 +550,7 @@ def test_exists_s3_ok(mocked_s3_client): ############ remove -@mock.patch.dict(os.environ, {}, clear=True) +@mock.patch.dict(os.environ, {}, clear=True) @mock.patch('os.remove') def test_remove_file_ok(mock_remove): mock_remove.return_value = None @@ -565,7 +565,7 @@ def test_remove_file_ok(mock_remove): except Exception as exc: assert False, f"FILE deletion (not found) raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) @mock.patch('rok4.Storage.rados.Rados') def test_remove_ceph_ok(mocked_rados_client): @@ -588,7 +588,7 @@ def test_remove_ceph_ok(mocked_rados_client): assert False, f"CEPH deletion (not found) raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) @mock.patch('rok4.Storage.boto3.client') def test_remove_s3_ok(mocked_s3_client): @@ -613,11 +613,11 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) +@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) def test_get_osgeo_path_s3_ok(): disconnect_s3_clients() - + try: path = get_osgeo_path("s3://bucket@b/to/object.ext") assert path == "/vsis3/bucket/to/object.ext" @@ -628,4 +628,4 @@ def test_get_osgeo_path_s3_ok(): @mock.patch.dict(os.environ, {}, clear=True) def test_get_osgeo_path_nok(): with pytest.raises(NotImplementedError): - get_osgeo_path("ceph://pool/data.ext") \ No newline at end of file + get_osgeo_path("ceph://pool/data.ext") diff --git a/tests/test_TileMatrixSet.py b/tests/test_TileMatrixSet.py index 4cf86f2..7f41688 100644 --- a/tests/test_TileMatrixSet.py +++ b/tests/test_TileMatrixSet.py @@ -127,4 +127,4 @@ def test_4326_conversions(mocked_get_data_str): assert tm.bbox_to_tiles((45,5,48,6)) == (269425,61166,270882,65535) assert tm.point_to_indices(45,5) == (269425, 65535, 199, 255) except Exception as exc: - assert False, f"'TileMatrixSet creation raises an exception: {exc}" \ No newline at end of file + assert False, f"'TileMatrixSet creation raises an exception: {exc}" diff --git a/tests/test_Utils.py b/tests/test_Utils.py index 3ba87da..bc7acc9 100644 --- a/tests/test_Utils.py +++ b/tests/test_Utils.py @@ -256,7 +256,7 @@ def test_compute_format_bit_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, m def test_compute_format_uint8_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info): try: mocked_datasource = MagicMock(gdal.Dataset) - + band_number = random.randint(1, 4) mocked_datasource.RasterCount = band_number band_name = None @@ -291,7 +291,7 @@ def test_compute_format_uint8_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, def test_compute_format_float32_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info): try: mocked_datasource = MagicMock(gdal.Dataset) - + band_number = random.randint(1, 4) mocked_datasource.RasterCount = band_number band_name = None @@ -345,7 +345,7 @@ def test_compute_format_unsupported_nok(mocked_GetDataTypeName, mocked_GetDataTy with pytest.raises(Exception): compute_format(mocked_datasource) - + mocked_GetDataTypeName.assert_called() mocked_GetDataTypeSize.assert_called() mocked_GetColorInterpretationName.assert_called() @@ -373,4 +373,3 @@ def test_compute_format_no_band_nok(mocked_GetDataTypeName, mocked_GetDataTypeSi mocked_Info.assert_not_called() except Exception as exc: assert False, f"Color format computation raises an exception: {exc}" - diff --git a/tests/test_Vector.py b/tests/test_Vector.py index c7eb2e4..d9f22fc 100644 --- a/tests/test_Vector.py +++ b/tests/test_Vector.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- from rok4.Vector import * from rok4.Exceptions import * from rok4.Storage import disconnect_ceph_clients