From a7c480a92f59c532115adc93997ed66cade817be Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Mon, 5 Jun 2023 17:00:36 +0200 Subject: [PATCH 1/2] Add black to dev dependencies --- pyproject.toml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a91d767..9f58bdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,13 +38,19 @@ dependencies = [ ] [project.optional-dependencies] +doc = [ + "pdoc3 >= 0.10.0" +] + +dev = [ + "black", + "pre-commit >3,<4" +] + test = [ "pytest >= 7.1.2", "coverage >= 7.0.5" ] -doc = [ - "pdoc3 >= 0.10.0" -] [project.urls] "Homepage" = "https://rok4.github.io/core-python" From 5c690f50ad4fe582251e7e3b8a9faca44089803f Mon Sep 17 00:00:00 2001 From: Julien Moura Date: Mon, 5 Jun 2023 17:00:58 +0200 Subject: [PATCH 2/2] Apply black to existing codebase --- src/rok4/Exceptions.py | 3 + src/rok4/Layer.py | 74 ++++---- src/rok4/Pyramid.py | 272 ++++++++++++++++---------- src/rok4/Raster.py | 40 ++-- src/rok4/Storage.py | 302 ++++++++++++++++------------- src/rok4/TileMatrixSet.py | 54 ++++-- src/rok4/Utils.py | 66 ++++--- src/rok4/Vector.py | 84 ++++---- tests/test_Layer.py | 79 ++++---- tests/test_Pyramid.py | 163 ++++++++++------ tests/test_Raster.py | 216 ++++++++++++--------- tests/test_Storage.py | 370 +++++++++++++++++++++++++----------- tests/test_TileMatrixSet.py | 132 ++++++++++--- tests/test_Utils.py | 158 +++++++-------- tests/test_Vector.py | 94 ++++++--- 15 files changed, 1302 insertions(+), 805 deletions(-) diff --git a/src/rok4/Exceptions.py b/src/rok4/Exceptions.py index 92c75e3..c3b2b59 100644 --- a/src/rok4/Exceptions.py +++ b/src/rok4/Exceptions.py @@ -8,6 +8,7 @@ def __init__(self, path, missing): self.missing = missing super().__init__(f"Missing attribute {missing} in '{path}'") + class MissingEnvironmentError(Exception): """ Exception raised when a needed environment variable is not defined @@ -17,6 +18,7 @@ def __init__(self, missing): self.missing = missing super().__init__(f"Missing environment variable {missing}") + class StorageError(Exception): """ Exception raised when an issue occured when using a storage @@ -27,6 +29,7 @@ def __init__(self, type, issue): self.issue = issue super().__init__(f"Issue occured using a {type} storage : {issue}") + class FormatError(Exception): """ Exception raised when a format is expected but not respected diff --git a/src/rok4/Layer.py b/src/rok4/Layer.py index 3f421f3..316a9ca 100644 --- a/src/rok4/Layer.py +++ b/src/rok4/Layer.py @@ -17,6 +17,7 @@ from rok4.Storage import * from rok4.Utils import * + class Layer: """A data layer, raster or vector @@ -34,7 +35,7 @@ class Layer: """ @classmethod - def from_descriptor(cls, descriptor: str) -> 'Layer': + def from_descriptor(cls, descriptor: str) -> "Layer": """Create a layer from its descriptor Args: @@ -58,7 +59,7 @@ def from_descriptor(cls, descriptor: str) -> 'Layer': layer = cls() storage_type, path, root, base_name = get_infos_from_path(descriptor) - layer.__name = base_name[:-5] # on supprime l'extension.json + layer.__name = base_name[:-5] # on supprime l'extension.json try: # Attributs communs @@ -71,7 +72,6 @@ def from_descriptor(cls, descriptor: str) -> 'Layer': for k in data["keywords"]: layer.__keywords.append(k) - if layer.type == PyramidType.RASTER: if "resampling" in data: layer.__resampling = data["resampling"] @@ -83,7 +83,12 @@ def from_descriptor(cls, descriptor: str) -> 'Layer': # Les bbox, native et géographique if "bbox" in data: - layer.__geobbox = (data["bbox"]["south"], data["bbox"]["west"], data["bbox"]["north"], data["bbox"]["east"]) + layer.__geobbox = ( + data["bbox"]["south"], + data["bbox"]["west"], + data["bbox"]["north"], + data["bbox"]["east"], + ) layer.__bbox = reproject_bbox(layer.__geobbox, "EPSG:4326", layer.__tms.srs, 5) # On force l'emprise de la couche, on recalcule donc les tuiles limites correspondantes pour chaque niveau for l in layer.__levels.values(): @@ -95,11 +100,10 @@ def from_descriptor(cls, descriptor: str) -> 'Layer': except KeyError as e: raise MissingAttributeError(descriptor, e) - return layer @classmethod - def from_parameters(cls, pyramids: List[Dict[str, str]], name: str, **kwargs) -> 'Layer': + def from_parameters(cls, pyramids: List[Dict[str, str]], name: str, **kwargs) -> "Layer": """Create a default layer from parameters Args: @@ -121,7 +125,9 @@ def from_parameters(cls, pyramids: List[Dict[str, str]], name: str, **kwargs) -> # Informations obligatoires if not re.match("^[A-Za-z0-9_-]*$", name): - raise Exception(f"Layer's name have to contain only letters, number, hyphen and underscore, to be URL and storage compliant ({name})") + raise Exception( + f"Layer's name have to contain only letters, number, hyphen and underscore, to be URL and storage compliant ({name})" + ) layer.__name = name layer.__load_pyramids(pyramids) @@ -156,7 +162,6 @@ def from_parameters(cls, pyramids: List[Dict[str, str]], name: str, **kwargs) -> return layer - def __init__(self) -> None: self.__format = None self.__tms = None @@ -181,7 +186,6 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None: ## Toutes les pyramides doivent avoir les même caractéristiques channels = None for p in pyramids: - pyramid = Pyramid.from_descriptor(p["path"]) bottom_level = p.get("bottom_level", None) top_level = p.get("top_level", None) @@ -193,18 +197,24 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None: top_level = pyramid.top_level.id if self.__format is not None and self.__format != pyramid.format: - raise Exception(f"Used pyramids have to own the same format : {self.__format} != {pyramid.format}") + raise Exception( + f"Used pyramids have to own the same format : {self.__format} != {pyramid.format}" + ) else: self.__format = pyramid.format if self.__tms is not None and self.__tms.id != pyramid.tms.id: - raise Exception(f"Used pyramids have to use the same TMS : {self.__tms.id} != {pyramid.tms.id}") + raise Exception( + f"Used pyramids have to use the same TMS : {self.__tms.id} != {pyramid.tms.id}" + ) else: self.__tms = pyramid.tms if self.type == PyramidType.RASTER: if channels is not None and channels != pyramid.raster_specifications["channels"]: - raise Exception(f"Used RASTER pyramids have to own the same number of channels : {channels} != {pyramid.raster_specifications['channels']}") + raise Exception( + f"Used RASTER pyramids have to own the same number of channels : {channels} != {pyramid.raster_specifications['channels']}" + ) else: channels = pyramid.raster_specifications["channels"] self.__resampling = pyramid.raster_specifications["interpolation"] @@ -215,11 +225,9 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None: raise Exception(f"Level {l.id} is present in two used pyramids") self.__levels[l.id] = l - self.__pyramids.append({ - "pyramid": pyramid, - "bottom_level": bottom_level, - "top_level": top_level - }) + self.__pyramids.append( + {"pyramid": pyramid, "bottom_level": bottom_level, "top_level": top_level} + ) self.__best_level = sorted(self.__levels.values(), key=lambda l: l.resolution)[0] @@ -237,38 +245,30 @@ def serializable(self) -> Dict: "title": self.__title, "abstract": self.__abstract, "keywords": self.__keywords, - "wmts": { - "authorized": True - }, - "tms": { - "authorized": True - }, + "wmts": {"authorized": True}, + "tms": {"authorized": True}, "bbox": { "south": self.__geobbox[0], "west": self.__geobbox[1], "north": self.__geobbox[2], - "east": self.__geobbox[3] + "east": self.__geobbox[3], }, - "pyramids": [] + "pyramids": [], } for p in self.__pyramids: - serialization["pyramids"].append({ - "bottom_level" : p["bottom_level"], - "top_level" : p["top_level"], - "path" : p["pyramid"].descriptor - }) + serialization["pyramids"].append( + { + "bottom_level": p["bottom_level"], + "top_level": p["top_level"], + "path": p["pyramid"].descriptor, + } + ) if self.type == PyramidType.RASTER: serialization["wms"] = { "authorized": True, - "crs": [ - "CRS:84", - "IGNF:WGS84G", - "EPSG:3857", - "EPSG:4258", - "EPSG:4326" - ] + "crs": ["CRS:84", "IGNF:WGS84G", "EPSG:3857", "EPSG:4258", "EPSG:4326"], } if self.__tms.srs.upper() not in serialization["wms"]["crs"]: diff --git a/src/rok4/Pyramid.py b/src/rok4/Pyramid.py index 75b433c..a3d7144 100644 --- a/src/rok4/Pyramid.py +++ b/src/rok4/Pyramid.py @@ -22,21 +22,25 @@ from rok4.Storage import * from rok4.Utils import * + class PyramidType(Enum): - """ Pyramid's data type """ + """Pyramid's data type""" + RASTER = "RASTER" VECTOR = "VECTOR" class SlabType(Enum): - """ Slab's type """ - DATA = "DATA" # Slab of data, raster or vector - MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data + """Slab's type""" + + DATA = "DATA" # Slab of data, raster or vector + MASK = "MASK" # Slab of mask, only for raster pyramid, image with one band : 0 is nodata, other values are data ROK4_IMAGE_HEADER_SIZE = 2048 """Slab's header size, 2048 bytes""" + def b36_number_encode(number: int) -> str: """Convert base-10 number to base-36 @@ -49,9 +53,9 @@ def b36_number_encode(number: int) -> str: str: base-36 number """ - alphabet='0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" - base36 = '' + base36 = "" if 0 <= number < len(alphabet): return alphabet[number] @@ -62,6 +66,7 @@ def b36_number_encode(number: int) -> str: return base36 + def b36_number_decode(number: str) -> int: """Convert base-36 number to base-10 @@ -73,6 +78,7 @@ def b36_number_decode(number: str) -> int: """ return int(number, 36) + def b36_path_decode(path: str) -> Tuple[int, int]: """Get slab's column and row from a base-36 based path @@ -83,8 +89,8 @@ def b36_path_decode(path: str) -> Tuple[int, int]: Tuple[int, int]: slab's column and row """ - path = path.replace('/', '') - path = re.sub(r'(\.TIFF?)', "", path.upper()) + path = path.replace("/", "") + path = re.sub(r"(\.TIFF?)", "", path.upper()) b36_column = "" b36_row = "" @@ -132,6 +138,7 @@ def b36_path_encode(column: int, row: int, slashs: int) -> str: return f"{b36_path}.tif" + class Level: """A pyramid's level, raster or vector @@ -143,7 +150,7 @@ class Level: """ @classmethod - def from_descriptor(cls, data: Dict, pyramid: 'Pyramid') -> 'Level': + def from_descriptor(cls, data: Dict, pyramid: "Pyramid") -> "Level": """Create a pyramid's level from the pyramid's descriptor levels element Args: @@ -165,27 +172,36 @@ def from_descriptor(cls, data: Dict, pyramid: 'Pyramid') -> 'Level': try: level.__id = data["id"] level.__tile_limits = data["tile_limits"] - level.__slab_size = (data["tiles_per_width"], data["tiles_per_height"],) + level.__slab_size = ( + data["tiles_per_width"], + data["tiles_per_height"], + ) # Informations sur le stockage : on les valide et stocke dans la pyramide if pyramid.storage_type.name != data["storage"]["type"]: - raise Exception(f"Pyramid {pyramid.descriptor} owns levels using different storage types ({ data['storage']['type'] }) than its one ({pyramid.storage_type.name})") + raise Exception( + f"Pyramid {pyramid.descriptor} owns levels using different storage types ({ data['storage']['type'] }) than its one ({pyramid.storage_type.name})" + ) if pyramid.storage_type == StorageType.FILE: pyramid.storage_depth = data["storage"]["path_depth"] if "mask_directory" in data["storage"] or "mask_prefix" in data["storage"]: if not pyramid.own_masks: - raise Exception(f"Pyramid {pyramid.__descriptor} does not define a mask format but level {level.__id} define mask storage informations") + raise Exception( + f"Pyramid {pyramid.__descriptor} does not define a mask format but level {level.__id} define mask storage informations" + ) else: if pyramid.own_masks: - raise Exception(f"Pyramid {pyramid.__descriptor} define a mask format but level {level.__id} does not define mask storage informations") + raise Exception( + f"Pyramid {pyramid.__descriptor} define a mask format but level {level.__id} does not define mask storage informations" + ) except KeyError as e: raise MissingAttributeError(pyramid.descriptor, f"levels[].{e}") # Attributs dans le cas d'un niveau vecteur - if level.__pyramid.type == PyramidType.VECTOR : + if level.__pyramid.type == PyramidType.VECTOR: try: level.__tables = data["tables"] @@ -195,7 +211,7 @@ def from_descriptor(cls, data: Dict, pyramid: 'Pyramid') -> 'Level': return level @classmethod - def from_other(cls, other: 'Level', pyramid: 'Pyramid') -> 'Level': + def from_other(cls, other: "Level", pyramid: "Pyramid") -> "Level": """Create a pyramid's level from another one Args: @@ -219,7 +235,7 @@ def from_other(cls, other: 'Level', pyramid: 'Pyramid') -> 'Level': level.__slab_size = other.__slab_size # Attributs dans le cas d'un niveau vecteur - if level.__pyramid.type == PyramidType.VECTOR : + if level.__pyramid.type == PyramidType.VECTOR: level.__tables = other.__tables return level @@ -227,7 +243,6 @@ def from_other(cls, other: 'Level', pyramid: 'Pyramid') -> 'Level': def __str__(self) -> str: return f"{self.__pyramid.type.name} pyramid's level '{self.__id}' ({self.__pyramid.storage_type.name} storage)" - @property def serializable(self) -> Dict: """Get the dict version of the pyramid object, pyramid's descriptor compliant @@ -239,7 +254,7 @@ def serializable(self) -> Dict: "id": self.__id, "tiles_per_width": self.__slab_size[0], "tiles_per_height": self.__slab_size[1], - "tile_limits": self.__tile_limits + "tile_limits": self.__tile_limits, } if self.__pyramid.type == PyramidType.VECTOR: @@ -249,16 +264,18 @@ def serializable(self) -> Dict: serialization["storage"] = { "type": "FILE", "image_directory": f"{self.__pyramid.name}/DATA/{self.__id}", - "path_depth": self.__pyramid.storage_depth + "path_depth": self.__pyramid.storage_depth, } if self.__pyramid.own_masks: - serialization["storage"]["mask_directory"] = f"{self.__pyramid.name}/MASK/{self.__id}" + serialization["storage"][ + "mask_directory" + ] = f"{self.__pyramid.name}/MASK/{self.__id}" elif self.__pyramid.storage_type == StorageType.CEPH: serialization["storage"] = { "type": "CEPH", "image_prefix": f"{self.__pyramid.name}/DATA_{self.__id}", - "pool_name": self.__pyramid.storage_root + "pool_name": self.__pyramid.storage_root, } if self.__pyramid.own_masks: serialization["storage"]["mask_prefix"] = f"{self.__pyramid.name}/MASK_{self.__id}" @@ -267,7 +284,7 @@ def serializable(self) -> Dict: serialization["storage"] = { "type": "S3", "image_prefix": f"{self.__pyramid.name}/DATA_{self.__id}", - "bucket_name": self.__pyramid.storage_root + "bucket_name": self.__pyramid.storage_root, } if self.__pyramid.own_masks: serialization["storage"]["mask_prefix"] = f"{self.__pyramid.name}/MASK_{self.__id}" @@ -285,8 +302,12 @@ def bbox(self) -> Tuple[float, float, float, float]: Returns: Tuple[float, float, float, float]: level terrain extent (xmin, ymin, xmax, ymax) """ - min_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox(self.__tile_limits["min_col"], self.__tile_limits["max_row"]) - max_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox(self.__tile_limits["max_col"], self.__tile_limits["min_row"]) + min_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox( + self.__tile_limits["min_col"], self.__tile_limits["max_row"] + ) + max_bbox = self.__pyramid.tms.get_level(self.__id).tile_to_bbox( + self.__tile_limits["max_col"], self.__tile_limits["min_row"] + ) return (min_bbox[0], min_bbox[1], max_bbox[2], max_bbox[3]) @@ -316,7 +337,12 @@ def is_in_limits(self, column: int, row: int) -> bool: Returns: bool: True if tiles' limits contain the provided tile's indices """ - return self.__tile_limits["min_row"] <= row and self.__tile_limits["max_row"] >= row and self.__tile_limits["min_col"] <= column and self.__tile_limits["max_col"] >= column + return ( + self.__tile_limits["min_row"] <= row + and self.__tile_limits["max_row"] >= row + and self.__tile_limits["min_col"] <= column + and self.__tile_limits["max_col"] >= column + ) def set_limits_from_bbox(self, bbox: Tuple[float, float, float, float]) -> None: """Set tile limits, based on provided bounding box @@ -326,12 +352,14 @@ def set_limits_from_bbox(self, bbox: Tuple[float, float, float, float]) -> None: """ - col_min, row_min, col_max, row_max = self.__pyramid.tms.get_level(self.__id).bbox_to_tiles(bbox) + col_min, row_min, col_max, row_max = self.__pyramid.tms.get_level(self.__id).bbox_to_tiles( + bbox + ) self.__tile_limits = { "min_row": row_min, "max_col": col_max, "max_row": row_max, - "min_col": col_min + "min_col": col_min, } @@ -366,7 +394,7 @@ class Pyramid: """ @classmethod - def from_descriptor(cls, descriptor: str) -> 'Pyramid': + def from_descriptor(cls, descriptor: str) -> "Pyramid": """Create a pyramid from its descriptor Args: @@ -399,13 +427,16 @@ def from_descriptor(cls, descriptor: str) -> 'Pyramid': except JSONDecodeError as e: raise FormatError("JSON", descriptor, e) - pyramid = cls() - pyramid.__storage["type"], path, pyramid.__storage["root"], base_name = get_infos_from_path(descriptor) - pyramid.__name = base_name[:-5] # on supprime l'extension.json + pyramid.__storage["type"], path, pyramid.__storage["root"], base_name = get_infos_from_path( + descriptor + ) + pyramid.__name = base_name[:-5] # on supprime l'extension.json pyramid.__descriptor = descriptor - pyramid.__list = get_path_from_infos(pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list") + pyramid.__list = get_path_from_infos( + pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list" + ) try: # Attributs communs @@ -413,7 +444,7 @@ def from_descriptor(cls, descriptor: str) -> 'Pyramid': pyramid.__format = data["format"] # Attributs d'une pyramide raster - if pyramid.type == PyramidType.RASTER : + if pyramid.type == PyramidType.RASTER: pyramid.__raster_specifications = data["raster_specifications"] if "mask_format" in data: @@ -427,7 +458,9 @@ def from_descriptor(cls, descriptor: str) -> 'Pyramid': pyramid.__levels[lev.id] = lev if pyramid.__tms.get_level(lev.id) is None: - raise Exception(f"Pyramid {descriptor} owns a level with the ID '{lev.id}', not defined in the TMS '{pyramid.tms.name}'") + raise Exception( + f"Pyramid {descriptor} owns a level with the ID '{lev.id}', not defined in the TMS '{pyramid.tms.name}'" + ) except KeyError as e: raise MissingAttributeError(descriptor, e) @@ -438,7 +471,7 @@ def from_descriptor(cls, descriptor: str) -> 'Pyramid': return pyramid @classmethod - def from_other(cls, other: 'Pyramid', name: str, storage: Dict) -> 'Pyramid': + def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid": """Create a pyramid from another one Args: @@ -471,13 +504,17 @@ def from_other(cls, other: 'Pyramid', name: str, storage: Dict) -> 'Pyramid': pyramid.__storage = storage pyramid.__masks = other.__masks - pyramid.__descriptor = get_path_from_infos(pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.json") - pyramid.__list = get_path_from_infos(pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list") + pyramid.__descriptor = get_path_from_infos( + pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.json" + ) + pyramid.__list = get_path_from_infos( + pyramid.__storage["type"], pyramid.__storage["root"], f"{pyramid.__name}.list" + ) pyramid.__tms = other.__tms pyramid.__format = other.__format # Attributs d'une pyramide raster - if pyramid.type == PyramidType.RASTER : + if pyramid.type == PyramidType.RASTER: if other.own_masks: pyramid.__masks = True else: @@ -489,7 +526,6 @@ def from_other(cls, other: 'Pyramid', name: str, storage: Dict) -> 'Pyramid': lev = Level.from_other(l, pyramid) pyramid.__levels[lev.id] = lev - except KeyError as e: raise MissingAttributeError(descriptor, e) @@ -500,10 +536,7 @@ def __init__(self) -> None: self.__levels = {} self.__masks = None - self.__content = { - "loaded": False, - "cache": {} - } + self.__content = {"loaded": False, "cache": {}} def __str__(self) -> str: return f"{self.type.name} pyramid '{self.__name}' ({self.__storage['type'].name} storage)" @@ -515,10 +548,7 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - serialization = { - "tile_matrix_set": self.__tms.name, - "format": self.__format - } + serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format} serialization["levels"] = [] sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True) @@ -585,13 +615,14 @@ def storage_root(self) -> str: Returns: str: Pyramid's storage root """ - return self.__storage["root"].split("@", 1)[0] # Suppression de l'éventuel hôte de spécification du cluster S3 + return self.__storage["root"].split("@", 1)[ + 0 + ] # Suppression de l'éventuel hôte de spécification du cluster S3 @property def storage_depth(self) -> int: return self.__storage.get("depth", None) - @property def storage_s3_cluster(self) -> str: """Get the pyramid's storage S3 cluster (host name) @@ -607,7 +638,6 @@ def storage_s3_cluster(self) -> str: else: return None - @storage_depth.setter def storage_depth(self, d: int) -> None: """Set the tree depth for a FILE storage @@ -619,7 +649,9 @@ def storage_depth(self, d: int) -> None: Exception: the depth is not equal to the already known depth """ if "depth" in self.__storage and self.__storage["depth"] != d: - raise Exception(f"Pyramid {pyramid.__descriptor} owns levels with different path depths") + raise Exception( + f"Pyramid {pyramid.__descriptor} owns levels with different path depths" + ) self.__storage["depth"] = d @property @@ -632,7 +664,16 @@ def format(self) -> str: @property def tile_extension(self) -> str: - if self.__format in ["TIFF_RAW_UINT8", "TIFF_LZW_UINT8", "TIFF_ZIP_UINT8", "TIFF_PKB_UINT8", "TIFF_RAW_FLOAT32", "TIFF_LZW_FLOAT32", "TIFF_ZIP_FLOAT32", "TIFF_PKB_FLOAT32"]: + if self.__format in [ + "TIFF_RAW_UINT8", + "TIFF_LZW_UINT8", + "TIFF_ZIP_UINT8", + "TIFF_PKB_UINT8", + "TIFF_RAW_FLOAT32", + "TIFF_LZW_FLOAT32", + "TIFF_ZIP_FLOAT32", + "TIFF_PKB_FLOAT32", + ]: return "tif" elif self.__format in ["TIFF_JPG_UINT8", "TIFF_JPG90_UINT8"]: return "jpg" @@ -641,10 +682,12 @@ def tile_extension(self) -> str: elif self.__format == "TIFF_PBF_MVT": return "pbf" else: - raise Exception(f"Unknown pyramid's format ({self.__format}), cannot return the tile extension") + raise Exception( + f"Unknown pyramid's format ({self.__format}), cannot return the tile extension" + ) @property - def bottom_level(self) -> 'Level': + def bottom_level(self) -> "Level": """Get the best resolution level in the pyramid Returns: @@ -653,7 +696,7 @@ def bottom_level(self) -> 'Level': return sorted(self.__levels.values(), key=lambda l: l.resolution)[0] @property - def top_level(self) -> 'Level': + def top_level(self) -> "Level": """Get the low resolution level in the pyramid Returns: @@ -686,7 +729,7 @@ def load_list(self) -> None: self.__content["loaded"] = True - def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: + def list_generator(self) -> Iterator[Tuple[Tuple[SlabType, str, int, int], Dict]]: """Get list content List is copied as temporary file, roots are read and informations about each slab is returned. If list is already loaded, we yield the cached content @@ -726,9 +769,8 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: for slab, infos in self.__content["cache"].items(): yield slab, infos else: - # Copie de la liste dans un fichier temporaire (cette liste peut être un objet) - list_obj = tempfile.NamedTemporaryFile(mode='r', delete=False) + list_obj = tempfile.NamedTemporaryFile(mode="r", delete=False) list_file = list_obj.name copy(self.__list, f"file://{list_file}") list_obj.close() @@ -737,7 +779,6 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: s3_cluster = self.storage_s3_cluster with open(list_file, "r") as listin: - # Lecture des racines for line in listin: line = line.rstrip() @@ -769,16 +810,16 @@ def list_generator(self) -> Iterator[Tuple[Tuple[SlabType,str,int,int], Dict]]: slab_type, level, column, row = self.get_infos_from_slab_path(slab_path) infos = { "root": roots[root_id], - "link": root_id != '0', + "link": root_id != "0", "slab": slab_path, - "md5": slab_md5 + "md5": slab_md5, } - yield ((slab_type,level,column,row), infos) + yield ((slab_type, level, column, row), infos) remove(f"file://{list_file}") - def get_level(self, level_id: str) -> 'Level': + def get_level(self, level_id: str) -> "Level": """Get one level according to its identifier Args: @@ -790,7 +831,6 @@ def get_level(self, level_id: str) -> 'Level': return self.__levels.get(level_id, None) - def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: """Get sorted levels in the provided range from bottom to top @@ -839,7 +879,9 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: begin = True else: if self.get_level(bottom_id) is None: - raise Exception(f"Pyramid {self.name} does not contain the provided bottom level {bottom_id}") + raise Exception( + f"Pyramid {self.name} does not contain the provided bottom level {bottom_id}" + ) if top_id is not None and self.get_level(top_id) is None: raise Exception(f"Pyramid {self.name} does not contain the provided top level {top_id}") @@ -863,13 +905,14 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]: end = True if not begin or not end: - raise Exception(f"Provided levels ids are not consistent to extract levels from the pyramid {self.name}") + raise Exception( + f"Provided levels ids are not consistent to extract levels from the pyramid {self.name}" + ) return levels def write_descriptor(self) -> None: - """Write the pyramid's descriptor to the final location (in the pyramid's storage root) - """ + """Write the pyramid's descriptor to the final location (in the pyramid's storage root)""" content = json.dumps(self.serializable) put_data_str(content, self.__descriptor) @@ -911,9 +954,9 @@ def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: # Le partie du chemin qui contient la colonne et ligne de la dalle est à la fin, en fonction de la profondeur choisie # depth = 2 -> on doit utiliser les 3 dernières parties pour la conversion - column, row = b36_path_decode('/'.join(parts[-(self.__storage["depth"]+1):])) - level = parts[-(self.__storage["depth"]+2)] - raw_slab_type = parts[-(self.__storage["depth"]+3)] + column, row = b36_path_decode("/".join(parts[-(self.__storage["depth"] + 1) :])) + level = parts[-(self.__storage["depth"] + 2)] + raw_slab_type = parts[-(self.__storage["depth"] + 3)] # Pour être retro compatible avec l'ancien nommage if raw_slab_type == "IMAGE": @@ -923,7 +966,7 @@ def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: return slab_type, level, column, row else: - parts = re.split(r'[/_]', path) + parts = re.split(r"[/_]", path) column = parts[-2] row = parts[-1] level = parts[-3] @@ -939,7 +982,9 @@ def get_infos_from_slab_path(self, path: str) -> Tuple[SlabType, str, int, int]: return slab_type, level, int(column), int(row) - def get_slab_path_from_infos(self, slab_type: SlabType, level: str, column: int, row: int, full: bool = True) -> str: + def get_slab_path_from_infos( + self, slab_type: SlabType, level: str, column: int, row: int, full: bool = True + ) -> str: """Get slab's storage path from the indices Args: @@ -953,16 +998,19 @@ def get_slab_path_from_infos(self, slab_type: SlabType, level: str, column: int, str: Absolute or relative slab's storage path """ if self.__storage["type"] == StorageType.FILE: - slab_path = os.path.join(slab_type.value, level, b36_path_encode(column, row, self.__storage["depth"])) + slab_path = os.path.join( + slab_type.value, level, b36_path_encode(column, row, self.__storage["depth"]) + ) else: slab_path = f"{slab_type.value}_{level}_{column}_{row}" if full: - return get_path_from_infos(self.__storage["type"], self.__storage["root"], self.__name, slab_path ) + return get_path_from_infos( + self.__storage["type"], self.__storage["root"], self.__name, slab_path + ) else: return slab_path - def get_tile_data_binary(self, level: str, column: int, row: int) -> str: """Get a pyramid's tile as binary string @@ -1041,21 +1089,27 @@ def get_tile_data_binary(self, level: str, column: int, row: int) -> str: # puis sont stockés les offsets (chacun sur 4 octets) # puis les tailles (chacune sur 4 octets) try: - binary_index = get_data_binary(slab_path, (ROK4_IMAGE_HEADER_SIZE, 2 * 4 * level_object.slab_width * level_object.slab_height)) + binary_index = get_data_binary( + slab_path, + ( + ROK4_IMAGE_HEADER_SIZE, + 2 * 4 * level_object.slab_width * level_object.slab_height, + ), + ) except FileNotFoundError as e: # L'absence de la dalle est gérée comme simplement une absence de données return None offsets = numpy.frombuffer( binary_index, - dtype = numpy.dtype('uint32'), - count = level_object.slab_width * level_object.slab_height + dtype=numpy.dtype("uint32"), + count=level_object.slab_width * level_object.slab_height, ) sizes = numpy.frombuffer( binary_index, - dtype = numpy.dtype('uint32'), - offset = 4 * level_object.slab_width * level_object.slab_height, - count = level_object.slab_width * level_object.slab_height + dtype=numpy.dtype("uint32"), + offset=4 * level_object.slab_width * level_object.slab_height, + count=level_object.slab_width * level_object.slab_height, ) if sizes[tile_index] == 0: @@ -1118,9 +1172,7 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr level_object = self.get_level(level) - if self.__format == "TIFF_JPG_UINT8" or self.__format == "TIFF_JPG90_UINT8": - try: img = Image.open(io.BytesIO(binary_tile)) except Exception as e: @@ -1129,11 +1181,12 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr data = numpy.asarray(img) elif self.__format == "TIFF_RAW_UINT8": - data = numpy.frombuffer( - binary_tile, - dtype = numpy.dtype('uint8') + data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("uint8")) + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], ) - data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) elif self.__format == "TIFF_PNG_UINT8": try: @@ -1145,32 +1198,35 @@ def get_tile_data_raster(self, level: str, column: int, row: int) -> numpy.ndarr elif self.__format == "TIFF_ZIP_UINT8": try: - data = numpy.frombuffer( - zlib.decompress( binary_tile ), - dtype = numpy.dtype('uint8') - ) + data = numpy.frombuffer(zlib.decompress(binary_tile), dtype=numpy.dtype("uint8")) except Exception as e: raise FormatError("ZIP", "binary tile", e) - data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], + ) elif self.__format == "TIFF_ZIP_FLOAT32": try: - data = numpy.frombuffer( - zlib.decompress( binary_tile ), - dtype = numpy.dtype('float32') - ) + data = numpy.frombuffer(zlib.decompress(binary_tile), dtype=numpy.dtype("float32")) except Exception as e: raise FormatError("ZIP", "binary tile", e) - data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], + ) elif self.__format == "TIFF_RAW_FLOAT32": - data = numpy.frombuffer( - binary_tile, - dtype = numpy.dtype('float32') + data = numpy.frombuffer(binary_tile, dtype=numpy.dtype("float32")) + data.shape = ( + level_object.tile_matrix.tile_size[0], + level_object.tile_matrix.tile_size[1], + self.__raster_specifications["channels"], ) - data.shape = (level_object.tile_matrix.tile_size[0], level_object.tile_matrix.tile_size[1], self.__raster_specifications["channels"]) else: raise NotImplementedError(f"Cannot get tile as raster data for format {self.__format}") @@ -1238,7 +1294,9 @@ def get_tile_data_vector(self, level: str, column: int, row: int) -> Dict: return data - def get_tile_indices(self, x: float, y: float, level: str = None, **kwargs) -> Tuple[str, int, int, int, int]: + def get_tile_indices( + self, x: float, y: float, level: str = None, **kwargs + ) -> Tuple[str, int, int, int, int]: """Get pyramid's tile and pixel indices from point's coordinates Used coordinates system have to be the pyramid one. If EPSG:4326, x is latitude and y longitude. @@ -1283,8 +1341,12 @@ def get_tile_indices(self, x: float, y: float, level: str = None, **kwargs) -> T if level_object is None: raise Exception(f"Cannot found the level to calculate indices") - if "srs" in kwargs and kwargs["srs"] is not None and kwargs["srs"].upper() != self.__tms.srs.upper(): + if ( + "srs" in kwargs + and kwargs["srs"] is not None + and kwargs["srs"].upper() != self.__tms.srs.upper() + ): sr = srs_to_spatialreference(kwargs["srs"]) - x, y = reproject_point((x, y), sr, self.__tms.sr ) + x, y = reproject_point((x, y), sr, self.__tms.sr) return (level_object.id,) + level_object.tile_matrix.point_to_indices(x, y) diff --git a/src/rok4/Raster.py b/src/rok4/Raster.py index 1d7d6b5..640f4bf 100644 --- a/src/rok4/Raster.py +++ b/src/rok4/Raster.py @@ -15,12 +15,13 @@ from osgeo import ogr, gdal from rok4.Storage import exists, get_osgeo_path, put_data_str -from rok4.Utils import ColorFormat, compute_bbox,compute_format +from rok4.Utils import ColorFormat, compute_bbox, compute_format # Enable GDAL/OGR exceptions ogr.UseExceptions() gdal.UseExceptions() + class Raster: """A structure describing raster data @@ -91,9 +92,11 @@ def from_file(cls, path: str) -> "Raster": if exists(mask_path): work_mask_path = get_osgeo_path(mask_path) mask_driver = gdal.IdentifyDriver(work_mask_path).ShortName - if 'GTiff' != mask_driver: - message = (f"Mask file '{mask_path}' is not a TIFF image." - + f" (GDAL driver : '{mask_driver}'") + if "GTiff" != mask_driver: + message = ( + f"Mask file '{mask_path}' is not a TIFF image." + + f" (GDAL driver : '{mask_driver}'" + ) raise Exception(message) self.mask = mask_path else: @@ -108,8 +111,14 @@ def from_file(cls, path: str) -> "Raster": @classmethod def from_parameters( - cls, path: str, bands: int, bbox: Tuple[float, float, float, float], - dimensions: Tuple[int, int], format: ColorFormat, mask: str = None) -> "Raster": + cls, + path: str, + bands: int, + bbox: Tuple[float, float, float, float], + dimensions: Tuple[int, int], + format: ColorFormat, + mask: str = None, + ) -> "Raster": """Creates a Raster object from parameters Args: @@ -297,13 +306,13 @@ def from_descriptor(cls, path: str) -> "RasterSet": parameters = copy.deepcopy(raster_dict) parameters["bbox"] = tuple(raster_dict["bbox"]) parameters["dimensions"] = tuple(raster_dict["dimensions"]) - parameters["format"] = ColorFormat[ raster_dict["format"] ] + parameters["format"] = ColorFormat[raster_dict["format"]] self.raster_list.append(Raster.from_parameters(**parameters)) self.bbox = tuple(serialization["bbox"]) self.colors = [] for color_dict in serialization["colors"]: color_item = copy.deepcopy(color_dict) - color_item["format"] = ColorFormat[ color_dict["format"] ] + color_item["format"] = ColorFormat[color_dict["format"]] self.colors.append(color_item) return self @@ -314,17 +323,9 @@ def serializable(self) -> Dict: Returns: Dict: descriptor structured object description """ - serialization = { - "bbox": list(self.bbox), - "srs": self.srs, - "colors": [], - "raster_list" : [] - } + serialization = {"bbox": list(self.bbox), "srs": self.srs, "colors": [], "raster_list": []} for color in self.colors: - color_serial = { - "bands": color["bands"], - "format": color["format"].name - } + color_serial = {"bands": color["bands"], "format": color["format"].name} serialization["colors"].append(color_serial) for raster in self.raster_list: raster_dict = { @@ -332,7 +333,7 @@ def serializable(self) -> Dict: "dimensions": list(raster.dimensions), "bbox": list(raster.bbox), "bands": raster.bands, - "format": raster.format.name + "format": raster.format.name, } if raster.mask is not None: raster_dict["mask"] = raster.mask @@ -352,4 +353,3 @@ def write_descriptor(self, path: str = None) -> None: print(content) else: put_data_str(content, path) - diff --git a/src/rok4/Storage.py b/src/rok4/Storage.py index 3e22fd3..11d59f1 100644 --- a/src/rok4/Storage.py +++ b/src/rok4/Storage.py @@ -44,14 +44,18 @@ from rok4.Exceptions import * + class StorageType(Enum): FILE = "file://" S3 = "s3://" CEPH = "ceph://" + __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None -def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',str]], str, str]: + + +def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union["boto3.client", str]], str, str]: """Get the S3 client Create it if not already done @@ -76,10 +80,12 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',st urls = os.environ["ROK4_S3_URL"].split(",") if len(keys) != len(secret_keys) or len(keys) != len(urls): - raise StorageError("S3", "S3 informations in environment variables are inconsistent : same number of element in each list is required") + raise StorageError( + "S3", + "S3 informations in environment variables are inconsistent : same number of element in each list is required", + ) for i in range(len(keys)): - h = re.sub("https?://", "", urls[i]) if h in __S3_CLIENTS: @@ -87,15 +93,15 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',st __S3_CLIENTS[h] = { "client": boto3.client( - 's3', - aws_access_key_id = keys[i], - aws_secret_access_key = secret_keys[i], - endpoint_url = urls[i] + "s3", + aws_access_key_id=keys[i], + aws_secret_access_key=secret_keys[i], + endpoint_url=urls[i], ), "key": keys[i], "secret_key": secret_keys[i], "url": urls[i], - "host": h + "host": h, } if i == 0: @@ -107,7 +113,6 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',st except Exception as e: raise StorageError("S3", e) - try: host = bucket_name.split("@")[1] except IndexError: @@ -120,16 +125,19 @@ def __get_s3_client(bucket_name: str) -> Tuple[Dict[str, Union['boto3.client',st return __S3_CLIENTS[host], bucket_name + def disconnect_s3_clients() -> None: - """Clean S3 clients - """ + """Clean S3 clients""" global __S3_CLIENTS, __S3_DEFAULT_CLIENT __S3_CLIENTS = {} __S3_DEFAULT_CLIENT = None + __CEPH_CLIENT = None __CEPH_IOCTXS = {} -def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx': + + +def __get_ceph_ioctx(pool: str) -> "rados.Ioctx": """Get the CEPH IO context Create it (client and context) if not already done @@ -149,9 +157,9 @@ def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx': if __CEPH_CLIENT is None: try: __CEPH_CLIENT = rados.Rados( - conffile = os.environ["ROK4_CEPH_CONFFILE"], - clustername = os.environ["ROK4_CEPH_CLUSTERNAME"], - name = os.environ["ROK4_CEPH_USERNAME"] + conffile=os.environ["ROK4_CEPH_CONFFILE"], + clustername=os.environ["ROK4_CEPH_CLUSTERNAME"], + name=os.environ["ROK4_CEPH_USERNAME"], ) __CEPH_CLIENT.connect() @@ -169,15 +177,17 @@ def __get_ceph_ioctx(pool: str) -> 'rados.Ioctx': return __CEPH_IOCTXS[pool] + def disconnect_ceph_clients() -> None: - """Clean CEPH clients - """ + """Clean CEPH clients""" global __CEPH_CLIENT, __CEPH_IOCTXS __CEPH_CLIENT = None __CEPH_IOCTXS = {} + __OBJECT_SYMLINK_SIGNATURE = "SYMLINK#" + def get_infos_from_path(path: str) -> Tuple[StorageType, str, str, str]: """Extract storage type, the unprefixed path, the container and the basename from path (Default: FILE storage) @@ -231,14 +241,15 @@ def hash_file(path: str) -> str: checker = hashlib.md5() - with open(path,'rb') as file: + with open(path, "rb") as file: chunk = 0 - while chunk != b'': + while chunk != b"": chunk = file.read(65536) checker.update(chunk) return checker.hexdigest() + def get_data_str(path: str) -> str: """Load full data into a string @@ -254,7 +265,7 @@ def get_data_str(path: str) -> str: str: Data content """ - return get_data_binary(path).decode('utf-8') + return get_data_binary(path).decode("utf-8") def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: @@ -273,27 +284,34 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: str: Data binary content """ - storage_type, path, tray_name, base_name = get_infos_from_path(path) + storage_type, path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.S3: - s3_client, bucket_name = __get_s3_client(tray_name) try: if range is None: - data = s3_client["client"].get_object( - Bucket=bucket_name, - Key=base_name, - )['Body'].read() + data = ( + s3_client["client"] + .get_object( + Bucket=bucket_name, + Key=base_name, + )["Body"] + .read() + ) else: - data = s3_client["client"].get_object( - Bucket=bucket_name, - Key=base_name, - Range=f"bytes={range[0]}-{range[0] + range[1] - 1}" - )['Body'].read() + data = ( + s3_client["client"] + .get_object( + Bucket=bucket_name, + Key=base_name, + Range=f"bytes={range[0]}-{range[0] + range[1] - 1}", + )["Body"] + .read() + ) except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == "404": + if e.response["Error"]["Code"] == "404": raise FileNotFoundError(f"{storage_type.value}{path}") else: raise StorageError("S3", e) @@ -302,7 +320,6 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("S3", e) elif storage_type == StorageType.CEPH: - ioctx = __get_ceph_ioctx(tray_name) try: @@ -319,9 +336,8 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: raise StorageError("CEPH", e) elif storage_type == StorageType.FILE: - try: - f = open(path, 'rb') + f = open(path, "rb") if range is None: data = f.read() else: @@ -341,6 +357,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str: return data + def put_data_str(data: str, path: str) -> None: """Store string data into a file or an object @@ -355,34 +372,29 @@ def put_data_str(data: str, path: str) -> None: StorageError: Storage write issue """ - storage_type, path, tray_name, base_name = get_infos_from_path(path) + storage_type, path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.S3: - s3_client, bucket_name = __get_s3_client(tray_name) try: s3_client["client"].put_object( - Body = data.encode('utf-8'), - Bucket = bucket_name, - Key = base_name + Body=data.encode("utf-8"), Bucket=bucket_name, Key=base_name ) except Exception as e: raise StorageError("S3", e) elif storage_type == StorageType.CEPH: - ioctx = __get_ceph_ioctx(tray_name) try: - ioctx.write_full(base_name, data.encode('utf-8')) + ioctx.write_full(base_name, data.encode("utf-8")) except Exception as e: raise StorageError("CEPH", e) elif storage_type == StorageType.FILE: - try: - f = open(path, 'w') + f = open(path, "w") f.write(data) f.close() except Exception as e: @@ -406,20 +418,20 @@ def get_size(path: str) -> int: int: file/object size, in bytes """ - storage_type, path, tray_name, base_name = get_infos_from_path(path) + storage_type, path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.S3: - s3_client, bucket_name = __get_s3_client(tray_name) try: - size = s3_client["client"].head_object(Bucket=bucket_name, Key=base_name)["ContentLength"] + size = s3_client["client"].head_object(Bucket=bucket_name, Key=base_name)[ + "ContentLength" + ] return int(size) except Exception as e: raise StorageError("S3", e) elif storage_type == StorageType.CEPH: - ioctx = __get_ceph_ioctx(tray_name) try: @@ -429,7 +441,6 @@ def get_size(path: str) -> int: raise StorageError("CEPH", e) elif storage_type == StorageType.FILE: - try: file_stats = os.stat(path) return file_stats.st_size @@ -454,23 +465,21 @@ def exists(path: str) -> bool: bool: file/object existing status """ - storage_type, path, tray_name, base_name = get_infos_from_path(path) + storage_type, path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.S3: - s3_client, bucket_name = __get_s3_client(tray_name) try: s3_client["client"].head_object(Bucket=bucket_name, Key=base_name) return True except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == "404": + if e.response["Error"]["Code"] == "404": return False else: raise StorageError("S3", e) elif storage_type == StorageType.CEPH: - ioctx = __get_ceph_ioctx(tray_name) try: @@ -482,12 +491,12 @@ def exists(path: str) -> bool: raise StorageError("CEPH", e) elif storage_type == StorageType.FILE: - return os.path.exists(path) else: raise StorageError("UNKNOWN", "Unhandled storage type to test if exists") + def remove(path: str) -> None: """Remove the file/object @@ -498,22 +507,17 @@ def remove(path: str) -> None: MissingEnvironmentError: Missing object storage informations StorageError: Storage removal issue """ - storage_type, path, tray_name, base_name = get_infos_from_path(path) + storage_type, path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.S3: - s3_client, bucket_name = __get_s3_client(tray_name) try: - s3_client["client"].delete_object( - Bucket=bucket_name, - Key=base_name - ) + s3_client["client"].delete_object(Bucket=bucket_name, Key=base_name) except Exception as e: raise StorageError("S3", e) elif storage_type == StorageType.CEPH: - ioctx = __get_ceph_ioctx(tray_name) try: @@ -524,7 +528,6 @@ def remove(path: str) -> None: raise StorageError("CEPH", e) elif storage_type == StorageType.FILE: - try: os.remove(path) except FileNotFoundError as e: @@ -535,6 +538,7 @@ def remove(path: str) -> None: else: raise StorageError("UNKNOWN", "Unhandled storage type to remove things") + def copy(from_path: str, to_path: str, from_md5: str = None) -> None: """Copy a file or object to a file or object place. If MD5 sum is provided, it is compared to sum after the copy. @@ -548,28 +552,29 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: MissingEnvironmentError: Missing object storage informations """ - from_type, from_path, from_tray, from_base_name = get_infos_from_path(from_path) - to_type, to_path, to_tray, to_base_name = get_infos_from_path(to_path) + from_type, from_path, from_tray, from_base_name = get_infos_from_path(from_path) + to_type, to_path, to_tray, to_base_name = get_infos_from_path(to_path) # Réalisation de la copie, selon les types de stockage - if from_type == StorageType.FILE and to_type == StorageType.FILE : - + if from_type == StorageType.FILE and to_type == StorageType.FILE: try: if to_tray != "": os.makedirs(to_tray, exist_ok=True) copyfile(from_path, to_path) - if from_md5 is not None : + if from_md5 is not None: to_md5 = hash_file(to_path) if to_md5 != from_md5: - raise StorageError(f"FILE", f"Invalid MD5 sum control for copy file {from_path} to {to_path} : {from_md5} != {to_md5}") + raise StorageError( + f"FILE", + f"Invalid MD5 sum control for copy file {from_path} to {to_path} : {from_md5} != {to_md5}", + ) except Exception as e: raise StorageError(f"FILE", f"Cannot copy file {from_path} to {to_path} : {e}") - elif from_type == StorageType.S3 and to_type == StorageType.FILE : - + elif from_type == StorageType.S3 and to_type == StorageType.FILE: s3_client, from_bucket = __get_s3_client(from_tray) try: @@ -578,68 +583,80 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: s3_client["client"].download_file(from_bucket, from_base_name, to_path) - if from_md5 is not None : + if from_md5 is not None: to_md5 = hash_file(to_path) if to_md5 != from_md5: - raise StorageError("S3 and FILE", f"Invalid MD5 sum control for copy S3 object {from_path} to file {to_path} : {from_md5} != {to_md5}") + raise StorageError( + "S3 and FILE", + f"Invalid MD5 sum control for copy S3 object {from_path} to file {to_path} : {from_md5} != {to_md5}", + ) except Exception as e: - raise StorageError(f"S3 and FILE", f"Cannot copy S3 object {from_path} to file {to_path} : {e}") - - elif from_type == StorageType.FILE and to_type == StorageType.S3 : + raise StorageError( + f"S3 and FILE", f"Cannot copy S3 object {from_path} to file {to_path} : {e}" + ) + elif from_type == StorageType.FILE and to_type == StorageType.S3: s3_client, to_bucket = __get_s3_client(to_tray) try: s3_client["client"].upload_file(from_path, to_bucket, to_base_name) - if from_md5 is not None : - to_md5 = s3_client["client"].head_object(Bucket=to_bucket, Key=to_base_name)["ETag"].strip('"') + if from_md5 is not None: + to_md5 = ( + s3_client["client"] + .head_object(Bucket=to_bucket, Key=to_base_name)["ETag"] + .strip('"') + ) if to_md5 != from_md5: - raise StorageError(f"FILE and S3", f"Invalid MD5 sum control for copy file {from_path} to S3 object {to_path} : {from_md5} != {to_md5}") + raise StorageError( + f"FILE and S3", + f"Invalid MD5 sum control for copy file {from_path} to S3 object {to_path} : {from_md5} != {to_md5}", + ) except Exception as e: - raise StorageError(f"FILE and S3", f"Cannot copy file {from_path} to S3 object {to_path} : {e}") - - elif from_type == StorageType.S3 and to_type == StorageType.S3 : + raise StorageError( + f"FILE and S3", f"Cannot copy file {from_path} to S3 object {to_path} : {e}" + ) + elif from_type == StorageType.S3 and to_type == StorageType.S3: from_s3_client, from_bucket = __get_s3_client(from_tray) to_s3_client, to_bucket = __get_s3_client(to_tray) try: if to_s3_client["host"] == from_s3_client["host"]: to_s3_client["client"].copy( - { - 'Bucket': from_bucket, - 'Key': from_base_name - }, - to_bucket, to_base_name + {"Bucket": from_bucket, "Key": from_base_name}, to_bucket, to_base_name ) else: with tempfile.NamedTemporaryFile("w+b") as f: from_s3_client["client"].download_fileobj(from_bucket, from_base_name, f) to_s3_client["client"].upload_file(f.name, to_bucket, to_base_name) - if from_md5 is not None : - to_md5 = to_s3_client["client"].head_object(Bucket=to_bucket, Key=to_base_name)["ETag"].strip('"') + if from_md5 is not None: + to_md5 = ( + to_s3_client["client"] + .head_object(Bucket=to_bucket, Key=to_base_name)["ETag"] + .strip('"') + ) if to_md5 != from_md5: - raise StorageError(f"S3", f"Invalid MD5 sum control for copy S3 object {from_path} to {to_path} : {from_md5} != {to_md5}") + raise StorageError( + f"S3", + f"Invalid MD5 sum control for copy S3 object {from_path} to {to_path} : {from_md5} != {to_md5}", + ) except Exception as e: raise StorageError(f"S3", f"Cannot copy S3 object {from_path} to {to_path} : {e}") - - elif from_type == StorageType.CEPH and to_type == StorageType.FILE : - + elif from_type == StorageType.CEPH and to_type == StorageType.FILE: ioctx = __get_ceph_ioctx(from_tray) if from_md5 is not None: checker = hashlib.md5() try: - if to_tray != "": os.makedirs(to_tray, exist_ok=True) - f = open(to_path, 'wb') + f = open(to_path, "wb") offset = 0 size = 0 @@ -659,27 +676,29 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f.close() if from_md5 is not None and from_md5 != checker.hexdigest(): - raise StorageError(f"CEPH and FILE", f"Invalid MD5 sum control for copy CEPH object {from_path} to file {to_path} : {from_md5} != {checker.hexdigest()}") + raise StorageError( + f"CEPH and FILE", + f"Invalid MD5 sum control for copy CEPH object {from_path} to file {to_path} : {from_md5} != {checker.hexdigest()}", + ) except Exception as e: - raise StorageError(f"CEPH and FILE", f"Cannot copy CEPH object {from_path} to file {to_path} : {e}") - - - elif from_type == StorageType.FILE and to_type == StorageType.CEPH : + raise StorageError( + f"CEPH and FILE", f"Cannot copy CEPH object {from_path} to file {to_path} : {e}" + ) + elif from_type == StorageType.FILE and to_type == StorageType.CEPH: ioctx = __get_ceph_ioctx(to_tray) if from_md5 is not None: checker = hashlib.md5() try: - f = open(from_path, 'rb') + f = open(from_path, "rb") offset = 0 size = 0 while True: - chunk = f.read(65536) size = len(chunk) ioctx.write(to_base_name, chunk, offset) @@ -694,14 +713,17 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: f.close() if from_md5 is not None and from_md5 != checker.hexdigest(): - raise StorageError(f"FILE and CEPH", f"Invalid MD5 sum control for copy file {from_path} to CEPH object {to_path} : {from_md5} != {checker.hexdigest()}") + raise StorageError( + f"FILE and CEPH", + f"Invalid MD5 sum control for copy file {from_path} to CEPH object {to_path} : {from_md5} != {checker.hexdigest()}", + ) except Exception as e: - raise StorageError(f"FILE and CEPH", f"Cannot copy file {from_path} to CEPH object {to_path} : {e}") - - - elif from_type == StorageType.CEPH and to_type == StorageType.CEPH : + raise StorageError( + f"FILE and CEPH", f"Cannot copy file {from_path} to CEPH object {to_path} : {e}" + ) + elif from_type == StorageType.CEPH and to_type == StorageType.CEPH: from_ioctx = __get_ceph_ioctx(from_tray) to_ioctx = __get_ceph_ioctx(to_tray) @@ -709,7 +731,6 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: checker = hashlib.md5() try: - offset = 0 size = 0 @@ -726,13 +747,15 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: break if from_md5 is not None and from_md5 != checker.hexdigest(): - raise StorageError(f"FILE and CEPH", f"Invalid MD5 sum control for copy CEPH object {from_path} to {to_path} : {from_md5} != {checker.hexdigest()}") + raise StorageError( + f"FILE and CEPH", + f"Invalid MD5 sum control for copy CEPH object {from_path} to {to_path} : {from_md5} != {checker.hexdigest()}", + ) except Exception as e: raise StorageError(f"CEPH", f"Cannot copy CEPH object {from_path} to {to_path} : {e}") - elif from_type == StorageType.CEPH and to_type == StorageType.S3 : - + elif from_type == StorageType.CEPH and to_type == StorageType.S3: from_ioctx = __get_ceph_ioctx(from_tray) s3_client, to_bucket = __get_s3_client(to_tray) @@ -744,7 +767,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: offset = 0 size = 0 - with tempfile.NamedTemporaryFile("w+b",delete=False) as f: + with tempfile.NamedTemporaryFile("w+b", delete=False) as f: name_tmp = f.name while True: chunk = from_ioctx.read(from_base_name, 65536, offset) @@ -763,14 +786,21 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None: os.remove(name_tmp) if from_md5 is not None and from_md5 != checker.hexdigest(): - raise StorageError(f"CEPH and S3", f"Invalid MD5 sum control for copy CEPH object {from_path} to S3 object {to_path} : {from_md5} != {checker.hexdigest()}") + raise StorageError( + f"CEPH and S3", + f"Invalid MD5 sum control for copy CEPH object {from_path} to S3 object {to_path} : {from_md5} != {checker.hexdigest()}", + ) except Exception as e: - raise StorageError(f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}") + raise StorageError( + f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}" + ) else: - raise StorageError(f"{from_type.name} and {to_type.name}", f"Cannot copy from {from_type.name} to {to_type.name}") - + raise StorageError( + f"{from_type.name} and {to_type.name}", + f"Cannot copy from {from_type.name} to {to_type.name}", + ) def link(target_path: str, link_path: str, hard: bool = False) -> None: @@ -786,44 +816,51 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: MissingEnvironmentError: Missing object storage informations """ - target_type, target_path, target_tray, target_base_name = get_infos_from_path(target_path) - link_type, link_path, link_tray, link_base_name = get_infos_from_path(link_path) + target_type, target_path, target_tray, target_base_name = get_infos_from_path(target_path) + link_type, link_path, link_tray, link_base_name = get_infos_from_path(link_path) if target_type != link_type: - raise StorageError(f"{target_type.name} and {link_type.name}", f"Cannot make link between two different storage types") + raise StorageError( + f"{target_type.name} and {link_type.name}", + f"Cannot make link between two different storage types", + ) if hard and target_type != StorageType.FILE: raise StorageError(target_type.name, "Hard link is available only for FILE storage") # Réalisation du lien, selon les types de stockage if target_type == StorageType.S3: - target_s3_client, target_bucket = __get_s3_client(target_tray) link_s3_client, link_bucket = __get_s3_client(link_tray) if target_s3_client["host"] != link_s3_client["host"]: - raise StorageError(f"S3", f"Cannot make link {link_path} -> {target_path} : link works only on the same S3 cluster") + raise StorageError( + f"S3", + f"Cannot make link {link_path} -> {target_path} : link works only on the same S3 cluster", + ) try: target_s3_client["client"].put_object( - Body = f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode('utf-8'), - Bucket = link_bucket, - Key = link_base_name + Body=f"{__OBJECT_SYMLINK_SIGNATURE}{target_bucket}/{target_base_name}".encode( + "utf-8" + ), + Bucket=link_bucket, + Key=link_base_name, ) except Exception as e: raise StorageError("S3", e) elif target_type == StorageType.CEPH: - ioctx = __get_ceph_ioctx(link_tray) try: - ioctx.write_full(link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode('utf-8')) + ioctx.write_full( + link_base_name, f"{__OBJECT_SYMLINK_SIGNATURE}{target_path}".encode("utf-8") + ) except Exception as e: raise StorageError("CEPH", e) elif target_type == StorageType.FILE: - try: if hard: os.link(target_path, link_path) @@ -835,6 +872,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None: else: raise StorageError("UNKNOWN", "Unhandled storage type to make link") + def get_osgeo_path(path: str) -> str: """Return GDAL/OGR Open compliant path and configure storage access @@ -852,17 +890,15 @@ def get_osgeo_path(path: str) -> str: str: GDAL/OGR Open compliant path """ - storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) - + storage_type, unprefixed_path, tray_name, base_name = get_infos_from_path(path) if storage_type == StorageType.S3: - s3_client, bucket_name = __get_s3_client(tray_name) - gdal.SetConfigOption('AWS_SECRET_ACCESS_KEY', s3_client["secret_key"]) - gdal.SetConfigOption('AWS_ACCESS_KEY_ID', s3_client["key"]) - gdal.SetConfigOption('AWS_S3_ENDPOINT', s3_client["host"]) - gdal.SetConfigOption('AWS_VIRTUAL_HOSTING', 'FALSE') + gdal.SetConfigOption("AWS_SECRET_ACCESS_KEY", s3_client["secret_key"]) + gdal.SetConfigOption("AWS_ACCESS_KEY_ID", s3_client["key"]) + gdal.SetConfigOption("AWS_S3_ENDPOINT", s3_client["host"]) + gdal.SetConfigOption("AWS_VIRTUAL_HOSTING", "FALSE") return f"/vsis3/{bucket_name}/{base_name}" diff --git a/src/rok4/TileMatrixSet.py b/src/rok4/TileMatrixSet.py index 80bd570..472810c 100644 --- a/src/rok4/TileMatrixSet.py +++ b/src/rok4/TileMatrixSet.py @@ -18,6 +18,7 @@ import json import os + class TileMatrix: """A tile matrix is a tile matrix set's level. @@ -30,7 +31,7 @@ class TileMatrix: matrix_size (Tuple[int, int]): Number of tile in the level, widthwise and heightwise. """ - def __init__(self, level: Dict, tms: 'TileMatrixSet') -> None: + def __init__(self, level: Dict, tms: "TileMatrixSet") -> None: """Constructor method Args: @@ -45,12 +46,25 @@ def __init__(self, level: Dict, tms: 'TileMatrixSet') -> None: try: self.id = level["id"] if self.id.find("_") != -1: - raise Exception(f"TMS {tms.path} owns a level whom id contains an underscore ({self.id})") + raise Exception( + f"TMS {tms.path} owns a level whom id contains an underscore ({self.id})" + ) self.resolution = level["cellSize"] - self.origin = (level["pointOfOrigin"][0], level["pointOfOrigin"][1],) - self.tile_size = (level["tileWidth"], level["tileHeight"],) - self.matrix_size = (level["matrixWidth"], level["matrixHeight"],) - self.__latlon = (self.tms.sr.EPSGTreatsAsLatLong() or self.tms.sr.EPSGTreatsAsNorthingEasting()) + self.origin = ( + level["pointOfOrigin"][0], + level["pointOfOrigin"][1], + ) + self.tile_size = ( + level["tileWidth"], + level["tileHeight"], + ) + self.matrix_size = ( + level["matrixWidth"], + level["matrixHeight"], + ) + self.__latlon = ( + self.tms.sr.EPSGTreatsAsLatLong() or self.tms.sr.EPSGTreatsAsNorthingEasting() + ) except KeyError as e: raise MissingAttributeError(tms.path, f"tileMatrices[].{e}") @@ -93,14 +107,14 @@ def tile_to_bbox(self, tile_col: int, tile_row: int) -> Tuple[float, float, floa self.origin[1] - self.resolution * (tile_row + 1) * self.tile_size[1], self.origin[0] + self.resolution * tile_col * self.tile_size[0], self.origin[1] - self.resolution * tile_row * self.tile_size[1], - self.origin[0] + self.resolution * (tile_col + 1) * self.tile_size[0] + self.origin[0] + self.resolution * (tile_col + 1) * self.tile_size[0], ) else: return ( self.origin[0] + self.resolution * tile_col * self.tile_size[0], self.origin[1] - self.resolution * (tile_row + 1) * self.tile_size[1], self.origin[0] + self.resolution * (tile_col + 1) * self.tile_size[0], - self.origin[1] - self.resolution * tile_row * self.tile_size[1] + self.origin[1] - self.resolution * tile_row * self.tile_size[1], ) def bbox_to_tiles(self, bbox: Tuple[float, float, float, float]) -> Tuple[int, int, int, int]: @@ -120,14 +134,14 @@ def bbox_to_tiles(self, bbox: Tuple[float, float, float, float]) -> Tuple[int, i self.x_to_column(bbox[1]), self.y_to_row(bbox[2]), self.x_to_column(bbox[3]), - self.y_to_row(bbox[0]) + self.y_to_row(bbox[0]), ) else: return ( self.x_to_column(bbox[0]), self.y_to_row(bbox[3]), self.x_to_column(bbox[2]), - self.y_to_row(bbox[1]) + self.y_to_row(bbox[1]), ) def point_to_indices(self, x: float, y: float) -> Tuple[int, int, int, int]: @@ -150,7 +164,13 @@ def point_to_indices(self, x: float, y: float) -> Tuple[int, int, int, int]: absolute_pixel_column = int((x - self.origin[0]) / self.resolution) absolute_pixel_row = int((self.origin[1] - y) / self.resolution) - return absolute_pixel_column // self.tile_size[0], absolute_pixel_row // self.tile_size[1], absolute_pixel_column % self.tile_size[0], absolute_pixel_row % self.tile_size[1] + return ( + absolute_pixel_column // self.tile_size[0], + absolute_pixel_row // self.tile_size[1], + absolute_pixel_column % self.tile_size[0], + absolute_pixel_row % self.tile_size[1], + ) + class TileMatrixSet: """A tile matrix set is multi levels grid definition @@ -181,7 +201,7 @@ def __init__(self, name: str) -> None: self.name = name try: - self.path = os.path.join(os.environ["ROK4_TMS_DIRECTORY"], f"{self.name}.json"); + self.path = os.path.join(os.environ["ROK4_TMS_DIRECTORY"], f"{self.name}.json") except KeyError as e: raise MissingEnvironmentError(e) @@ -200,7 +220,9 @@ def __init__(self, name: str) -> None: raise Exception(f"TMS '{self.path}' has no level") if data["orderedAxes"] != ["X", "Y"] and data["orderedAxes"] != ["Lon", "Lat"]: - raise Exception(f"TMS '{self.path}' own invalid axes order : only X/Y or Lon/Lat are handled") + raise Exception( + f"TMS '{self.path}' own invalid axes order : only X/Y or Lon/Lat are handled" + ) except JSONDecodeError as e: raise FormatError("JSON", self.path, e) @@ -209,9 +231,11 @@ def __init__(self, name: str) -> None: raise MissingAttributeError(self.path, e) except RuntimeError as e: - raise Exception(f"Wrong attribute 'crs' ('{self.srs}') in '{self.path}', not recognize by OSR") + raise Exception( + f"Wrong attribute 'crs' ('{self.srs}') in '{self.path}', not recognize by OSR" + ) - def get_level(self, level_id: str) -> 'TileMatrix': + def get_level(self, level_id: str) -> "TileMatrix": """Get one level according to its identifier Args: diff --git a/src/rok4/Utils.py b/src/rok4/Utils.py index f31f872..da15aef 100644 --- a/src/rok4/Utils.py +++ b/src/rok4/Utils.py @@ -19,13 +19,16 @@ class ColorFormat(Enum): a common variable format name. The member's value is the allocated bit size associated to this format. """ + BIT = 1 UINT8 = 8 FLOAT32 = 32 __SR_BOOK = {} -def srs_to_spatialreference(srs: str) -> 'osgeo.osr.SpatialReference': + + +def srs_to_spatialreference(srs: str) -> "osgeo.osr.SpatialReference": """Convert coordinates system as string to OSR spatial reference Using a cache, to instanciate a Spatial Reference from a string only once. @@ -43,8 +46,7 @@ def srs_to_spatialreference(srs: str) -> 'osgeo.osr.SpatialReference': global __SR_BOOK if srs.upper() not in __SR_BOOK: - - authority, code = srs.split(':', 1) + authority, code = srs.split(":", 1) sr = osr.SpatialReference() if authority.upper() == "EPSG": @@ -54,10 +56,12 @@ def srs_to_spatialreference(srs: str) -> 'osgeo.osr.SpatialReference': __SR_BOOK[srs.upper()] = sr - return __SR_BOOK[srs.upper()] -def bbox_to_geometry(bbox: Tuple[float, float, float, float], densification: int = 0) -> 'osgeo.ogr.Geometry': + +def bbox_to_geometry( + bbox: Tuple[float, float, float, float], densification: int = 0 +) -> "osgeo.ogr.Geometry": """Convert bbox coordinates to OGR geometry Args: @@ -94,7 +98,6 @@ def bbox_to_geometry(bbox: Tuple[float, float, float, float], densification: int ring.AddPoint(bbox[0], bbox[3]) ring.AddPoint(bbox[0], bbox[1]) - geom = ogr.Geometry(ogr.wkbPolygon) geom.AddGeometry(ring) geom.SetCoordinateDimension(2) @@ -102,8 +105,9 @@ def bbox_to_geometry(bbox: Tuple[float, float, float, float], densification: int return geom - -def reproject_bbox(bbox: Tuple[float, float, float, float], srs_src: str, srs_dst: str, densification: int = 5) -> Tuple[float, float, float, float]: +def reproject_bbox( + bbox: Tuple[float, float, float, float], srs_src: str, srs_dst: str, densification: int = 5 +) -> Tuple[float, float, float, float]: """Return bounding box in other coordinates system Points are added to be sure output bounding box contains input bounding box @@ -119,10 +123,10 @@ def reproject_bbox(bbox: Tuple[float, float, float, float], srs_src: str, srs_ds """ sr_src = srs_to_spatialreference(srs_src) - sr_src_inv = (sr_src.EPSGTreatsAsLatLong() or sr_src.EPSGTreatsAsNorthingEasting()) + sr_src_inv = sr_src.EPSGTreatsAsLatLong() or sr_src.EPSGTreatsAsNorthingEasting() sr_dst = srs_to_spatialreference(srs_dst) - sr_dst_inv = (sr_dst.EPSGTreatsAsLatLong() or sr_dst.EPSGTreatsAsNorthingEasting()) + sr_dst_inv = sr_dst.EPSGTreatsAsLatLong() or sr_dst.EPSGTreatsAsNorthingEasting() if sr_src.IsSame(sr_dst) and sr_src_inv == sr_dst_inv: # Les système sont vraiment les même, avec le même ordre des axes @@ -144,7 +148,11 @@ def reproject_bbox(bbox: Tuple[float, float, float, float], srs_src: str, srs_ds return (env[0], env[2], env[1], env[3]) -def reproject_point(point: Tuple[float, float], sr_src: 'osgeo.osr.SpatialReference', sr_dst: 'osgeo.osr.SpatialReference') -> Tuple[float, float]: +def reproject_point( + point: Tuple[float, float], + sr_src: "osgeo.osr.SpatialReference", + sr_dst: "osgeo.osr.SpatialReference", +) -> Tuple[float, float]: """Reproject a point Args: @@ -156,8 +164,8 @@ def reproject_point(point: Tuple[float, float], sr_src: 'osgeo.osr.SpatialRefere Tuple[float, float]: X/Y in destination spatial reference """ - sr_src_inv = (sr_src.EPSGTreatsAsLatLong() or sr_src.EPSGTreatsAsNorthingEasting()) - sr_dst_inv = (sr_dst.EPSGTreatsAsLatLong() or sr_dst.EPSGTreatsAsNorthingEasting()) + sr_src_inv = sr_src.EPSGTreatsAsLatLong() or sr_src.EPSGTreatsAsNorthingEasting() + sr_dst_inv = sr_dst.EPSGTreatsAsLatLong() or sr_dst.EPSGTreatsAsNorthingEasting() if sr_src.IsSame(sr_dst) and sr_src_inv == sr_dst_inv: # Les système sont vraiment les même, avec le même ordre des axes @@ -198,31 +206,21 @@ def compute_bbox(source_dataset: gdal.Dataset) -> Tuple: height = source_dataset.RasterYSize x_range = ( transform_vector[0], - transform_vector[0] + width * transform_vector[1] + height * transform_vector[2] + transform_vector[0] + width * transform_vector[1] + height * transform_vector[2], ) y_range = ( transform_vector[3], - transform_vector[3] + width * transform_vector[4] + height * transform_vector[5] + transform_vector[3] + width * transform_vector[4] + height * transform_vector[5], ) spatial_ref = source_dataset.GetSpatialRef() if spatial_ref is not None and spatial_ref.GetDataAxisToSRSAxisMapping() == [2, 1]: # Coordonnées terrain de type (latitude, longitude) # => on permute les coordonnées terrain par rapport à l'image - bbox = ( - min(y_range), - min(x_range), - max(y_range), - max(x_range) - ) + bbox = (min(y_range), min(x_range), max(y_range), max(x_range)) else: # Coordonnées terrain de type (longitude, latitude) ou pas de SRS # => les coordonnées terrain sont dans le même ordre que celle de l'image - bbox = ( - min(x_range), - min(y_range), - max(x_range), - max(y_range) - ) + bbox = (min(x_range), min(y_range), max(x_range), max(y_range)) return bbox @@ -252,8 +250,12 @@ def compute_format(dataset: gdal.Dataset, path: str = None) -> ColorFormat: color_name = gdal.GetColorInterpretationName(color_interpretation) compression_regex_match = re.search(r"COMPRESSION\s*=\s*PACKBITS", gdal.Info(dataset)) - if (data_type_name == "Byte" and data_type_size == 8 - and color_name == "Palette" and compression_regex_match): + if ( + data_type_name == "Byte" + and data_type_size == 8 + and color_name == "Palette" + and compression_regex_match + ): # Compris par libTIFF comme du noir et blanc sur 1 bit color_format = ColorFormat.BIT elif data_type_name == "Byte" and data_type_size == 8: @@ -261,6 +263,8 @@ def compute_format(dataset: gdal.Dataset, path: str = None) -> ColorFormat: elif data_type_name == "Float32" and data_type_size == 32: color_format = ColorFormat.FLOAT32 else: - raise Exception(f"Unsupported color format for image {path} : " - + f"'{data_type_name}' ({data_type_size} bits)") + raise Exception( + f"Unsupported color format for image {path} : " + + f"'{data_type_name}' ({data_type_size} bits)" + ) return color_format diff --git a/src/rok4/Vector.py b/src/rok4/Vector.py index 9cb8c00..94e6def 100644 --- a/src/rok4/Vector.py +++ b/src/rok4/Vector.py @@ -15,7 +15,8 @@ # Enable GDAL/OGR exceptions ogr.UseExceptions() -class Vector : + +class Vector: """A data vector Attributes: @@ -25,7 +26,7 @@ class Vector : """ @classmethod - def from_file(cls, path: str, **kwargs) -> 'Vector' : + def from_file(cls, path: str, **kwargs) -> "Vector": """Constructor method of a Vector from a file (Shapefile, Geopackage, CSV and GeoJSON) Args: @@ -65,9 +66,8 @@ def from_file(cls, path: str, **kwargs) -> 'Vector' : path_split = path.split("/") if path_split[0] == "ceph:" or path.endswith(".csv"): - - if path.endswith(".shp") : - with tempfile.TemporaryDirectory() as tmp : + if path.endswith(".shp"): + with tempfile.TemporaryDirectory() as tmp: tmp_path = tmp + "/" + path_split[-1][:-4] copy(path, "file://" + tmp_path + ".shp") @@ -78,99 +78,111 @@ def from_file(cls, path: str, **kwargs) -> 'Vector' : dataSource = ogr.Open(tmp_path + ".shp", 0) - elif path.endswith(".gpkg") : - with tempfile.TemporaryDirectory() as tmp : + elif path.endswith(".gpkg"): + with tempfile.TemporaryDirectory() as tmp: tmp_path = tmp + "/" + path_split[-1][:-5] copy(path, "file://" + tmp_path + ".gpkg") dataSource = ogr.Open(tmp_path + ".gpkg", 0) - elif path.endswith(".geojson") : - with tempfile.TemporaryDirectory() as tmp : + elif path.endswith(".geojson"): + with tempfile.TemporaryDirectory() as tmp: tmp_path = tmp + "/" + path_split[-1][:-8] copy(path, "file://" + tmp_path + ".geojson") dataSource = ogr.Open(tmp_path + ".geojson", 0) - elif path.endswith(".csv") : + elif path.endswith(".csv"): # Récupération des informations optionnelles - if "csv" in kwargs : + if "csv" in kwargs: csv = kwargs["csv"] - else : + else: csv = {} - if "srs" in csv and csv["srs"] is not None : + if "srs" in csv and csv["srs"] is not None: srs = csv["srs"] - else : + else: srs = "EPSG:2154" - if "column_x" in csv and csv["column_x"] is not None : + if "column_x" in csv and csv["column_x"] is not None: column_x = csv["column_x"] - else : + else: column_x = "x" - if "column_y" in csv and csv["column_y"] is not None : + if "column_y" in csv and csv["column_y"] is not None: column_y = csv["column_y"] - else : + else: column_y = "y" - if "column_wkt" in csv : + if "column_wkt" in csv: column_wkt = csv["column_wkt"] - else : + else: column_wkt = None - with tempfile.TemporaryDirectory() as tmp : + with tempfile.TemporaryDirectory() as tmp: tmp_path = tmp + "/" + path_split[-1][:-4] name_fich = path_split[-1][:-4] copy(path, "file://" + tmp_path + ".csv") - with tempfile.NamedTemporaryFile(mode='w', suffix=".vrt", dir=tmp, delete=False) as tmp2 : + with tempfile.NamedTemporaryFile( + mode="w", suffix=".vrt", dir=tmp, delete=False + ) as tmp2: vrt_file = "\n" vrt_file += '\n' vrt_file += "" + tmp_path + ".csv\n" vrt_file += "" + name_fich + "\n" vrt_file += "" + srs + "\n" - if column_wkt == None : - vrt_file += '\n' - else : - vrt_file += '\n' + if column_wkt == None: + vrt_file += ( + '\n' + ) + else: + vrt_file += ( + '\n' + ) vrt_file += "\n" vrt_file += "" tmp2.write(vrt_file) dataSourceVRT = ogr.Open(tmp2.name, 0) os.remove(tmp2.name) - dataSource=ogr.GetDriverByName("ESRI Shapefile").CopyDataSource(dataSourceVRT, tmp_path + "shp") + dataSource = ogr.GetDriverByName("ESRI Shapefile").CopyDataSource( + dataSourceVRT, tmp_path + "shp" + ) - else : + else: raise Exception("This format of file cannot be loaded") - else : + else: dataSource = ogr.Open(get_osgeo_path(path), 0) multipolygon = ogr.Geometry(ogr.wkbGeometryCollection) - try : + try: layer = dataSource.GetLayer() - except AttributeError : + except AttributeError: raise Exception(f"The content of {self.path} cannot be read") layers = [] - for i in range (dataSource.GetLayerCount()) : + for i in range(dataSource.GetLayerCount()): layer = dataSource.GetLayer(i) name = layer.GetName() count = layer.GetFeatureCount() layerDefinition = layer.GetLayerDefn() attributes = [] for j in range(layerDefinition.GetFieldCount()): - fieldName = layerDefinition.GetFieldDefn(j).GetName() + fieldName = layerDefinition.GetFieldDefn(j).GetName() fieldTypeCode = layerDefinition.GetFieldDefn(j).GetType() fieldType = layerDefinition.GetFieldDefn(j).GetFieldTypeName(fieldTypeCode) attributes += [(fieldName, fieldType)] - for feature in layer : + for feature in layer: geom = feature.GetGeometryRef() - if geom != None : + if geom != None: multipolygon.AddGeometry(geom) layers += [(name, count, attributes)] @@ -180,7 +192,7 @@ def from_file(cls, path: str, **kwargs) -> 'Vector' : return self @classmethod - def from_parameters(cls, path: str, bbox : tuple, layers : list) -> 'Vector' : + def from_parameters(cls, path: str, bbox: tuple, layers: list) -> "Vector": """Constructor method of a Vector from a parameters Args: diff --git a/tests/test_Layer.py b/tests/test_Layer.py index 26b0963..0a60d5c 100644 --- a/tests/test_Layer.py +++ b/tests/test_Layer.py @@ -8,12 +8,15 @@ from unittest.mock import * from unittest import mock + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Layer.get_data_str', return_value='{"pyramids" : [{"bottom_level" : "10","top_level" : "10","path" : "s3://pyramids/SCAN1000.json"}],"title" : "SCAN1000","bbox":{"east": 11.250000000000997,"west": -5.624999999999043,"north": 52.48278022207774,"south": 40.9798980696195},"styles" : ["normal","hypso"],"abstract" : "Diffusion de la donnée BDORTHO","resampling" : "linear","keywords" : ["PM","TIFF_JPG_UINT8"]}') -@mock.patch('rok4.Layer.Pyramid.from_descriptor') -@mock.patch('rok4.Layer.put_data_str', return_value=None) +@mock.patch( + "rok4.Layer.get_data_str", + return_value='{"pyramids" : [{"bottom_level" : "10","top_level" : "10","path" : "s3://pyramids/SCAN1000.json"}],"title" : "SCAN1000","bbox":{"east": 11.250000000000997,"west": -5.624999999999043,"north": 52.48278022207774,"south": 40.9798980696195},"styles" : ["normal","hypso"],"abstract" : "Diffusion de la donnée BDORTHO","resampling" : "linear","keywords" : ["PM","TIFF_JPG_UINT8"]}', +) +@mock.patch("rok4.Layer.Pyramid.from_descriptor") +@mock.patch("rok4.Layer.put_data_str", return_value=None) def test_descriptor_ok(mocked_put_data_str, mocked_pyramid_class, mocked_get_data_str): - tms_instance = MagicMock() tms_instance.srs = "EPSG:3857" @@ -26,7 +29,7 @@ def test_descriptor_ok(mocked_put_data_str, mocked_pyramid_class, mocked_get_dat "channels": 3, "nodata": "255,255,255", "photometric": "rgb", - "interpolation": "bicubic" + "interpolation": "bicubic", } pyramid_instance.format = "TIFF_JPG_UINT8" pyramid_instance.tms = tms_instance @@ -37,19 +40,23 @@ def test_descriptor_ok(mocked_put_data_str, mocked_pyramid_class, mocked_get_dat try: layer = Layer.from_descriptor("s3://layers/SCAN1000.json") assert layer.type == PyramidType.RASTER - mocked_get_data_str.assert_called_once_with('s3://layers/SCAN1000.json') + mocked_get_data_str.assert_called_once_with("s3://layers/SCAN1000.json") layer.write_descriptor("s3://layers_backup/") - mocked_put_data_str.assert_called_once_with('{"title": "SCAN1000", "abstract": "Diffusion de la donn\\u00e9e BDORTHO", "keywords": ["PM", "TIFF_JPG_UINT8"], "wmts": {"authorized": true}, "tms": {"authorized": true}, "bbox": {"south": 40.9798980696195, "west": -5.624999999999043, "north": 52.48278022207774, "east": 11.250000000000997}, "pyramids": [{"bottom_level": "10", "top_level": "10", "path": "s3://pyramids/SCAN1000.json"}], "wms": {"authorized": true, "crs": ["CRS:84", "IGNF:WGS84G", "EPSG:3857", "EPSG:4258", "EPSG:4326"]}, "styles": ["normal", "hypso"], "resampling": "linear"}', 's3://layers_backup/SCAN1000.json') + mocked_put_data_str.assert_called_once_with( + '{"title": "SCAN1000", "abstract": "Diffusion de la donn\\u00e9e BDORTHO", "keywords": ["PM", "TIFF_JPG_UINT8"], "wmts": {"authorized": true}, "tms": {"authorized": true}, "bbox": {"south": 40.9798980696195, "west": -5.624999999999043, "north": 52.48278022207774, "east": 11.250000000000997}, "pyramids": [{"bottom_level": "10", "top_level": "10", "path": "s3://pyramids/SCAN1000.json"}], "wms": {"authorized": true, "crs": ["CRS:84", "IGNF:WGS84G", "EPSG:3857", "EPSG:4258", "EPSG:4326"]}, "styles": ["normal", "hypso"], "resampling": "linear"}', + "s3://layers_backup/SCAN1000.json", + ) except Exception as exc: assert False, f"Layer creation from descriptor raises an exception: {exc}" -@mock.patch('rok4.Layer.Pyramid.from_descriptor') -@mock.patch('rok4.Layer.reproject_bbox', return_value=(0, 0, 100, 100)) -@mock.patch('rok4.Layer.put_data_str', return_value=None) -def test_parameters_vector_ok(mocked_put_data_str, mocked_utils_reproject_bbox, mocked_pyramid_class): - +@mock.patch("rok4.Layer.Pyramid.from_descriptor") +@mock.patch("rok4.Layer.reproject_bbox", return_value=(0, 0, 100, 100)) +@mock.patch("rok4.Layer.put_data_str", return_value=None) +def test_parameters_vector_ok( + mocked_put_data_str, mocked_utils_reproject_bbox, mocked_pyramid_class +): tms_instance = MagicMock() tms_instance.srs = "EPSG:3857" @@ -67,29 +74,35 @@ def test_parameters_vector_ok(mocked_put_data_str, mocked_utils_reproject_bbox, try: layer = Layer.from_parameters( - [{ - "path": "file:///home/ign/pyramids/SCAN1000.json", - "bottom_level": "10", - "top_level": "10" - }], + [ + { + "path": "file:///home/ign/pyramids/SCAN1000.json", + "bottom_level": "10", + "top_level": "10", + } + ], "layername", - title = "title", - abstract = "abstract" + title="title", + abstract="abstract", ) assert layer.type == PyramidType.VECTOR assert layer.geobbox == (0, 0, 100, 100) layer.write_descriptor("file:///home/ign/layers/") - mocked_put_data_str.assert_called_once_with('{"title": "title", "abstract": "abstract", "keywords": ["VECTOR", "layername"], "wmts": {"authorized": true}, "tms": {"authorized": true}, "bbox": {"south": 0, "west": 0, "north": 100, "east": 100}, "pyramids": [{"bottom_level": "10", "top_level": "10", "path": "file:///home/ign/pyramids/SCAN1000.json"}]}', 'file:///home/ign/layers/layername.json') + mocked_put_data_str.assert_called_once_with( + '{"title": "title", "abstract": "abstract", "keywords": ["VECTOR", "layername"], "wmts": {"authorized": true}, "tms": {"authorized": true}, "bbox": {"south": 0, "west": 0, "north": 100, "east": 100}, "pyramids": [{"bottom_level": "10", "top_level": "10", "path": "file:///home/ign/pyramids/SCAN1000.json"}]}', + "file:///home/ign/layers/layername.json", + ) except Exception as exc: assert False, f"Layer creation from parameters raises an exception: {exc}" -@mock.patch('rok4.Layer.Pyramid.from_descriptor') -@mock.patch('rok4.Layer.reproject_bbox', return_value=(0, 0, 100, 100)) -@mock.patch('rok4.Layer.put_data_str', return_value=None) -def test_parameters_raster_ok(mocked_put_data_str, mocked_utils_reproject_bbox, mocked_pyramid_class): - +@mock.patch("rok4.Layer.Pyramid.from_descriptor") +@mock.patch("rok4.Layer.reproject_bbox", return_value=(0, 0, 100, 100)) +@mock.patch("rok4.Layer.put_data_str", return_value=None) +def test_parameters_raster_ok( + mocked_put_data_str, mocked_utils_reproject_bbox, mocked_pyramid_class +): tms_instance = MagicMock() tms_instance.srs = "EPSG:3857" @@ -104,7 +117,7 @@ def test_parameters_raster_ok(mocked_put_data_str, mocked_utils_reproject_bbox, "channels": 1, "nodata": "-99999", "photometric": "gray", - "interpolation": "nn" + "interpolation": "nn", } pyramid_instance.tms = tms_instance pyramid_instance.bottom_level.id = "10" @@ -115,18 +128,18 @@ def test_parameters_raster_ok(mocked_put_data_str, mocked_utils_reproject_bbox, try: layer = Layer.from_parameters( - [{ - "path": "file:///home/ign/pyramids/RGEALTI.json" - }], + [{"path": "file:///home/ign/pyramids/RGEALTI.json"}], "layername", - title = "title", - abstract = "abstract" + title="title", + abstract="abstract", ) assert layer.type == PyramidType.RASTER assert layer.geobbox == (0, 0, 100, 100) layer.write_descriptor("file:///home/ign/layers/") - mocked_put_data_str.assert_called_once_with('{"title": "title", "abstract": "abstract", "keywords": ["RASTER", "layername"], "wmts": {"authorized": true}, "tms": {"authorized": true}, "bbox": {"south": 0, "west": 0, "north": 100, "east": 100}, "pyramids": [{"bottom_level": "10", "top_level": "10", "path": "file:///home/ign/pyramids/RGEALTI.json"}], "wms": {"authorized": true, "crs": ["CRS:84", "IGNF:WGS84G", "EPSG:3857", "EPSG:4258", "EPSG:4326"]}, "styles": ["normal"], "resampling": "nn"}', 'file:///home/ign/layers/layername.json') - + mocked_put_data_str.assert_called_once_with( + '{"title": "title", "abstract": "abstract", "keywords": ["RASTER", "layername"], "wmts": {"authorized": true}, "tms": {"authorized": true}, "bbox": {"south": 0, "west": 0, "north": 100, "east": 100}, "pyramids": [{"bottom_level": "10", "top_level": "10", "path": "file:///home/ign/pyramids/RGEALTI.json"}], "wms": {"authorized": true, "crs": ["CRS:84", "IGNF:WGS84G", "EPSG:3857", "EPSG:4258", "EPSG:4326"]}, "styles": ["normal"], "resampling": "nn"}', + "file:///home/ign/layers/layername.json", + ) except Exception as exc: assert False, f"Layer creation from parameters raises an exception: {exc}" diff --git a/tests/test_Pyramid.py b/tests/test_Pyramid.py index c1936dd..6eebfd7 100644 --- a/tests/test_Pyramid.py +++ b/tests/test_Pyramid.py @@ -9,56 +9,73 @@ from unittest.mock import * from unittest import mock -@mock.patch('rok4.Pyramid.get_data_str', side_effect=StorageError('FILE', 'Not found')) + +@mock.patch("rok4.Pyramid.get_data_str", side_effect=StorageError("FILE", "Not found")) def test_wrong_file(mocked_get_data_str): with pytest.raises(StorageError): pyramid = Pyramid.from_descriptor("file:///pyramid.json") -@mock.patch('rok4.Pyramid.get_data_str', return_value='{"format": "TIFF_PBF_MVT","levels":[{"id": "100","tables":') + +@mock.patch( + "rok4.Pyramid.get_data_str", + return_value='{"format": "TIFF_PBF_MVT","levels":[{"id": "100","tables":', +) def test_bad_json(mocked_get_data_str): with pytest.raises(FormatError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") - assert str(exc.value) == "Expected format JSON to read 'file:///pyramid.json' : Expecting value: line 1 column 59 (char 58)" - mocked_get_data_str.assert_called_once_with('file:///pyramid.json') + assert ( + str(exc.value) + == "Expected format JSON to read 'file:///pyramid.json' : Expecting value: line 1 column 59 (char 58)" + ) + mocked_get_data_str.assert_called_once_with("file:///pyramid.json") + -@mock.patch('rok4.Pyramid.get_data_str', return_value='{"format": "TIFF_PBF_MVT","levels":[]}') +@mock.patch("rok4.Pyramid.get_data_str", return_value='{"format": "TIFF_PBF_MVT","levels":[]}') def test_missing_tms(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") assert str(exc.value) == "Missing attribute 'tile_matrix_set' in 'file:///pyramid.json'" - mocked_get_data_str.assert_called_once_with('file:///pyramid.json') + mocked_get_data_str.assert_called_once_with("file:///pyramid.json") @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.get_data_str', return_value='{"format": "TIFF_PBF_MVT","levels":[{}], "tile_matrix_set": "PM"}') -@mock.patch('rok4.Pyramid.TileMatrixSet', side_effect=StorageError('FILE', 'TMS not found')) +@mock.patch( + "rok4.Pyramid.get_data_str", + return_value='{"format": "TIFF_PBF_MVT","levels":[{}], "tile_matrix_set": "PM"}', +) +@mock.patch("rok4.Pyramid.TileMatrixSet", side_effect=StorageError("FILE", "TMS not found")) def test_wrong_tms(mocked_tms_constructor, mocked_get_data_str): with pytest.raises(StorageError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") assert str(exc.value) == "Issue occured using a FILE storage : TMS not found" - mocked_tms_constructor.assert_called_once_with('PM') - mocked_get_data_str.assert_called_once_with('file:///pyramid.json') + mocked_tms_constructor.assert_called_once_with("PM") + mocked_get_data_str.assert_called_once_with("file:///pyramid.json") + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.get_data_str', return_value='{"format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}') -@mock.patch('rok4.Pyramid.TileMatrixSet') +@mock.patch( + "rok4.Pyramid.get_data_str", + return_value='{"format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}', +) +@mock.patch("rok4.Pyramid.TileMatrixSet") def test_raster_missing_raster_specifications(mocked_tms_class, mocked_get_data_str): - with pytest.raises(MissingAttributeError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") assert str(exc.value) == "Missing attribute 'raster_specifications' in 'file:///pyramid.json'" - mocked_get_data_str.assert_called_once_with('file:///pyramid.json') + mocked_get_data_str.assert_called_once_with("file:///pyramid.json") @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.get_data_str', return_value='{"raster_specifications":{"channels":3,"nodata":"255,0,0","photometric":"rgb","interpolation":"bicubic"}, "format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"unknown"}], "tile_matrix_set": "PM"}') -@mock.patch('rok4.Pyramid.TileMatrixSet') +@mock.patch( + "rok4.Pyramid.get_data_str", + return_value='{"raster_specifications":{"channels":3,"nodata":"255,0,0","photometric":"rgb","interpolation":"bicubic"}, "format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"unknown"}], "tile_matrix_set": "PM"}', +) +@mock.patch("rok4.Pyramid.TileMatrixSet") def test_wrong_level(mocked_tms_class, mocked_get_data_str): - tms_instance = MagicMock() tms_instance.get_level.return_value = None tms_instance.name = "PM" @@ -67,28 +84,37 @@ def test_wrong_level(mocked_tms_class, mocked_get_data_str): with pytest.raises(Exception) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") - mocked_tms_class.assert_called_once_with('PM') - mocked_get_data_str.assert_called_once_with('file:///pyramid.json') - tms_instance.get_level.assert_called_once_with('unknown') - assert str(exc.value) == "Pyramid file:///pyramid.json owns a level with the ID 'unknown', not defined in the TMS 'PM'" + mocked_tms_class.assert_called_once_with("PM") + mocked_get_data_str.assert_called_once_with("file:///pyramid.json") + tms_instance.get_level.assert_called_once_with("unknown") + assert ( + str(exc.value) + == "Pyramid file:///pyramid.json owns a level with the ID 'unknown', not defined in the TMS 'PM'" + ) + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.get_data_str', return_value='{"format": "TIFF_PBF_MVT","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}') -@mock.patch('rok4.Pyramid.TileMatrixSet', autospec=True) +@mock.patch( + "rok4.Pyramid.get_data_str", + return_value='{"format": "TIFF_PBF_MVT","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}', +) +@mock.patch("rok4.Pyramid.TileMatrixSet", autospec=True) def test_vector_missing_tables(mocked_tms_class, mocked_get_data_str): - with pytest.raises(MissingAttributeError) as exc: pyramid = Pyramid.from_descriptor("file:///pyramid.json") assert str(exc.value) == "Missing attribute levels[].'tables' in 'file:///pyramid.json'" - mocked_get_data_str.assert_called_once_with('file:///pyramid.json') + mocked_get_data_str.assert_called_once_with("file:///pyramid.json") + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.get_data_str', return_value='{"raster_specifications":{"channels":3,"nodata":"255,0,0","photometric":"rgb","interpolation":"bicubic"}, "format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_prefix":"SCAN1000/DATA_0","pool_name":"pool1","type":"CEPH"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}') -@mock.patch('rok4.Pyramid.TileMatrixSet') -@mock.patch('rok4.Pyramid.put_data_str', return_value=None) +@mock.patch( + "rok4.Pyramid.get_data_str", + return_value='{"raster_specifications":{"channels":3,"nodata":"255,0,0","photometric":"rgb","interpolation":"bicubic"}, "format": "TIFF_JPG_UINT8","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_prefix":"SCAN1000/DATA_0","pool_name":"pool1","type":"CEPH"},"tiles_per_width":16,"id":"0"}], "tile_matrix_set": "PM"}', +) +@mock.patch("rok4.Pyramid.TileMatrixSet") +@mock.patch("rok4.Pyramid.put_data_str", return_value=None) def test_raster_ok(mocked_put_data_str, mocked_tms_class, mocked_get_data_str): - tms_instance = MagicMock() tms_instance.name = "PM" tms_instance.srs = "EPSG:3857" @@ -97,7 +123,7 @@ def test_raster_ok(mocked_put_data_str, mocked_tms_class, mocked_get_data_str): tm_instance = MagicMock() tm_instance.id = "0" tm_instance.resolution = 1 - tm_instance.point_to_indices.return_value = (0,0,128,157) + tm_instance.point_to_indices.return_value = (0, 0, 128, 157) tms_instance.get_level.return_value = tm_instance @@ -110,7 +136,7 @@ def test_raster_ok(mocked_put_data_str, mocked_tms_class, mocked_get_data_str): assert pyramid.name == "sub/pyramid" assert pyramid.storage_type == StorageType.CEPH assert pyramid.storage_root == "pool1" - mocked_get_data_str.assert_called_once_with('ceph://pool1/sub/pyramid.json') + mocked_get_data_str.assert_called_once_with("ceph://pool1/sub/pyramid.json") clone = Pyramid.from_other(pyramid, "titi", {"type": "FILE", "root": "/data/ign"}) assert clone.name == "titi" @@ -119,23 +145,33 @@ def test_raster_ok(mocked_put_data_str, mocked_tms_class, mocked_get_data_str): assert clone.tile_extension == "jpg" assert clone.get_level("0") is not None assert clone.get_level("4") is None - assert clone.get_infos_from_slab_path("/data/ign/titi/IMAGE/12/00/4A/F7.tif") == (SlabType.DATA, "12", 159, 367) - assert clone.get_tile_indices(102458, 6548125, srs = "EPSG:3857") == ("0",0,0,128,157) - assert clone.get_tile_indices(43, 2, srs = "EPSG:4326") == ("0",0,0,128,157) - + assert clone.get_infos_from_slab_path("/data/ign/titi/IMAGE/12/00/4A/F7.tif") == ( + SlabType.DATA, + "12", + 159, + 367, + ) + assert clone.get_tile_indices(102458, 6548125, srs="EPSG:3857") == ("0", 0, 0, 128, 157) + assert clone.get_tile_indices(43, 2, srs="EPSG:4326") == ("0", 0, 0, 128, 157) assert len(clone.get_levels()) == 1 clone.write_descriptor() - mocked_put_data_str.assert_called_once_with('{"tile_matrix_set": "PM", "format": "TIFF_JPG_UINT8", "levels": [{"id": "0", "tiles_per_width": 16, "tiles_per_height": 16, "tile_limits": {"min_col": 0, "max_row": 15, "max_col": 15, "min_row": 0}, "storage": {"type": "FILE", "image_directory": "titi/DATA/0", "path_depth": 2}}], "raster_specifications": {"channels": 3, "nodata": "255,0,0", "photometric": "rgb", "interpolation": "bicubic"}}', 'file:///data/ign/titi.json') + mocked_put_data_str.assert_called_once_with( + '{"tile_matrix_set": "PM", "format": "TIFF_JPG_UINT8", "levels": [{"id": "0", "tiles_per_width": 16, "tiles_per_height": 16, "tile_limits": {"min_col": 0, "max_row": 15, "max_col": 15, "min_row": 0}, "storage": {"type": "FILE", "image_directory": "titi/DATA/0", "path_depth": 2}}], "raster_specifications": {"channels": 3, "nodata": "255,0,0", "photometric": "rgb", "interpolation": "bicubic"}}', + "file:///data/ign/titi.json", + ) except Exception as exc: assert False, f"Pyramid creation raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.get_data_str', return_value='{"format": "TIFF_PBF_MVT","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0","tables":[{"name":"table","geometry":"POINT","attributes":[{"type":"bigint","name":"fid","count":1531}]}]}], "tile_matrix_set": "PM"}') -@mock.patch('rok4.Pyramid.TileMatrixSet') +@mock.patch( + "rok4.Pyramid.get_data_str", + return_value='{"format": "TIFF_PBF_MVT","levels":[{"tiles_per_height":16,"tile_limits":{"min_col":0,"max_row":15,"max_col":15,"min_row":0},"storage":{"image_directory":"SCAN1000/DATA/0","path_depth":2,"type":"FILE"},"tiles_per_width":16,"id":"0","tables":[{"name":"table","geometry":"POINT","attributes":[{"type":"bigint","name":"fid","count":1531}]}]}], "tile_matrix_set": "PM"}', +) +@mock.patch("rok4.Pyramid.TileMatrixSet") def test_vector_ok(mocked_tms_class, mocked_get_data_str): - try: pyramid = Pyramid.from_descriptor("file:///pyramid.json") assert pyramid.get_level("0") is not None @@ -143,7 +179,7 @@ def test_vector_ok(mocked_tms_class, mocked_get_data_str): assert pyramid.name == "pyramid" assert pyramid.storage_depth == 2 assert pyramid.storage_type == StorageType.FILE - mocked_get_data_str.assert_called_once_with('file:///pyramid.json') + mocked_get_data_str.assert_called_once_with("file:///pyramid.json") clone = Pyramid.from_other(pyramid, "toto", {"type": "S3", "root": "bucket"}) assert clone.name == "toto" @@ -159,16 +195,15 @@ def test_vector_ok(mocked_tms_class, mocked_get_data_str): @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.TileMatrixSet') +@mock.patch("rok4.Pyramid.TileMatrixSet") def test_tile_read_raster(mocked_tms_class): - tms_instance = MagicMock() tms_instance.name = "UTM20W84MART_1M_MNT" tms_instance.srs = "IGNF:UTM20W84MART" tm_instance = MagicMock() tm_instance.id = "8" - tm_instance.tile_size = (256,256) + tm_instance.tile_size = (256, 256) tms_instance.get_level.return_value = tm_instance @@ -176,26 +211,24 @@ def test_tile_read_raster(mocked_tms_class): try: pyramid = Pyramid.from_descriptor("file://tests/fixtures/TIFF_ZIP_FLOAT32.json") - data = pyramid.get_tile_data_raster("8",2748,40537) + data = pyramid.get_tile_data_raster("8", 2748, 40537) - assert data.shape == (256,256,1) + assert data.shape == (256, 256, 1) assert data[128][128][0] == 447.25 except Exception as exc: assert False, f"Pyramid raster tile read raises an exception: {exc}" - @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.TileMatrixSet') +@mock.patch("rok4.Pyramid.TileMatrixSet") def test_tile_read_vector(mocked_tms_class): - tms_instance = MagicMock() tms_instance.name = "PM" tms_instance.srs = "EPSG:3857" tm_instance = MagicMock() tm_instance.id = "4" - tm_instance.tile_size = (256,256) + tm_instance.tile_size = (256, 256) tms_instance.get_level.return_value = tm_instance @@ -215,16 +248,15 @@ def test_tile_read_vector(mocked_tms_class): @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('rok4.Pyramid.TileMatrixSet') +@mock.patch("rok4.Pyramid.TileMatrixSet") def test_list_read(mocked_tms_class): - tms_instance = MagicMock() tms_instance.name = "PM" tms_instance.srs = "EPSG:3857" tm_instance = MagicMock() tm_instance.id = "4" - tm_instance.tile_size = (256,256) + tm_instance.tile_size = (256, 256) tms_instance.get_level.return_value = tm_instance @@ -233,22 +265,37 @@ def test_list_read(mocked_tms_class): try: pyramid = Pyramid.from_descriptor("file://tests/fixtures/TIFF_PBF_MVT.json") pyramid.load_list() - pyramid.load_list() # on passe par la détection d'une liste déjà chrargée ainsi + pyramid.load_list() # on passe par la détection d'une liste déjà chrargée ainsi for (slab_type, level, column, row), infos in pyramid.list_generator(): assert slab_type == SlabType.DATA - assert level == '4' + assert level == "4" assert column == 2 assert row == 1 - assert infos == {'link': False, 'md5': None, 'root': 'TIFF_PBF_MVT', 'slab': 'DATA/4/00/00/21.tif'} + assert infos == { + "link": False, + "md5": None, + "root": "TIFF_PBF_MVT", + "slab": "DATA/4/00/00/21.tif", + } except Exception as exc: assert False, f"Pyramid vector list read raises an exception: {exc}" def test_b36_path_decode(): - assert b36_path_decode("3E/42/01.tif") == (4032, 18217,) - assert b36_path_decode("3E/42/01.TIFF") == (4032, 18217,) - assert b36_path_decode("3E/42/01") == (4032, 18217,) + assert b36_path_decode("3E/42/01.tif") == ( + 4032, + 18217, + ) + assert b36_path_decode("3E/42/01.TIFF") == ( + 4032, + 18217, + ) + assert b36_path_decode("3E/42/01") == ( + 4032, + 18217, + ) + def test_b36_path_encode(): assert b36_path_encode(4032, 18217, 2) == "3E/42/01.tif" diff --git a/tests/test_Raster.py b/tests/test_Raster.py index 608c3e2..de35de1 100644 --- a/tests/test_Raster.py +++ b/tests/test_Raster.py @@ -28,10 +28,16 @@ def test_default(self): raster = Raster() assert raster.bands is None - assert (isinstance(raster.bbox, tuple) and len(raster.bbox) == 4 - and all(coordinate is None for coordinate in raster.bbox)) - assert (isinstance(raster.dimensions, tuple) and len(raster.dimensions) == 2 - and all(dimension is None for dimension in raster.dimensions)) + assert ( + isinstance(raster.bbox, tuple) + and len(raster.bbox) == 4 + and all(coordinate is None for coordinate in raster.bbox) + ) + assert ( + isinstance(raster.dimensions, tuple) + and len(raster.dimensions) == 2 + and all(dimension is None for dimension in raster.dimensions) + ) assert raster.format is None assert raster.mask is None assert raster.path is None @@ -66,12 +72,14 @@ def test_image_not_found(self, m_exists): @mock.patch("rok4.Raster.gdal.Open") @mock.patch("rok4.Raster.compute_bbox") @mock.patch("rok4.Raster.exists", side_effect=[True, False]) - def test_image(self, m_exists, m_compute_bbox, m_gdal_open, m_compute_format, - m_get_osgeo_path): + def test_image(self, m_exists, m_compute_bbox, m_gdal_open, m_compute_format, m_get_osgeo_path): """Constructor called nominally on an image without mask.""" m_compute_bbox.return_value = self.bbox - m_dataset_properties = {"RasterCount": 3, "RasterXSize": self.image_size[0], - "RasterYSize": self.image_size[1]} + m_dataset_properties = { + "RasterCount": 3, + "RasterXSize": self.image_size[0], + "RasterYSize": self.image_size[1], + } m_gdal_open.return_value = type("", (object,), m_dataset_properties) m_get_osgeo_path.return_value = self.osgeo_image_path @@ -83,11 +91,14 @@ def test_image(self, m_exists, m_compute_bbox, m_gdal_open, m_compute_format, assert raster.path == self.source_image_path assert raster.mask is None m_compute_bbox.assert_called_once() - assert (isinstance(raster.bbox, tuple) and len(raster.bbox) == 4 - and math.isclose(raster.bbox[0], self.bbox[0], rel_tol=1e-5) - and math.isclose(raster.bbox[1], self.bbox[1], rel_tol=1e-5) - and math.isclose(raster.bbox[2], self.bbox[2], rel_tol=1e-5) - and math.isclose(raster.bbox[3], self.bbox[3], rel_tol=1e-5)) + assert ( + isinstance(raster.bbox, tuple) + and len(raster.bbox) == 4 + and math.isclose(raster.bbox[0], self.bbox[0], rel_tol=1e-5) + and math.isclose(raster.bbox[1], self.bbox[1], rel_tol=1e-5) + and math.isclose(raster.bbox[2], self.bbox[2], rel_tol=1e-5) + and math.isclose(raster.bbox[3], self.bbox[3], rel_tol=1e-5) + ) assert raster.bands == 3 m_compute_format.assert_called_once() assert raster.format == ColorFormat.UINT8 @@ -99,31 +110,45 @@ def test_image(self, m_exists, m_compute_bbox, m_gdal_open, m_compute_format, @mock.patch("rok4.Raster.gdal.Open") @mock.patch("rok4.Raster.compute_bbox") @mock.patch("rok4.Raster.exists", side_effect=[True, True]) - def test_image_and_mask(self, m_exists, m_compute_bbox, m_gdal_open, m_identifydriver, - m_compute_format, m_get_osgeo_path): + def test_image_and_mask( + self, + m_exists, + m_compute_bbox, + m_gdal_open, + m_identifydriver, + m_compute_format, + m_get_osgeo_path, + ): """Constructor called nominally on an image with mask.""" m_compute_bbox.return_value = self.bbox - m_dataset_properties = {"RasterCount": 3, "RasterXSize": self.image_size[0], - "RasterYSize": self.image_size[1]} + m_dataset_properties = { + "RasterCount": 3, + "RasterXSize": self.image_size[0], + "RasterYSize": self.image_size[1], + } m_gdal_open.return_value = type("", (object,), m_dataset_properties) - m_get_osgeo_path.side_effect=[self.osgeo_image_path, self.osgeo_mask_path] + m_get_osgeo_path.side_effect = [self.osgeo_image_path, self.osgeo_mask_path] m_identifydriver.return_value = type("", (object,), {"ShortName": "GTiff"}) raster = Raster.from_file(self.source_image_path) m_exists.assert_has_calls([call(self.source_image_path), call(self.source_mask_path)]) - m_get_osgeo_path.assert_has_calls([call(self.source_image_path), - call(self.source_mask_path)]) + m_get_osgeo_path.assert_has_calls( + [call(self.source_image_path), call(self.source_mask_path)] + ) m_identifydriver.assert_called_once_with(self.osgeo_mask_path) m_gdal_open.assert_called_once_with(self.osgeo_image_path) assert raster.path == self.source_image_path assert raster.mask == self.source_mask_path m_compute_bbox.assert_called_once() - assert (isinstance(raster.bbox, tuple) and len(raster.bbox) == 4 - and math.isclose(raster.bbox[0], self.bbox[0], rel_tol=1e-5) - and math.isclose(raster.bbox[1], self.bbox[1], rel_tol=1e-5) - and math.isclose(raster.bbox[2], self.bbox[2], rel_tol=1e-5) - and math.isclose(raster.bbox[3], self.bbox[3], rel_tol=1e-5)) + assert ( + isinstance(raster.bbox, tuple) + and len(raster.bbox) == 4 + and math.isclose(raster.bbox[0], self.bbox[0], rel_tol=1e-5) + and math.isclose(raster.bbox[1], self.bbox[1], rel_tol=1e-5) + and math.isclose(raster.bbox[2], self.bbox[2], rel_tol=1e-5) + and math.isclose(raster.bbox[3], self.bbox[3], rel_tol=1e-5) + ) assert raster.bands == 3 m_compute_format.assert_called_once() assert raster.format == ColorFormat.UINT8 @@ -147,18 +172,20 @@ def test_unsupported_image_format(self, m_exists, m_gdal_open, m_get_osgeo_path) @mock.patch("rok4.Raster.gdal.IdentifyDriver") @mock.patch("rok4.Raster.gdal.Open", side_effect=None) @mock.patch("rok4.Raster.exists", side_effect=[True, True]) - def test_unsupported_mask_format(self, m_exists, m_gdal_open, m_identifydriver, - m_get_osgeo_path): + def test_unsupported_mask_format( + self, m_exists, m_gdal_open, m_identifydriver, m_get_osgeo_path + ): """Test case : Constructor called on an unsupported mask file or object.""" - m_get_osgeo_path.side_effect=[self.osgeo_image_path, self.osgeo_mask_path] + m_get_osgeo_path.side_effect = [self.osgeo_image_path, self.osgeo_mask_path] m_identifydriver.return_value = type("", (object,), {"ShortName": "JPG"}) with pytest.raises(Exception): Raster.from_file(self.source_image_path) m_exists.assert_has_calls([call(self.source_image_path), call(self.source_mask_path)]) - m_get_osgeo_path.assert_has_calls([call(self.source_image_path), - call(self.source_mask_path)]) + m_get_osgeo_path.assert_has_calls( + [call(self.source_image_path), call(self.source_mask_path)] + ) m_identifydriver.assert_called_once_with(self.osgeo_mask_path) m_gdal_open.assert_called_once_with(self.osgeo_image_path) @@ -173,24 +200,26 @@ def test_image(self): "bbox": (-5.4, 41.3, 9.8, 51.3), "dimensions": (1920, 1080), "format": ColorFormat.UINT8, - "path": "file:///path/to/image.tif" + "path": "file:///path/to/image.tif", } raster = Raster.from_parameters(**parameters) assert raster.path == parameters["path"] - assert (isinstance(raster.bbox, tuple) and len(raster.bbox) == 4 - and math.isclose(raster.bbox[0], parameters["bbox"][0], rel_tol=1e-5) - and math.isclose(raster.bbox[1], parameters["bbox"][1], rel_tol=1e-5) - and math.isclose(raster.bbox[2], parameters["bbox"][2], rel_tol=1e-5) - and math.isclose(raster.bbox[3], parameters["bbox"][3], rel_tol=1e-5)) + assert ( + isinstance(raster.bbox, tuple) + and len(raster.bbox) == 4 + and math.isclose(raster.bbox[0], parameters["bbox"][0], rel_tol=1e-5) + and math.isclose(raster.bbox[1], parameters["bbox"][1], rel_tol=1e-5) + and math.isclose(raster.bbox[2], parameters["bbox"][2], rel_tol=1e-5) + and math.isclose(raster.bbox[3], parameters["bbox"][3], rel_tol=1e-5) + ) assert raster.bands == parameters["bands"] assert raster.format == parameters["format"] assert raster.dimensions == parameters["dimensions"] assert raster.mask is None def test_image_and_mask(self): - """Parameters describing an image with mask""" parameters = { "bands": 4, @@ -198,17 +227,20 @@ def test_image_and_mask(self): "dimensions": (1920, 1080), "format": ColorFormat.UINT8, "mask": "file:///path/to/image.msk", - "path": "file:///path/to/image.tif" + "path": "file:///path/to/image.tif", } raster = Raster.from_parameters(**parameters) assert raster.path == parameters["path"] - assert (isinstance(raster.bbox, tuple) and len(raster.bbox) == 4 - and math.isclose(raster.bbox[0], parameters["bbox"][0], rel_tol=1e-5) - and math.isclose(raster.bbox[1], parameters["bbox"][1], rel_tol=1e-5) - and math.isclose(raster.bbox[2], parameters["bbox"][2], rel_tol=1e-5) - and math.isclose(raster.bbox[3], parameters["bbox"][3], rel_tol=1e-5)) + assert ( + isinstance(raster.bbox, tuple) + and len(raster.bbox) == 4 + and math.isclose(raster.bbox[0], parameters["bbox"][0], rel_tol=1e-5) + and math.isclose(raster.bbox[1], parameters["bbox"][1], rel_tol=1e-5) + and math.isclose(raster.bbox[2], parameters["bbox"][2], rel_tol=1e-5) + and math.isclose(raster.bbox[3], parameters["bbox"][3], rel_tol=1e-5) + ) assert raster.bands == parameters["bands"] assert raster.format == parameters["format"] assert raster.dimensions == parameters["dimensions"] @@ -225,10 +257,13 @@ def test_default(self): """Default property values.""" rasterset = RasterSet() - assert (isinstance(rasterset.bbox, tuple) and len(rasterset.bbox) == 4 - and all(coordinate is None for coordinate in rasterset.bbox)) - assert (isinstance(rasterset.colors, list) and not rasterset.colors) - assert (isinstance(rasterset.raster_list, list) and not rasterset.raster_list) + assert ( + isinstance(rasterset.bbox, tuple) + and len(rasterset.bbox) == 4 + and all(coordinate is None for coordinate in rasterset.bbox) + ) + assert isinstance(rasterset.colors, list) and not rasterset.colors + assert isinstance(rasterset.raster_list, list) and not rasterset.raster_list assert rasterset.srs is None @@ -244,27 +279,23 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): for n in range(0, file_number, 1): file_list.append(f"s3://test_bucket/image_{n+1}.tif") file_list_string = "\n".join(file_list) - m_open = mock_open(read_data = file_list_string) + m_open = mock_open(read_data=file_list_string) list_path = "s3://test_bucket/raster_set.list" list_local_path = "/tmp/raster_set.list" m_get_osgeo_path.return_value = list_local_path raster_list = [] colors = [] - serial_in = { - "raster_list": [], - "colors": [] - } + serial_in = {"raster_list": [], "colors": []} for n in range(0, file_number, 1): raster = MagicMock(Raster) raster.path = file_list[n] raster.bbox = ( - -0.75 + math.floor(n/3), - -1.33 + n - 3 * math.floor(n/3), - 0.25 + math.floor(n/3), - -0.33 + n - 3 * math.floor(n/3) + -0.75 + math.floor(n / 3), + -1.33 + n - 3 * math.floor(n / 3), + 0.25 + math.floor(n / 3), + -0.33 + n - 3 * math.floor(n / 3), ) - raster.format = random.choice([ColorFormat.BIT, ColorFormat.UINT8, - ColorFormat.FLOAT32]) + raster.format = random.choice([ColorFormat.BIT, ColorFormat.UINT8, ColorFormat.FLOAT32]) if raster.format == ColorFormat.BIT: raster.bands = 1 else: @@ -276,25 +307,23 @@ def test_ok_at_least_3_files(self, m_from_file, m_get_osgeo_path): color_dict = {"bands": raster.bands, "format": raster.format} if color_dict not in colors: colors.append(color_dict) - serial_in["colors"].append({"bands": raster.bands, - "format": raster.format.name}) + serial_in["colors"].append({"bands": raster.bands, "format": raster.format.name}) raster.dimensions = (5000, 5000) raster_list.append(raster) - raster_serial = {"path": raster.path, "bands": raster.bands, - "format": raster.format.name, "bbox": list(raster.bbox), - "dimensions": list(raster.dimensions)} + raster_serial = { + "path": raster.path, + "bands": raster.bands, + "format": raster.format.name, + "bbox": list(raster.bbox), + "dimensions": list(raster.dimensions), + } if raster.mask: raster_serial["mask"] = raster.mask serial_in["raster_list"].append(raster_serial) m_from_file.side_effect = raster_list srs = "EPSG:4326" serial_in["srs"] = srs - bbox = ( - -0.75, - -1.33, - 0.25 + math.floor((file_number-1)/3), - 1.67 - ) + bbox = (-0.75, -1.33, 0.25 + math.floor((file_number - 1) / 3), 1.67) serial_in["bbox"] = list(bbox) with mock.patch("rok4.Raster.open", m_open): @@ -333,7 +362,7 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): "dimensions": [5000, 5000], "format": "UINT8", "mask": "file:///path/to/images/550000_6220000.msk", - "path": "file:///path/to/images/550000_6220000.tif" + "path": "file:///path/to/images/550000_6220000.tif", }, { "bands": 3, @@ -341,7 +370,7 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): "dimensions": [5000, 5000], "format": "UINT8", "mask": "file:///path/to/images/560000_6220000.msk", - "path": "file:///path/to/images/560000_6220000.tif" + "path": "file:///path/to/images/560000_6220000.tif", }, { "bands": 3, @@ -349,10 +378,10 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): "dimensions": [5000, 5000], "format": "UINT8", "mask": "file:///path/to/images/550000_6230000.msk", - "path": "file:///path/to/images/550000_6230000.tif" - } + "path": "file:///path/to/images/550000_6230000.tif", + }, ], - "srs": "IGNF:LAMB93" + "srs": "IGNF:LAMB93", } desc_path = "file:///path/to/descriptor.json" local_path = "/path/to/descriptor.json" @@ -370,7 +399,7 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): raster_args_list.append(raster_properties) m_from_parameters.side_effect = raster_list m_get_osgeo_path.return_value = local_path - m_open = mock_open(read_data = desc_content) + m_open = mock_open(read_data=desc_content) with mock.patch("rok4.Raster.open", m_open): rasterset = RasterSet.from_descriptor(desc_path) @@ -383,15 +412,14 @@ def test_simple_ok(self, m_from_parameters, m_get_osgeo_path): for i in range(0, len(raster_args_list), 1): assert m_from_parameters.call_args_list[i] == call(**raster_args_list[i]) assert rasterset.raster_list == raster_list - assert (isinstance(rasterset.bbox, tuple) and len(rasterset.bbox) == 4) + assert isinstance(rasterset.bbox, tuple) and len(rasterset.bbox) == 4 assert isinstance(rasterset.colors, list) and rasterset.colors for i in range(0, len(serial_in["colors"]), 1): expected_color = copy.deepcopy(serial_in["colors"][i]) expected_color["format"] = ColorFormat[serial_in["colors"][i]["format"]] assert rasterset.colors[i] == expected_color serial_out = rasterset.serializable - assert (isinstance(serial_out["bbox"], list) - and len(serial_out["bbox"]) == 4) + assert isinstance(serial_out["bbox"], list) and len(serial_out["bbox"]) == 4 for i in range(0, 4, 1): assert math.isclose(rasterset.bbox[i], serial_in["bbox"][i], rel_tol=1e-5) assert math.isclose(serial_out["bbox"][i], serial_in["bbox"][i], rel_tol=1e-5) @@ -415,7 +443,7 @@ def test_ok_with_output_path(self, m_put_data_str): "dimensions": [5000, 5000], "format": "UINT8", "mask": "s3://rok4bucket/images/550000_6220000.msk", - "path": "s3://rok4bucket/images/550000_6220000.tif" + "path": "s3://rok4bucket/images/550000_6220000.tif", }, { "bands": 3, @@ -423,7 +451,7 @@ def test_ok_with_output_path(self, m_put_data_str): "dimensions": [5000, 5000], "format": "UINT8", "mask": "s3://rok4bucket/images/560000_6220000.msk", - "path": "s3://rok4bucket/images/560000_6220000.tif" + "path": "s3://rok4bucket/images/560000_6220000.tif", }, { "bands": 3, @@ -431,10 +459,10 @@ def test_ok_with_output_path(self, m_put_data_str): "dimensions": [5000, 5000], "format": "UINT8", "mask": "s3://rok4bucket/images/550000_6230000.msk", - "path": "s3://rok4bucket/images/550000_6230000.tif" - } + "path": "s3://rok4bucket/images/550000_6230000.tif", + }, ], - "srs": "IGNF:LAMB93" + "srs": "IGNF:LAMB93", } content = json.dumps(serial_in, sort_keys=True) path = "s3://rok4bucket/dst_descriptor.json" @@ -443,8 +471,9 @@ def test_ok_with_output_path(self, m_put_data_str): rasterset.srs = serial_in["srs"] rasterset.colors = [] for color_dict in serial_in["colors"]: - rasterset.colors.append({"bands": color_dict["bands"], - "format": ColorFormat[color_dict["format"]]}) + rasterset.colors.append( + {"bands": color_dict["bands"], "format": ColorFormat[color_dict["format"]]} + ) rasterset.raster_list = [] for raster_dict in serial_in["raster_list"]: raster_args = copy.deepcopy(raster_dict) @@ -460,7 +489,6 @@ def test_ok_with_output_path(self, m_put_data_str): m_put_data_str.assert_called_once_with(content, path) - @mock.patch("rok4.Raster.print") def test_ok_no_output_path(self, m_print): serial_in = { @@ -473,7 +501,7 @@ def test_ok_no_output_path(self, m_print): "dimensions": [5000, 5000], "format": "UINT8", "mask": "s3://rok4bucket/images/550000_6220000.msk", - "path": "s3://rok4bucket/images/550000_6220000.tif" + "path": "s3://rok4bucket/images/550000_6220000.tif", }, { "bands": 3, @@ -481,7 +509,7 @@ def test_ok_no_output_path(self, m_print): "dimensions": [5000, 5000], "format": "UINT8", "mask": "s3://rok4bucket/images/560000_6220000.msk", - "path": "s3://rok4bucket/images/560000_6220000.tif" + "path": "s3://rok4bucket/images/560000_6220000.tif", }, { "bands": 3, @@ -489,10 +517,10 @@ def test_ok_no_output_path(self, m_print): "dimensions": [5000, 5000], "format": "UINT8", "mask": "s3://rok4bucket/images/550000_6230000.msk", - "path": "s3://rok4bucket/images/550000_6230000.tif" - } + "path": "s3://rok4bucket/images/550000_6230000.tif", + }, ], - "srs": "IGNF:LAMB93" + "srs": "IGNF:LAMB93", } content = json.dumps(serial_in, sort_keys=True) rasterset = RasterSet() @@ -500,8 +528,9 @@ def test_ok_no_output_path(self, m_print): rasterset.srs = serial_in["srs"] rasterset.colors = [] for color_dict in serial_in["colors"]: - rasterset.colors.append({"bands": color_dict["bands"], - "format": ColorFormat[color_dict["format"]]}) + rasterset.colors.append( + {"bands": color_dict["bands"], "format": ColorFormat[color_dict["format"]]} + ) rasterset.raster_list = [] for raster_dict in serial_in["raster_list"]: raster_args = copy.deepcopy(raster_dict) @@ -516,4 +545,3 @@ def test_ok_no_output_path(self, m_print): assert False, f"Writing RasterSet's descriptor raises an exception: {exc}" m_print.assert_called_once_with(content) - diff --git a/tests/test_Storage.py b/tests/test_Storage.py index 69d2ece..0bd45ac 100644 --- a/tests/test_Storage.py +++ b/tests/test_Storage.py @@ -10,32 +10,48 @@ from unittest import mock from unittest.mock import * + @mock.patch.dict(os.environ, {}, clear=True) @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_hash_file_ok(mock_file): try: md5 = hash_file("/path/to/file.ext") - mock_file.assert_called_with("/path/to/file.ext", 'rb') + mock_file.assert_called_with("/path/to/file.ext", "rb") assert md5 == "8d777f385d3dfec8815d20f7496026dc" except Exception as exc: assert False, f"FILE md5 sum raises an exception: {exc}" + @mock.patch.dict(os.environ, {}, clear=True) def test_get_infos_from_path(): assert (StorageType.S3, "toto/titi", "toto", "titi") == get_infos_from_path("s3://toto/titi") - assert (StorageType.FILE, "/toto/titi/tutu.json", "/toto/titi", "tutu.json") == get_infos_from_path("file:///toto/titi/tutu.json") - assert (StorageType.CEPH, "toto/titi/tutu", "toto", "titi/tutu") == get_infos_from_path("ceph://toto/titi/tutu") - assert (StorageType.FILE, "wrong://toto/titi", "wrong://toto", "titi") == get_infos_from_path("wrong://toto/titi") + assert ( + StorageType.FILE, + "/toto/titi/tutu.json", + "/toto/titi", + "tutu.json", + ) == get_infos_from_path("file:///toto/titi/tutu.json") + assert (StorageType.CEPH, "toto/titi/tutu", "toto", "titi/tutu") == get_infos_from_path( + "ceph://toto/titi/tutu" + ) + assert (StorageType.FILE, "wrong://toto/titi", "wrong://toto", "titi") == get_infos_from_path( + "wrong://toto/titi" + ) @mock.patch.dict(os.environ, {}, clear=True) def test_get_path_from_infos(): assert get_path_from_infos(StorageType.S3, "toto", "toto/titi") == "s3://toto/toto/titi" - assert get_path_from_infos(StorageType.FILE, "/toto/titi", "tutu.json") == "file:///toto/titi/tutu.json" + assert ( + get_path_from_infos(StorageType.FILE, "/toto/titi", "tutu.json") + == "file:///toto/titi/tutu.json" + ) assert get_path_from_infos(StorageType.CEPH, "toto", "titi/tutu") == "ceph://toto/titi/tutu" + ############ get_data_str + @mock.patch.dict(os.environ, {}, clear=True) def test_s3_missing_env(): with pytest.raises(MissingEnvironmentError): @@ -47,16 +63,24 @@ def test_ceph_missing_env(): with pytest.raises(MissingEnvironmentError): data = get_data_str("ceph://bucket/path/to/object") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "a,b", "ROK4_S3_SECRETKEY": "b,c", "ROK4_S3_KEY": "c,d,e"}, clear=True) + +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "a,b", "ROK4_S3_SECRETKEY": "b,c", "ROK4_S3_KEY": "c,d,e"}, + clear=True, +) def test_s3_invalid_envs(): with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "a", "ROK4_S3_SECRETKEY": "b", "ROK4_S3_KEY": "c"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') + +@mock.patch.dict( + os.environ, {"ROK4_S3_URL": "a", "ROK4_S3_SECRETKEY": "b", "ROK4_S3_KEY": "c"}, clear=True +) +@mock.patch("rok4.Storage.boto3.client") def test_s3_invalid_endpoint(mocked_s3_client): s3_instance = MagicMock() - mocked_s3_client.side_effect = Exception('Invalid URL') + mocked_s3_client.side_effect = Exception("Invalid URL") with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") @@ -80,27 +104,34 @@ def test_file_read_ok(mock_file): except Exception as exc: assert False, f"FILE read raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') + +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_s3_read_nok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() - s3_instance.get_object.side_effect = Exception('Bucket or object not found') + s3_instance.get_object.side_effect = Exception("Bucket or object not found") mocked_s3_client.return_value = s3_instance with pytest.raises(StorageError): data = get_data_str("s3://bucket/path/to/object") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -def test_s3_read_ok(mocked_s3_client): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +def test_s3_read_ok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() s3_body = MagicMock() s3_body.read.return_value = b"data" - s3_instance.get_object.return_value = { - "Body": s3_body - } + s3_instance.get_object.return_value = {"Body": s3_body} mocked_s3_client.return_value = s3_instance try: @@ -110,10 +141,13 @@ def test_s3_read_ok(mocked_s3_client): assert False, f"S3 read raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") def test_ceph_read_ok(mocked_rados_client): - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.stat.return_value = (4, "date") @@ -131,23 +165,30 @@ def test_ceph_read_ok(mocked_rados_client): ############ put_data_str -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -def test_s3_write_nok(mocked_s3_client): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +def test_s3_write_nok(mocked_s3_client): disconnect_s3_clients() s3_instance = MagicMock() - s3_instance.put_object.side_effect = Exception('Cannot write S3 object') + s3_instance.put_object.side_effect = Exception("Cannot write S3 object") mocked_s3_client.return_value = s3_instance with pytest.raises(StorageError): put_data_str("data", "s3://bucket/path/to/object") -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_s3_write_ok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.put_object.return_value = None @@ -158,10 +199,13 @@ def test_s3_write_ok(mocked_s3_client): assert False, f"S3 write raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") def test_ceph_write_ok(mocked_rados_client): - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write_full.return_value = None @@ -174,12 +218,14 @@ def test_ceph_write_ok(mocked_rados_client): except Exception as exc: assert False, f"CEPH write raises an exception: {exc}" + ############ copy + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('os.makedirs', return_value=None) -@mock.patch('rok4.Storage.copyfile', return_value=None) -@mock.patch('rok4.Storage.hash_file', return_value="toto") +@mock.patch("os.makedirs", return_value=None) +@mock.patch("rok4.Storage.copyfile", return_value=None) +@mock.patch("rok4.Storage.hash_file", return_value="toto") def test_copy_file_file_ok(mock_hash_file, mock_copyfile, mock_makedirs): try: copy("file:///path/to/source.ext", "file:///path/to/destination.ext", "toto") @@ -190,19 +236,21 @@ def test_copy_file_file_ok(mock_hash_file, mock_copyfile, mock_makedirs): assert False, f"FILE -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -@mock.patch('os.makedirs', return_value=None) -@mock.patch('rok4.Storage.hash_file', return_value="toto") +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +@mock.patch("os.makedirs", return_value=None) +@mock.patch("rok4.Storage.hash_file", return_value="toto") def test_copy_s3_file_ok(mock_hash_file, mock_makedirs, mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.download_file.return_value = None mocked_s3_client.return_value = s3_instance try: - copy("s3://bucket/source.ext", "file:///path/to/destination.ext", "toto") mock_hash_file.assert_called_once_with("/path/to/destination.ext") mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) @@ -210,15 +258,18 @@ def test_copy_s3_file_ok(mock_hash_file, mock_makedirs, mocked_s3_client): assert False, f"S3 -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') -@mock.patch('os.makedirs', return_value=None) -@mock.patch('rok4.Storage.hash_file', return_value="toto") +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") +@mock.patch("os.makedirs", return_value=None) +@mock.patch("rok4.Storage.hash_file", return_value="toto") def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() - s3_instance.download_file.side_effect = Exception('Cannot download S3 object') + s3_instance.download_file.side_effect = Exception("Cannot download S3 object") mocked_s3_client.return_value = s3_instance with pytest.raises(StorageError): @@ -226,10 +277,13 @@ def test_copy_s3_file_nok(mock_hash_file, mock_makedirs, mocked_s3_client): mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_copy_file_s3_ok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.upload_file.return_value = None @@ -242,10 +296,13 @@ def test_copy_file_s3_ok(mocked_s3_client): assert False, f"FILE -> S3 copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_copy_s3_s3_ok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.copy.return_value = None @@ -258,10 +315,13 @@ def test_copy_s3_s3_ok(mocked_s3_client): assert False, f"S3 -> S3 copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_copy_s3_s3_intercluster_ok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.copy.return_value = None @@ -274,10 +334,13 @@ def test_copy_s3_s3_intercluster_ok(mocked_s3_client): assert False, f"S3 -> S3 inter cluster copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_copy_s3_s3_intercluster_nok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.copy.return_value = None @@ -287,12 +350,16 @@ def test_copy_s3_s3_intercluster_nok(mocked_s3_client): with pytest.raises(StorageError): copy("s3://bucket@a/source.ext", "s3://bucket@c/destination.ext", "toto") -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -@mock.patch('os.makedirs', return_value=None) + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("os.makedirs", return_value=None) @patch("builtins.open", new_callable=mock_open) def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.read.return_value = b"data" @@ -301,16 +368,24 @@ def test_copy_ceph_file_ok(mock_file, mock_makedirs, mocked_rados_client): mocked_rados_client.return_value = ceph_instance try: - copy("ceph://pool/source.ext", "file:///path/to/destination.ext", "8d777f385d3dfec8815d20f7496026dc") + copy( + "ceph://pool/source.ext", + "file:///path/to/destination.ext", + "8d777f385d3dfec8815d20f7496026dc", + ) mock_makedirs.assert_called_once_with("/path/to", exist_ok=True) except Exception as exc: assert False, f"CEPH -> FILE copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_copy_file_ceph_ok(mock_file, mocked_rados_client): - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write.return_value = None @@ -319,15 +394,23 @@ def test_copy_file_ceph_ok(mock_file, mocked_rados_client): mocked_rados_client.return_value = ceph_instance try: - copy("file:///path/to/source.ext", "ceph://pool/destination.ext", "8d777f385d3dfec8815d20f7496026dc") + copy( + "file:///path/to/source.ext", + "ceph://pool/destination.ext", + "8d777f385d3dfec8815d20f7496026dc", + ) except Exception as exc: assert False, f"FILE -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') + +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_copy_ceph_ceph_ok(mock_file, mocked_rados_client): - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.read.return_value = b"data" @@ -337,17 +420,31 @@ def test_copy_ceph_ceph_ok(mock_file, mocked_rados_client): mocked_rados_client.return_value = ceph_instance try: - copy("ceph://pool1/source.ext", "ceph://pool2/destination.ext", "8d777f385d3dfec8815d20f7496026dc") + copy( + "ceph://pool1/source.ext", + "ceph://pool2/destination.ext", + "8d777f385d3dfec8815d20f7496026dc", + ) except Exception as exc: assert False, f"CEPH -> CEPH copy raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c", "ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + { + "ROK4_CEPH_CONFFILE": "a", + "ROK4_CEPH_CLUSTERNAME": "b", + "ROK4_CEPH_USERNAME": "c", + "ROK4_S3_URL": "https://a,https://b", + "ROK4_S3_SECRETKEY": "a,b", + "ROK4_S3_KEY": "a,b", + }, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +@mock.patch("rok4.Storage.boto3.client") @patch("builtins.open", new_callable=mock_open, read_data=b"data") def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): - disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.read.return_value = b"data" @@ -355,7 +452,6 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): ceph_instance.open_ioctx.return_value = ioctx_instance mocked_rados_client.return_value = ceph_instance - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.upload_file.return_value = None @@ -363,24 +459,30 @@ def test_copy_ceph_s3_ok(mock_file, mocked_s3_client, mocked_rados_client): mocked_s3_client.return_value = s3_instance try: - copy("ceph://pool1/source.ext", "s3://bucket/destination.ext", "8d777f385d3dfec8815d20f7496026dc") + copy( + "ceph://pool1/source.ext", + "s3://bucket/destination.ext", + "8d777f385d3dfec8815d20f7496026dc", + ) except Exception as exc: assert False, f"CEPH -> S3 copy raises an exception: {exc}" - ############ link + def test_link_type_nok(): with pytest.raises(StorageError): link("ceph://pool1/target.ext", "file:///path/to/link.ext") + def test_link_hard_nok(): with pytest.raises(StorageError): link("ceph://pool1/source.ext", "ceph://pool2/destination.ext", True) + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('os.symlink', return_value=None) +@mock.patch("os.symlink", return_value=None) def test_link_file_ok(mock_link): try: link("file:///path/to/target.ext", "file:///path/to/link.ext") @@ -390,7 +492,7 @@ def test_link_file_ok(mock_link): @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('os.link', return_value=None) +@mock.patch("os.link", return_value=None) def test_hlink_file_ok(mock_link): try: link("file:///path/to/target.ext", "file:///path/to/link.ext", True) @@ -398,10 +500,14 @@ def test_hlink_file_ok(mock_link): except Exception as exc: assert False, f"FILE hard link raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -def test_link_ceph_ok(mocked_rados_client): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +def test_link_ceph_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.write.return_value = None @@ -415,10 +521,13 @@ def test_link_ceph_ok(mocked_rados_client): assert False, f"CEPH link raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_link_s3_ok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.put_object.return_value = None @@ -430,10 +539,13 @@ def test_link_s3_ok(mocked_s3_client): assert False, f"S3 link raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_link_s3_nok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.put_object.return_value = None @@ -442,10 +554,12 @@ def test_link_s3_nok(mocked_s3_client): with pytest.raises(StorageError): link("s3://bucket1@a/target.ext", "s3://bucket2@b/link.ext") + ############ get_size + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('os.stat') +@mock.patch("os.stat") def test_size_file_ok(mock_stat): mock_stat.return_value.st_size = 12 try: @@ -454,10 +568,14 @@ def test_size_file_ok(mock_stat): except Exception as exc: assert False, f"FILE size raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -def test_size_ceph_ok(mocked_rados_client): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +def test_size_ceph_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.stat.return_value = (12, "date") @@ -472,10 +590,13 @@ def test_size_ceph_ok(mocked_rados_client): assert False, f"CEPH size raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_size_s3_ok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.head_object.return_value = {"ContentLength": 12} @@ -490,8 +611,9 @@ def test_size_s3_ok(mocked_s3_client): ############ exists + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('os.path.exists', return_value=True) +@mock.patch("os.path.exists", return_value=True) def test_exists_file_ok(mock_exists): try: assert exists("file:///path/to/file.ext") @@ -504,10 +626,14 @@ def test_exists_file_ok(mock_exists): except Exception as exc: assert False, f"FILE not exists raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -def test_exists_ceph_ok(mocked_rados_client): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +def test_exists_ceph_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.stat.return_value = None @@ -527,10 +653,13 @@ def test_exists_ceph_ok(mocked_rados_client): assert False, f"CEPH not exists raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_exists_s3_ok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.head_object.return_value = None @@ -541,7 +670,9 @@ def test_exists_s3_ok(mocked_s3_client): except Exception as exc: assert False, f"S3 exists raises an exception: {exc}" - s3_instance.head_object.side_effect = botocore.exceptions.ClientError(operation_name='InvalidKeyPair.Duplicate', error_response={"Error": {"Code": "404"}}) + s3_instance.head_object.side_effect = botocore.exceptions.ClientError( + operation_name="InvalidKeyPair.Duplicate", error_response={"Error": {"Code": "404"}} + ) try: assert not exists("s3://bucket/object.ext") except Exception as exc: @@ -550,8 +681,9 @@ def test_exists_s3_ok(mocked_s3_client): ############ remove + @mock.patch.dict(os.environ, {}, clear=True) -@mock.patch('os.remove') +@mock.patch("os.remove") def test_remove_file_ok(mock_remove): mock_remove.return_value = None try: @@ -565,10 +697,14 @@ def test_remove_file_ok(mock_remove): except Exception as exc: assert False, f"FILE deletion (not found) raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, clear=True) -@mock.patch('rok4.Storage.rados.Rados') -def test_remove_ceph_ok(mocked_rados_client): +@mock.patch.dict( + os.environ, + {"ROK4_CEPH_CONFFILE": "a", "ROK4_CEPH_CLUSTERNAME": "b", "ROK4_CEPH_USERNAME": "c"}, + clear=True, +) +@mock.patch("rok4.Storage.rados.Rados") +def test_remove_ceph_ok(mocked_rados_client): disconnect_ceph_clients() ioctx_instance = MagicMock() ioctx_instance.remove_object.return_value = None @@ -588,10 +724,13 @@ def test_remove_ceph_ok(mocked_rados_client): assert False, f"CEPH deletion (not found) raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -@mock.patch('rok4.Storage.boto3.client') +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +@mock.patch("rok4.Storage.boto3.client") def test_remove_s3_ok(mocked_s3_client): - disconnect_s3_clients() s3_instance = MagicMock() s3_instance.delete_object.return_value = None @@ -605,6 +744,7 @@ def test_remove_s3_ok(mocked_s3_client): ############ get_osgeo_path + @mock.patch.dict(os.environ, {}, clear=True) def test_get_osgeo_path_file_ok(): try: @@ -613,9 +753,13 @@ def test_get_osgeo_path_file_ok(): except Exception as exc: assert False, f"FILE osgeo path raises an exception: {exc}" -@mock.patch.dict(os.environ, {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, clear=True) -def test_get_osgeo_path_s3_ok(): +@mock.patch.dict( + os.environ, + {"ROK4_S3_URL": "https://a,https://b", "ROK4_S3_SECRETKEY": "a,b", "ROK4_S3_KEY": "a,b"}, + clear=True, +) +def test_get_osgeo_path_s3_ok(): disconnect_s3_clients() try: diff --git a/tests/test_TileMatrixSet.py b/tests/test_TileMatrixSet.py index 7f41688..4750f50 100644 --- a/tests/test_TileMatrixSet.py +++ b/tests/test_TileMatrixSet.py @@ -6,125 +6,195 @@ from unittest.mock import * from unittest import mock + @mock.patch.dict(os.environ, {}, clear=True) def test_missing_env(): with pytest.raises(MissingEnvironmentError): tms = TileMatrixSet("tms") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', side_effect=StorageError('FILE', 'Not found')) +@mock.patch("rok4.TileMatrixSet.get_data_str", side_effect=StorageError("FILE", "Not found")) def test_wrong_file(mocked_get_data_str): with pytest.raises(StorageError): tms = TileMatrixSet("tms") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', +) def test_bad_json(mocked_get_data_str): with pytest.raises(FormatError) as exc: tms = TileMatrixSet("tms") - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"]}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"]}', +) def test_missing_id(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: tms = TileMatrixSet("tms") assert str(exc.value) == "Missing attribute 'id' in 'file:///path/to/tms.json'" - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM"}', +) def test_missing_crs(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: tms = TileMatrixSet("tms") assert str(exc.value) == "Missing attribute 'crs' in 'file:///path/to/tms.json'" - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"crs":"epsg:123456","orderedAxes":["X","Y"],"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"crs":"epsg:123456","orderedAxes":["X","Y"],"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM"}', +) def test_wrong_crs(mocked_get_data_str): with pytest.raises(Exception) as exc: tms = TileMatrixSet("tms") - assert str(exc.value) == "Wrong attribute 'crs' ('epsg:123456') in 'file:///path/to/tms.json', not recognize by OSR" - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + assert ( + str(exc.value) + == "Wrong attribute 'crs' ('epsg:123456') in 'file:///path/to/tms.json', not recognize by OSR" + ) + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"crs":"epsg:4326","orderedAxes":["Lat","Lon"],"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"crs":"epsg:4326","orderedAxes":["Lat","Lon"],"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"id":"PM"}', +) def test_wrong_axes_order(mocked_get_data_str): with pytest.raises(Exception) as exc: tms = TileMatrixSet("tms") - assert str(exc.value) == "TMS 'file:///path/to/tms.json' own invalid axes order : only X/Y or Lon/Lat are handled" - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + assert ( + str(exc.value) + == "TMS 'file:///path/to/tms.json' own invalid axes order : only X/Y or Lon/Lat are handled" + ) + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', +) def test_missing_levels(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: tms = TileMatrixSet("tms") assert str(exc.value) == "Missing attribute 'tileMatrices' in 'file:///path/to/tms.json'" - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"tileMatrices":[],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"tileMatrices":[],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', +) def test_no_levels(mocked_get_data_str): with pytest.raises(Exception) as exc: tms = TileMatrixSet("tms") assert str(exc.value) == "TMS 'file:///path/to/tms.json' has no level" - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"tileMatrices":[{"tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM","crs":"EPSG:3857"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"tileMatrices":[{"tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"orderedAxes":["X","Y"],"id":"PM","crs":"EPSG:3857"}', +) def test_wrong_level(mocked_get_data_str): with pytest.raises(MissingAttributeError) as exc: tms = TileMatrixSet("tms") assert str(exc.value) == "Missing attribute tileMatrices[].'id' in 'file:///path/to/tms.json'" - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"tileMatrices":[{"id":"level_0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"tileMatrices":[{"id":"level_0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', +) def test_wrong_level_id(mocked_get_data_str): with pytest.raises(Exception) as exc: tms = TileMatrixSet("tms") - assert str(exc.value) == "TMS file:///path/to/tms.json owns a level whom id contains an underscore (level_0)" - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + assert ( + str(exc.value) + == "TMS file:///path/to/tms.json owns a level whom id contains an underscore (level_0)" + ) + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"tileMatrices":[{"id":"0","tileWidth":256,"scaleDenominator":559082264.028718,"matrixWidth":1,"cellSize":156543.033928041,"matrixHeight":1,"tileHeight":256,"pointOfOrigin":[-20037508.3427892,20037508.3427892]}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', +) def test_ok(mocked_get_data_str): try: tms = TileMatrixSet("tms") assert tms.get_level("0") is not None assert tms.get_level("4") is None - mocked_get_data_str.assert_called_once_with('file:///path/to/tms.json') + mocked_get_data_str.assert_called_once_with("file:///path/to/tms.json") except Exception as exc: assert False, f"'TileMatrixSet creation raises an exception: {exc}" + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"tileMatrices":[{"id":"17","cellSize":1.19432856695588,"matrixHeight":131072,"pointOfOrigin":[-20037508.3427892,20037508.3427892],"tileHeight":256,"tileWidth":256,"scaleDenominator":4265.45916769957,"matrixWidth":131072}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"tileMatrices":[{"id":"17","cellSize":1.19432856695588,"matrixHeight":131072,"pointOfOrigin":[-20037508.3427892,20037508.3427892],"tileHeight":256,"tileWidth":256,"scaleDenominator":4265.45916769957,"matrixWidth":131072}],"crs":"EPSG:3857","orderedAxes":["X","Y"],"id":"PM"}', +) def test_pm_conversions(mocked_get_data_str): try: tms = TileMatrixSet("tms") tm = tms.get_level("17") assert tm.x_to_column(670654.2832369965) == 67729 assert tm.y_to_row(5980575.503117723) == 45975 - assert tm.tile_to_bbox(67728, 45975) == (670199.864004489, 5980433.093032133, 670505.6121176295, 5980738.841145273) - assert tm.bbox_to_tiles((670034.4267107458, 5980565.948489188,670649.5059227281, 5980936.190344945)) == (67727, 45974,67729, 45975) + assert tm.tile_to_bbox(67728, 45975) == ( + 670199.864004489, + 5980433.093032133, + 670505.6121176295, + 5980738.841145273, + ) + assert tm.bbox_to_tiles( + (670034.4267107458, 5980565.948489188, 670649.5059227281, 5980936.190344945) + ) == (67727, 45974, 67729, 45975) assert tm.point_to_indices(670654.2832369965, 5980575.503117723) == (67729, 45975, 124, 136) except Exception as exc: assert False, f"'TileMatrixSet creation raises an exception: {exc}" + @mock.patch.dict(os.environ, {"ROK4_TMS_DIRECTORY": "file:///path/to"}, clear=True) -@mock.patch('rok4.TileMatrixSet.get_data_str', return_value='{"crs":"EPSG:4326","tileMatrices":[{"tileWidth":256,"scaleDenominator":1066.36480348451,"matrixWidth":524288,"cellSize":2.68220901489258e-06,"matrixHeight":262144,"pointOfOrigin":[-180,90],"tileHeight":256,"id":"18"}],"orderedAxes":["Lon","Lat"],"id":"4326"}') +@mock.patch( + "rok4.TileMatrixSet.get_data_str", + return_value='{"crs":"EPSG:4326","tileMatrices":[{"tileWidth":256,"scaleDenominator":1066.36480348451,"matrixWidth":524288,"cellSize":2.68220901489258e-06,"matrixHeight":262144,"pointOfOrigin":[-180,90],"tileHeight":256,"id":"18"}],"orderedAxes":["Lon","Lat"],"id":"4326"}', +) def test_4326_conversions(mocked_get_data_str): try: tms = TileMatrixSet("tms") tm = tms.get_level("18") assert tm.x_to_column(5) == 269425 assert tm.y_to_row(45) == 65535 - assert tm.tile_to_bbox(269425, 65535) == (44.99999999999997, 4.999465942382926, 45.000686645507784, 5.000152587890739) - assert tm.bbox_to_tiles((45,5,48,6)) == (269425,61166,270882,65535) - assert tm.point_to_indices(45,5) == (269425, 65535, 199, 255) + assert tm.tile_to_bbox(269425, 65535) == ( + 44.99999999999997, + 4.999465942382926, + 45.000686645507784, + 5.000152587890739, + ) + assert tm.bbox_to_tiles((45, 5, 48, 6)) == (269425, 61166, 270882, 65535) + assert tm.point_to_indices(45, 5) == (269425, 65535, 199, 255) except Exception as exc: assert False, f"'TileMatrixSet creation raises an exception: {exc}" diff --git a/tests/test_Utils.py b/tests/test_Utils.py index bc7acc9..cdba571 100644 --- a/tests/test_Utils.py +++ b/tests/test_Utils.py @@ -18,6 +18,7 @@ def test_srs_to_spatialreference_ignf_ok(): except Exception as exc: assert False, f"SpatialReference creation raises an exception: {exc}" + def test_srs_to_spatialreference_epsg_ok(): try: sr = srs_to_spatialreference("EPSG:3857") @@ -25,21 +26,25 @@ def test_srs_to_spatialreference_epsg_ok(): except Exception as exc: assert False, f"SpatialReference creation raises an exception: {exc}" + def test_srs_to_spatialreference_ignf_nok(): with pytest.raises(Exception): sr = srs_to_spatialreference("IGNF:TOTO") + def test_srs_to_spatialreference_epsg_nok(): with pytest.raises(Exception): sr = srs_to_spatialreference("EPSG:123456") + def test_bbox_to_geometry_ok(): try: - geom = bbox_to_geometry((0,0,5,10)) + geom = bbox_to_geometry((0, 0, 5, 10)) assert geom.Area() == 50 except Exception as exc: assert False, f"Geometry creation from bbox raises an exception: {exc}" + def test_reproject_bbox_ok(): try: bbox = reproject_bbox((-90, -180, 90, 180), "EPSG:4326", "EPSG:3857") @@ -49,18 +54,19 @@ def test_reproject_bbox_ok(): except Exception as exc: assert False, f"Bbox reprojection raises an exception: {exc}" + def test_reproject_point_ok(): try: sr_4326 = srs_to_spatialreference("EPSG:4326") sr_3857 = srs_to_spatialreference("EPSG:3857") sr_ignf = srs_to_spatialreference("IGNF:WGS84G") - x,y = reproject_point((43, 3), sr_4326, sr_3857) + x, y = reproject_point((43, 3), sr_4326, sr_3857) assert math.isclose(x, 333958.4723798207, rel_tol=1e-5) assert math.isclose(y, 5311971.846945471, rel_tol=1e-5) - x,y = reproject_point((43, 3), sr_4326, sr_ignf) - assert (x,y) == (3, 43) + x, y = reproject_point((43, 3), sr_4326, sr_ignf) + assert (x, y) == (3, 43) bbox = reproject_bbox((43, 3, 44, 4), "EPSG:4326", "IGNF:WGS84G") assert bbox[0] == 3 @@ -70,15 +76,13 @@ def test_reproject_point_ok(): # Tests for the rok4.Utils.compute_bbox function. + def test_compute_bbox_epsg_3857_ok(): try: mocked_datasource = MagicMock(gdal.Dataset) random.seed() - image_size = ( - random.randint(1, 1000), - random.randint(1, 1000) - ) + image_size = (random.randint(1, 1000), random.randint(1, 1000)) mocked_datasource.RasterXSize = image_size[0] mocked_datasource.RasterYSize = image_size[1] @@ -88,28 +92,27 @@ def test_compute_bbox_epsg_3857_ok(): random.uniform(-10000, 10000), random.uniform(-10000000.0, 10000000.0), random.uniform(-10000, 10000), - random.uniform(-10000, 10000) + random.uniform(-10000, 10000), ) - mocked_datasource.GetGeoTransform = Mock(return_value = transform_tuple) + mocked_datasource.GetGeoTransform = Mock(return_value=transform_tuple) mocked_spatial_ref = MagicMock(osr.SpatialReference) - mocked_spatial_ref.GetDataAxisToSRSAxisMapping = Mock(return_value = [1,2]) - mocked_datasource.GetSpatialRef = Mock(return_value = mocked_spatial_ref) + mocked_spatial_ref.GetDataAxisToSRSAxisMapping = Mock(return_value=[1, 2]) + mocked_datasource.GetSpatialRef = Mock(return_value=mocked_spatial_ref) x_range = ( transform_tuple[0], - transform_tuple[0] + image_size[0] * transform_tuple[1] + image_size[1] * transform_tuple[2] + transform_tuple[0] + + image_size[0] * transform_tuple[1] + + image_size[1] * transform_tuple[2], ) y_range = ( transform_tuple[3], - transform_tuple[3] + image_size[0] * transform_tuple[4] + image_size[1] * transform_tuple[5] + transform_tuple[3] + + image_size[0] * transform_tuple[4] + + image_size[1] * transform_tuple[5], ) - expected = ( - min(x_range), - min(y_range), - max(x_range), - max(y_range) - ) + expected = (min(x_range), min(y_range), max(x_range), max(y_range)) result = compute_bbox(mocked_datasource) assert math.isclose(result[0], expected[0], rel_tol=1e-5) assert math.isclose(result[1], expected[1], rel_tol=1e-5) @@ -126,10 +129,7 @@ def test_compute_bbox_epsg_4326_ok(): mocked_datasource = MagicMock(gdal.Dataset) random.seed() - image_size = ( - random.randint(1, 1000), - random.randint(1, 1000) - ) + image_size = (random.randint(1, 1000), random.randint(1, 1000)) mocked_datasource.RasterXSize = image_size[0] mocked_datasource.RasterYSize = image_size[1] @@ -139,28 +139,27 @@ def test_compute_bbox_epsg_4326_ok(): random.uniform(-10000, 10000), random.uniform(-10000000.0, 10000000.0), random.uniform(-10000, 10000), - random.uniform(-10000, 10000) + random.uniform(-10000, 10000), ) - mocked_datasource.GetGeoTransform = Mock(return_value = transform_tuple) + mocked_datasource.GetGeoTransform = Mock(return_value=transform_tuple) mocked_spatial_ref = MagicMock(osr.SpatialReference) - mocked_spatial_ref.GetDataAxisToSRSAxisMapping = Mock(return_value = [2,1]) - mocked_datasource.GetSpatialRef = Mock(return_value = mocked_spatial_ref) + mocked_spatial_ref.GetDataAxisToSRSAxisMapping = Mock(return_value=[2, 1]) + mocked_datasource.GetSpatialRef = Mock(return_value=mocked_spatial_ref) x_range = ( transform_tuple[0], - transform_tuple[0] + image_size[0] * transform_tuple[1] + image_size[1] * transform_tuple[2] + transform_tuple[0] + + image_size[0] * transform_tuple[1] + + image_size[1] * transform_tuple[2], ) y_range = ( transform_tuple[3], - transform_tuple[3] + image_size[0] * transform_tuple[4] + image_size[1] * transform_tuple[5] + transform_tuple[3] + + image_size[0] * transform_tuple[4] + + image_size[1] * transform_tuple[5], ) - expected = ( - min(y_range), - min(x_range), - max(y_range), - max(x_range) - ) + expected = (min(y_range), min(x_range), max(y_range), max(x_range)) result = compute_bbox(mocked_datasource) assert math.isclose(result[0], expected[0], rel_tol=1e-5) assert math.isclose(result[1], expected[1], rel_tol=1e-5) @@ -177,10 +176,7 @@ def test_compute_bbox_no_srs_ok(): mocked_datasource = MagicMock(gdal.Dataset) random.seed() - image_size = ( - random.randint(1, 1000), - random.randint(1, 1000) - ) + image_size = (random.randint(1, 1000), random.randint(1, 1000)) mocked_datasource.RasterXSize = image_size[0] mocked_datasource.RasterYSize = image_size[1] @@ -190,26 +186,25 @@ def test_compute_bbox_no_srs_ok(): random.uniform(-10000, 10000), random.uniform(-10000000.0, 10000000.0), random.uniform(-10000, 10000), - random.uniform(-10000, 10000) + random.uniform(-10000, 10000), ) - mocked_datasource.GetGeoTransform = Mock(return_value = transform_tuple) - mocked_datasource.GetSpatialRef = Mock(return_value = None) + mocked_datasource.GetGeoTransform = Mock(return_value=transform_tuple) + mocked_datasource.GetSpatialRef = Mock(return_value=None) x_range = ( transform_tuple[0], - transform_tuple[0] + image_size[0] * transform_tuple[1] + image_size[1] * transform_tuple[2] + transform_tuple[0] + + image_size[0] * transform_tuple[1] + + image_size[1] * transform_tuple[2], ) y_range = ( transform_tuple[3], - transform_tuple[3] + image_size[0] * transform_tuple[4] + image_size[1] * transform_tuple[5] + transform_tuple[3] + + image_size[0] * transform_tuple[4] + + image_size[1] * transform_tuple[5], ) - expected = ( - min(x_range), - min(y_range), - max(x_range), - max(y_range) - ) + expected = (min(x_range), min(y_range), max(x_range), max(y_range)) result = compute_bbox(mocked_datasource) assert math.isclose(result[0], expected[0], rel_tol=1e-5) assert math.isclose(result[1], expected[1], rel_tol=1e-5) @@ -222,11 +217,14 @@ def test_compute_bbox_no_srs_ok(): # Tests for the rok4.Utils.compute_format function. -@mock.patch('rok4.Utils.gdal.Info') -@mock.patch('rok4.Utils.gdal.GetColorInterpretationName', return_value="Palette") -@mock.patch('rok4.Utils.gdal.GetDataTypeSize', return_value=8) -@mock.patch('rok4.Utils.gdal.GetDataTypeName', return_value="Byte") -def test_compute_format_bit_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info): + +@mock.patch("rok4.Utils.gdal.Info") +@mock.patch("rok4.Utils.gdal.GetColorInterpretationName", return_value="Palette") +@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=8) +@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="Byte") +def test_compute_format_bit_ok( + mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info +): try: mocked_datasource = MagicMock(gdal.Dataset) @@ -249,11 +247,13 @@ def test_compute_format_bit_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, m assert False, f"Color format computation raises an exception: {exc}" -@mock.patch('rok4.Utils.gdal.Info') -@mock.patch('rok4.Utils.gdal.GetColorInterpretationName') -@mock.patch('rok4.Utils.gdal.GetDataTypeSize', return_value=8) -@mock.patch('rok4.Utils.gdal.GetDataTypeName', return_value="Byte") -def test_compute_format_uint8_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info): +@mock.patch("rok4.Utils.gdal.Info") +@mock.patch("rok4.Utils.gdal.GetColorInterpretationName") +@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=8) +@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="Byte") +def test_compute_format_uint8_ok( + mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info +): try: mocked_datasource = MagicMock(gdal.Dataset) @@ -284,11 +284,13 @@ def test_compute_format_uint8_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, assert False, f"Color format computation raises an exception: {exc}" -@mock.patch('rok4.Utils.gdal.Info') -@mock.patch('rok4.Utils.gdal.GetColorInterpretationName') -@mock.patch('rok4.Utils.gdal.GetDataTypeSize', return_value=32) -@mock.patch('rok4.Utils.gdal.GetDataTypeName', return_value="Float32") -def test_compute_format_float32_ok(mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info): +@mock.patch("rok4.Utils.gdal.Info") +@mock.patch("rok4.Utils.gdal.GetColorInterpretationName") +@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=32) +@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="Float32") +def test_compute_format_float32_ok( + mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info +): try: mocked_datasource = MagicMock(gdal.Dataset) @@ -319,11 +321,13 @@ def test_compute_format_float32_ok(mocked_GetDataTypeName, mocked_GetDataTypeSiz assert False, f"Color format computation raises an exception: {exc}" -@mock.patch('rok4.Utils.gdal.Info') -@mock.patch('rok4.Utils.gdal.GetColorInterpretationName') -@mock.patch('rok4.Utils.gdal.GetDataTypeSize', return_value=16) -@mock.patch('rok4.Utils.gdal.GetDataTypeName', return_value="UInt16") -def test_compute_format_unsupported_nok(mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info): +@mock.patch("rok4.Utils.gdal.Info") +@mock.patch("rok4.Utils.gdal.GetColorInterpretationName") +@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=16) +@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="UInt16") +def test_compute_format_unsupported_nok( + mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info +): try: mocked_datasource = MagicMock(gdal.Dataset) @@ -354,11 +358,13 @@ def test_compute_format_unsupported_nok(mocked_GetDataTypeName, mocked_GetDataTy assert False, f"Color format computation raises an exception: {exc}" -@mock.patch('rok4.Utils.gdal.Info') -@mock.patch('rok4.Utils.gdal.GetColorInterpretationName') -@mock.patch('rok4.Utils.gdal.GetDataTypeSize', return_value=16) -@mock.patch('rok4.Utils.gdal.GetDataTypeName', return_value="UInt16") -def test_compute_format_no_band_nok(mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info): +@mock.patch("rok4.Utils.gdal.Info") +@mock.patch("rok4.Utils.gdal.GetColorInterpretationName") +@mock.patch("rok4.Utils.gdal.GetDataTypeSize", return_value=16) +@mock.patch("rok4.Utils.gdal.GetDataTypeName", return_value="UInt16") +def test_compute_format_no_band_nok( + mocked_GetDataTypeName, mocked_GetDataTypeSize, mocked_GetColorInterpretationName, mocked_Info +): try: mocked_datasource = MagicMock(gdal.Dataset) diff --git a/tests/test_Vector.py b/tests/test_Vector.py index d9f22fc..221f5e4 100644 --- a/tests/test_Vector.py +++ b/tests/test_Vector.py @@ -15,67 +15,115 @@ def test_missing_env(): with pytest.raises(MissingEnvironmentError): vector = Vector.from_file("ceph:///ign_std/vector.shp") -@mock.patch('rok4.Vector.copy', side_effect=StorageError('CEPH', 'Not found')) + +@mock.patch("rok4.Vector.copy", side_effect=StorageError("CEPH", "Not found")) def test_wrong_file(mocked_copy): with pytest.raises(StorageError): vector = Vector.from_file("ceph:///vector.geojson") + def test_wrong_format(): with pytest.raises(Exception) as exc: vector = Vector.from_file("ceph:///vector.tif") assert str(exc.value) == "This format of file cannot be loaded" -@mock.patch('rok4.Vector.ogr.Open', return_value="not a shape") + +@mock.patch("rok4.Vector.ogr.Open", return_value="not a shape") def test_wrong_content(mocked_copy): with pytest.raises(Exception) as exc: vector = Vector.from_file("file:///vector.shp") assert str(exc.value) == "The content of file:///vector.shp cannot be read" -@mock.patch('rok4.Vector.copy') -@mock.patch('rok4.Vector.ogr.Open', return_value="not a shape") + +@mock.patch("rok4.Vector.copy") +@mock.patch("rok4.Vector.ogr.Open", return_value="not a shape") def test_wrong_content_ceph(mocked_open, mocked_copy): with pytest.raises(Exception) as exc: vector = Vector.from_file("file:///vector.shp") assert str(exc.value) == "The content of file:///vector.shp cannot be read" -def test_ok_csv1() : - try : - vector_csv1 = Vector.from_file("file://tests/fixtures/vector.csv" , csv={"delimiter":";", "column_x":"x", "column_y":"y"}) - assert str(vector_csv1.layers) == "[('vector', 3, [('id', 'String'), ('x', 'String'), ('y', 'String')])]" + +def test_ok_csv1(): + try: + vector_csv1 = Vector.from_file( + "file://tests/fixtures/vector.csv", + csv={"delimiter": ";", "column_x": "x", "column_y": "y"}, + ) + assert ( + str(vector_csv1.layers) + == "[('vector', 3, [('id', 'String'), ('x', 'String'), ('y', 'String')])]" + ) except Exception as exc: assert False, f"Vector creation raises an exception: {exc}" -def test_ok_csv2() : - try : - vector_csv2 = Vector.from_file("file://tests/fixtures/vector2.csv" , csv={"delimiter":";", "column_wkt":"WKT"}) + +def test_ok_csv2(): + try: + vector_csv2 = Vector.from_file( + "file://tests/fixtures/vector2.csv", csv={"delimiter": ";", "column_wkt": "WKT"} + ) assert str(vector_csv2.layers) == "[('vector2', 1, [('id', 'String'), ('WKT', 'String')])]" except Exception as exc: assert False, f"Vector creation raises an exception: {exc}" -def test_ok_geojson() : - try : + +def test_ok_geojson(): + try: vector = Vector.from_file("file://tests/fixtures/vector.geojson") - assert str(vector.layers) == "[('vector', 1, [('id', 'String'), ('id_fantoir', 'String'), ('numero', 'Integer'), ('rep', 'String'), ('nom_voie', 'String'), ('code_postal', 'Integer'), ('code_insee', 'Integer'), ('nom_commune', 'String'), ('code_insee_ancienne_commune', 'String'), ('nom_ancienne_commune', 'String'), ('x', 'Real'), ('y', 'Real'), ('lon', 'Real'), ('lat', 'Real'), ('type_position', 'String'), ('alias', 'String'), ('nom_ld', 'String'), ('libelle_acheminement', 'String'), ('nom_afnor', 'String'), ('source_position', 'String'), ('source_nom_voie', 'String'), ('certification_commune', 'Integer'), ('cad_parcelles', 'String')])]" + assert ( + str(vector.layers) + == "[('vector', 1, [('id', 'String'), ('id_fantoir', 'String'), ('numero', 'Integer'), ('rep', 'String'), ('nom_voie', 'String'), ('code_postal', 'Integer'), ('code_insee', 'Integer'), ('nom_commune', 'String'), ('code_insee_ancienne_commune', 'String'), ('nom_ancienne_commune', 'String'), ('x', 'Real'), ('y', 'Real'), ('lon', 'Real'), ('lat', 'Real'), ('type_position', 'String'), ('alias', 'String'), ('nom_ld', 'String'), ('libelle_acheminement', 'String'), ('nom_afnor', 'String'), ('source_position', 'String'), ('source_nom_voie', 'String'), ('certification_commune', 'Integer'), ('cad_parcelles', 'String')])]" + ) except Exception as exc: assert False, f"Vector creation raises an exception: {exc}" -def test_ok_gpkg() : - try : + +def test_ok_gpkg(): + try: vector = Vector.from_file("file://tests/fixtures/vector.gpkg") - assert str(vector.layers) == "[('Table1', 2, [('id', 'String')]), ('Table2', 2, [('id', 'Integer'), ('nom', 'String')])]" + assert ( + str(vector.layers) + == "[('Table1', 2, [('id', 'String')]), ('Table2', 2, [('id', 'Integer'), ('nom', 'String')])]" + ) except Exception as exc: assert False, f"Vector creation raises an exception: {exc}" -def test_ok_shp() : - try : + +def test_ok_shp(): + try: vector = Vector.from_file("file://tests/fixtures/ARRONDISSEMENT.shp") - assert str(vector.layers) == "[('ARRONDISSEMENT', 14, [('ID', 'String'), ('NOM', 'String'), ('INSEE_ARR', 'String'), ('INSEE_DEP', 'String'), ('INSEE_REG', 'String'), ('ID_AUT_ADM', 'String'), ('DATE_CREAT', 'String'), ('DATE_MAJ', 'String'), ('DATE_APP', 'Date'), ('DATE_CONF', 'Date')])]" + assert ( + str(vector.layers) + == "[('ARRONDISSEMENT', 14, [('ID', 'String'), ('NOM', 'String'), ('INSEE_ARR', 'String'), ('INSEE_DEP', 'String'), ('INSEE_REG', 'String'), ('ID_AUT_ADM', 'String'), ('DATE_CREAT', 'String'), ('DATE_MAJ', 'String'), ('DATE_APP', 'Date'), ('DATE_CONF', 'Date')])]" + ) except Exception as exc: assert False, f"Vector creation raises an exception: {exc}" -def test_ok_parameters() : - try : - vector = Vector.from_parameters("file://tests/fixtures/ARRONDISSEMENT.shp", (1,2,3,4), [('ARRONDISSEMENT', 14, [('ID', 'String'), ('NOM', 'String'), ('INSEE_ARR', 'String'), ('INSEE_DEP', 'String'), ('INSEE_REG', 'String'), ('ID_AUT_ADM', 'String'), ('DATE_CREAT', 'String'), ('DATE_MAJ', 'String'), ('DATE_APP', 'Date'), ('DATE_CONF', 'Date')])]) + +def test_ok_parameters(): + try: + vector = Vector.from_parameters( + "file://tests/fixtures/ARRONDISSEMENT.shp", + (1, 2, 3, 4), + [ + ( + "ARRONDISSEMENT", + 14, + [ + ("ID", "String"), + ("NOM", "String"), + ("INSEE_ARR", "String"), + ("INSEE_DEP", "String"), + ("INSEE_REG", "String"), + ("ID_AUT_ADM", "String"), + ("DATE_CREAT", "String"), + ("DATE_MAJ", "String"), + ("DATE_APP", "Date"), + ("DATE_CONF", "Date"), + ], + ) + ], + ) assert str(vector.path) == "file://tests/fixtures/ARRONDISSEMENT.shp" except Exception as exc: assert False, f"Vector creation raises an exception: {exc}"