diff --git a/src/rok4/layer.py b/src/rok4/layer.py
index 6a72d0c..c6c21ed 100644
--- a/src/rok4/layer.py
+++ b/src/rok4/layer.py
@@ -5,18 +5,21 @@
- `Layer` - Descriptor to broadcast pyramids' data
"""
+# -- IMPORTS --
+
+# standard library
import json
import os
import re
from json.decoder import JSONDecodeError
-from typing import Dict, List, Tuple, Union
+from typing import Dict, List, Tuple
+# package
from rok4.enums import PyramidType
-from rok4.exceptions import *
+from rok4.exceptions import FormatError, MissingAttributeError
from rok4.pyramid import Pyramid
-from rok4.storage import *
-from rok4.tile_matrix_set import TileMatrixSet
-from rok4.utils import *
+from rok4.storage import get_data_str, get_infos_from_path, put_data_str
+from rok4.utils import reproject_bbox
class Layer:
@@ -92,8 +95,8 @@ def from_descriptor(cls, descriptor: str) -> "Layer":
)
layer.__bbox = reproject_bbox(layer.__geobbox, "EPSG:4326", layer.__tms.srs, 5)
# On force l'emprise de la couche, on recalcule donc les tuiles limites correspondantes pour chaque niveau
- for l in layer.__levels.values():
- l.set_limits_from_bbox(layer.__bbox)
+ for level in layer.__levels.values():
+ level.set_limits_from_bbox(layer.__bbox)
else:
layer.__bbox = layer.__best_level.bbox
layer.__geobbox = reproject_bbox(layer.__bbox, layer.__tms.srs, "EPSG:4326", 5)
@@ -184,7 +187,7 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None:
Exception: Overlapping in usage pyramids' levels
"""
- ## Toutes les pyramides doivent avoir les même caractéristiques
+ # Toutes les pyramides doivent avoir les même caractéristiques
channels = None
for p in pyramids:
pyramid = Pyramid.from_descriptor(p["path"])
@@ -221,16 +224,16 @@ def __load_pyramids(self, pyramids: List[Dict[str, str]]) -> None:
self.__resampling = pyramid.raster_specifications["interpolation"]
levels = pyramid.get_levels(bottom_level, top_level)
- for l in levels:
- if l.id in self.__levels:
- raise Exception(f"Level {l.id} is present in two used pyramids")
- self.__levels[l.id] = l
+ for level in levels:
+ if level.id in self.__levels:
+ raise Exception(f"Level {level.id} is present in two used pyramids")
+ self.__levels[level.id] = level
self.__pyramids.append(
{"pyramid": pyramid, "bottom_level": bottom_level, "top_level": top_level}
)
- self.__best_level = sorted(self.__levels.values(), key=lambda l: l.resolution)[0]
+ self.__best_level = sorted(self.__levels.values(), key=lambda level: level.resolution)[0]
def __str__(self) -> str:
return f"{self.type.name} layer '{self.__name}'"
diff --git a/src/rok4/pyramid.py b/src/rok4/pyramid.py
index a2ae4a9..c87da7a 100644
--- a/src/rok4/pyramid.py
+++ b/src/rok4/pyramid.py
@@ -6,24 +6,40 @@
- `Level` - Level of a pyramid
"""
+# -- IMPORTS --
+
+# standard library
import io
import json
import os
import re
+import tempfile
import zlib
from json.decoder import JSONDecodeError
-from typing import Dict, Iterator, List, Tuple, Union
+from typing import Dict, Iterator, List, Tuple
+# 3rd party
import mapbox_vector_tile
import numpy
from PIL import Image
+# package
from rok4.enums import PyramidType, SlabType, StorageType
-from rok4.exceptions import *
-from rok4.storage import *
+from rok4.exceptions import FormatError, MissingAttributeError
+from rok4.storage import (
+ copy,
+ get_data_binary,
+ get_data_str,
+ get_infos_from_path,
+ get_path_from_infos,
+ put_data_str,
+ remove,
+ size_path,
+)
from rok4.tile_matrix_set import TileMatrix, TileMatrixSet
-from rok4.utils import *
+from rok4.utils import reproject_point, srs_to_spatialreference
+# -- GLOBALS --
ROK4_IMAGE_HEADER_SIZE = 2048
"""Slab's header size, 2048 bytes"""
@@ -441,8 +457,8 @@ def from_descriptor(cls, descriptor: str) -> "Pyramid":
pyramid.__masks = False
# Niveaux
- for l in data["levels"]:
- lev = Level.from_descriptor(l, pyramid)
+ for level in data["levels"]:
+ lev = Level.from_descriptor(level, pyramid)
pyramid.__levels[lev.id] = lev
if pyramid.__tms.get_level(lev.id) is None:
@@ -510,12 +526,12 @@ def from_other(cls, other: "Pyramid", name: str, storage: Dict) -> "Pyramid":
pyramid.__raster_specifications = other.__raster_specifications
# Niveaux
- for l in other.__levels.values():
- lev = Level.from_other(l, pyramid)
+ for level in other.__levels.values():
+ lev = Level.from_other(level, pyramid)
pyramid.__levels[lev.id] = lev
except KeyError as e:
- raise MissingAttributeError(descriptor, e)
+ raise MissingAttributeError(pyramid.descriptor, e)
return pyramid
@@ -540,10 +556,12 @@ def serializable(self) -> Dict:
serialization = {"tile_matrix_set": self.__tms.name, "format": self.__format}
serialization["levels"] = []
- sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution, reverse=True)
+ sorted_levels = sorted(
+ self.__levels.values(), key=lambda level: level.resolution, reverse=True
+ )
- for l in sorted_levels:
- serialization["levels"].append(l.serializable)
+ for level in sorted_levels:
+ serialization["levels"].append(level.serializable)
if self.type == PyramidType.RASTER:
serialization["raster_specifications"] = self.__raster_specifications
@@ -639,9 +657,7 @@ def storage_depth(self, d: int) -> None:
Exception: the depth is not equal to the already known depth
"""
if "depth" in self.__storage and self.__storage["depth"] != d:
- raise Exception(
- f"Pyramid {pyramid.__descriptor} owns levels with different path depths"
- )
+ raise Exception(f"Pyramid {self.__descriptor} owns levels with different path depths")
self.__storage["depth"] = d
@property
@@ -683,7 +699,7 @@ def bottom_level(self) -> "Level":
Returns:
Level: the bottom level
"""
- return sorted(self.__levels.values(), key=lambda l: l.resolution)[0]
+ return sorted(self.__levels.values(), key=lambda level: level.resolution)[0]
@property
def top_level(self) -> "Level":
@@ -692,7 +708,7 @@ def top_level(self) -> "Level":
Returns:
Level: the top level
"""
- return sorted(self.__levels.values(), key=lambda l: l.resolution)[-1]
+ return sorted(self.__levels.values(), key=lambda level: level.resolution)[-1]
@property
def type(self) -> PyramidType:
@@ -862,7 +878,7 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]:
List[Level]: asked sorted levels
"""
- sorted_levels = sorted(self.__levels.values(), key=lambda l: l.resolution)
+ sorted_levels = sorted(self.__levels.values(), key=lambda level: level.resolution)
levels = []
@@ -881,13 +897,13 @@ def get_levels(self, bottom_id: str = None, top_id: str = None) -> List[Level]:
end = False
- for l in sorted_levels:
- if not begin and l.id == bottom_id:
+ for level in sorted_levels:
+ if not begin and level.id == bottom_id:
begin = True
if begin:
- levels.append(l)
- if top_id is not None and l.id == top_id:
+ levels.append(level)
+ if top_id is not None and level.id == top_id:
end = True
break
else:
@@ -1059,7 +1075,7 @@ def get_tile_data_binary(self, level: str, column: int, row: int) -> str:
raise Exception(f"No level {level} in the pyramid")
if level_object.slab_width == 1 and level_object.slab_height == 1:
- raise NotImplementedError(f"One-tile slab pyramid is not handled")
+ raise NotImplementedError("One-tile slab pyramid is not handled")
if not level_object.is_in_limits(column, row):
return None
@@ -1090,7 +1106,7 @@ def get_tile_data_binary(self, level: str, column: int, row: int) -> str:
2 * 4 * level_object.slab_width * level_object.slab_height,
),
)
- except FileNotFoundError as e:
+ except FileNotFoundError:
# L'absence de la dalle est gérée comme simplement une absence de données
return None
@@ -1277,7 +1293,7 @@ def get_tile_data_vector(self, level: str, column: int, row: int) -> Dict:
if binary_tile is None:
return None
- level_object = self.get_level(level)
+ self.get_level(level)
if self.__format == "TIFF_PBF_MVT":
try:
@@ -1334,7 +1350,7 @@ def get_tile_indices(
level_object = self.get_level(level)
if level_object is None:
- raise Exception(f"Cannot found the level to calculate indices")
+ raise Exception("Cannot found the level to calculate indices")
if (
"srs" in kwargs
diff --git a/src/rok4/raster.py b/src/rok4/raster.py
index 3d2d813..d52617e 100644
--- a/src/rok4/raster.py
+++ b/src/rok4/raster.py
@@ -6,17 +6,24 @@
- RasterSet - Structure describing a set of raster data.
"""
+# -- IMPORTS --
+
+# standard library
import copy
import json
import re
from typing import Dict, Tuple
+# 3rd party
from osgeo import gdal, ogr
+# package
from rok4.enums import ColorFormat
from rok4.storage import exists, get_osgeo_path, put_data_str
from rok4.utils import compute_bbox, compute_format
+# -- GLOBALS --
+
# Enable GDAL/OGR exceptions
ogr.UseExceptions()
gdal.UseExceptions()
diff --git a/src/rok4/storage.py b/src/rok4/storage.py
index f0a5ffa..3b17bea 100644
--- a/src/rok4/storage.py
+++ b/src/rok4/storage.py
@@ -30,24 +30,31 @@
To precise the cluster to use, bucket name should be bucket_name@s3.storage.fr or bucket_name@s4.storage.fr. If no host is defined (no @) in the bucket name, first S3 cluster is used
"""
+# -- IMPORTS --
+
+# standard library
import hashlib
import os
import re
import tempfile
from shutil import copyfile
-from typing import Dict, List, Tuple, Union
+from typing import Dict, Tuple, Union
+# 3rd party
import boto3
import botocore.exceptions
import rados
import requests
from osgeo import gdal
-gdal.UseExceptions()
-
+# package
from rok4.enums import StorageType
-from rok4.exceptions import *
+from rok4.exceptions import MissingEnvironmentError, StorageError
+# -- GLOBALS --
+
+# Enable GDAL/OGR exceptions
+gdal.UseExceptions()
__S3_CLIENTS = {}
__S3_DEFAULT_CLIENT = None
@@ -331,7 +338,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
else:
data = ioctx.read(base_name, range[1], range[0])
- except rados.ObjectNotFound as e:
+ except rados.ObjectNotFound:
raise FileNotFoundError(f"{storage_type.value}{path}")
except Exception as e:
@@ -348,7 +355,7 @@ def get_data_binary(path: str, range: Tuple[int, int] = None) -> str:
f.close()
- except FileNotFoundError as e:
+ except FileNotFoundError:
raise FileNotFoundError(f"{storage_type.value}{path}")
except Exception as e:
@@ -507,7 +514,7 @@ def exists(path: str) -> bool:
try:
ioctx.stat(base_name)
return True
- except rados.ObjectNotFound as e:
+ except rados.ObjectNotFound:
return False
except Exception as e:
raise StorageError("CEPH", e)
@@ -554,7 +561,7 @@ def remove(path: str) -> None:
try:
ioctx.remove_object(base_name)
- except rados.ObjectNotFound as e:
+ except rados.ObjectNotFound:
pass
except Exception as e:
raise StorageError("CEPH", e)
@@ -562,7 +569,7 @@ def remove(path: str) -> None:
elif storage_type == StorageType.FILE:
try:
os.remove(path)
- except FileNotFoundError as e:
+ except FileNotFoundError:
pass
except Exception as e:
raise StorageError("FILE", e)
@@ -599,12 +606,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
to_md5 = hash_file(to_path)
if to_md5 != from_md5:
raise StorageError(
- f"FILE",
+ "FILE",
f"Invalid MD5 sum control for copy file {from_path} to {to_path} : {from_md5} != {to_md5}",
)
except Exception as e:
- raise StorageError(f"FILE", f"Cannot copy file {from_path} to {to_path} : {e}")
+ raise StorageError("FILE", f"Cannot copy file {from_path} to {to_path} : {e}")
elif from_type == StorageType.S3 and to_type == StorageType.FILE:
s3_client, from_bucket = __get_s3_client(from_tray)
@@ -625,7 +632,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
except Exception as e:
raise StorageError(
- f"S3 and FILE", f"Cannot copy S3 object {from_path} to file {to_path} : {e}"
+ "S3 and FILE", f"Cannot copy S3 object {from_path} to file {to_path} : {e}"
)
elif from_type == StorageType.FILE and to_type == StorageType.S3:
@@ -642,12 +649,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
)
if to_md5 != from_md5:
raise StorageError(
- f"FILE and S3",
+ "FILE and S3",
f"Invalid MD5 sum control for copy file {from_path} to S3 object {to_path} : {from_md5} != {to_md5}",
)
except Exception as e:
raise StorageError(
- f"FILE and S3", f"Cannot copy file {from_path} to S3 object {to_path} : {e}"
+ "FILE and S3", f"Cannot copy file {from_path} to S3 object {to_path} : {e}"
)
elif from_type == StorageType.S3 and to_type == StorageType.S3:
@@ -672,12 +679,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
)
if to_md5 != from_md5:
raise StorageError(
- f"S3",
+ "S3",
f"Invalid MD5 sum control for copy S3 object {from_path} to {to_path} : {from_md5} != {to_md5}",
)
except Exception as e:
- raise StorageError(f"S3", f"Cannot copy S3 object {from_path} to {to_path} : {e}")
+ raise StorageError("S3", f"Cannot copy S3 object {from_path} to {to_path} : {e}")
elif from_type == StorageType.CEPH and to_type == StorageType.FILE:
ioctx = __get_ceph_ioctx(from_tray)
@@ -709,13 +716,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
if from_md5 is not None and from_md5 != checker.hexdigest():
raise StorageError(
- f"CEPH and FILE",
+ "CEPH and FILE",
f"Invalid MD5 sum control for copy CEPH object {from_path} to file {to_path} : {from_md5} != {checker.hexdigest()}",
)
except Exception as e:
raise StorageError(
- f"CEPH and FILE", f"Cannot copy CEPH object {from_path} to file {to_path} : {e}"
+ "CEPH and FILE", f"Cannot copy CEPH object {from_path} to file {to_path} : {e}"
)
elif from_type == StorageType.FILE and to_type == StorageType.CEPH:
@@ -746,13 +753,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
if from_md5 is not None and from_md5 != checker.hexdigest():
raise StorageError(
- f"FILE and CEPH",
+ "FILE and CEPH",
f"Invalid MD5 sum control for copy file {from_path} to CEPH object {to_path} : {from_md5} != {checker.hexdigest()}",
)
except Exception as e:
raise StorageError(
- f"FILE and CEPH", f"Cannot copy file {from_path} to CEPH object {to_path} : {e}"
+ "FILE and CEPH", f"Cannot copy file {from_path} to CEPH object {to_path} : {e}"
)
elif from_type == StorageType.CEPH and to_type == StorageType.CEPH:
@@ -780,12 +787,12 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
if from_md5 is not None and from_md5 != checker.hexdigest():
raise StorageError(
- f"FILE and CEPH",
+ "FILE and CEPH",
f"Invalid MD5 sum control for copy CEPH object {from_path} to {to_path} : {from_md5} != {checker.hexdigest()}",
)
except Exception as e:
- raise StorageError(f"CEPH", f"Cannot copy CEPH object {from_path} to {to_path} : {e}")
+ raise StorageError("CEPH", f"Cannot copy CEPH object {from_path} to {to_path} : {e}")
elif from_type == StorageType.CEPH and to_type == StorageType.S3:
from_ioctx = __get_ceph_ioctx(from_tray)
@@ -819,13 +826,13 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
if from_md5 is not None and from_md5 != checker.hexdigest():
raise StorageError(
- f"CEPH and S3",
+ "CEPH and S3",
f"Invalid MD5 sum control for copy CEPH object {from_path} to S3 object {to_path} : {from_md5} != {checker.hexdigest()}",
)
except Exception as e:
raise StorageError(
- f"CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}"
+ "CEPH and S3", f"Cannot copy CEPH object {from_path} to S3 object {to_path} : {e}"
)
elif (
@@ -840,7 +847,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
except Exception as e:
raise StorageError(
- f"HTTP(S) and FILE",
+ "HTTP(S) and FILE",
f"Cannot copy HTTP(S) object {from_path} to FILE object {to_path} : {e}",
)
@@ -860,7 +867,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
except Exception as e:
raise StorageError(
- f"HTTP(S) and CEPH",
+ "HTTP(S) and CEPH",
f"Cannot copy HTTP(S) object {from_path} to CEPH object {to_path} : {e}",
)
@@ -883,7 +890,7 @@ def copy(from_path: str, to_path: str, from_md5: str = None) -> None:
except Exception as e:
raise StorageError(
- f"HTTP(S) and S3",
+ "HTTP(S) and S3",
f"Cannot copy HTTP(S) object {from_path} to S3 object {to_path} : {e}",
)
@@ -913,7 +920,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None:
if target_type != link_type:
raise StorageError(
f"{target_type.name} and {link_type.name}",
- f"Cannot make link between two different storage types",
+ "Cannot make link between two different storage types",
)
if hard and target_type != StorageType.FILE:
@@ -926,7 +933,7 @@ def link(target_path: str, link_path: str, hard: bool = False) -> None:
if target_s3_client["host"] != link_s3_client["host"]:
raise StorageError(
- f"S3",
+ "S3",
f"Cannot make link {link_path} -> {target_path} : link works only on the same S3 cluster",
)
diff --git a/src/rok4/tile_matrix_set.py b/src/rok4/tile_matrix_set.py
index 976a63e..84a3d22 100644
--- a/src/rok4/tile_matrix_set.py
+++ b/src/rok4/tile_matrix_set.py
@@ -9,14 +9,20 @@
- ROK4_TMS_DIRECTORY
"""
+# -- IMPORTS --
+
+# standard library
import json
import os
from json.decoder import JSONDecodeError
from typing import Dict, List, Tuple
-from rok4.exceptions import *
+# package
+from rok4.exceptions import FormatError, MissingAttributeError, MissingEnvironmentError
from rok4.storage import get_data_str
-from rok4.utils import *
+from rok4.utils import srs_to_spatialreference
+
+# -- GLOBALS --
class TileMatrix:
@@ -212,8 +218,8 @@ def __init__(self, name: str) -> None:
self.srs = data["crs"]
self.sr = srs_to_spatialreference(self.srs)
self.levels = {}
- for l in data["tileMatrices"]:
- lev = TileMatrix(l, self)
+ for level in data["tileMatrices"]:
+ lev = TileMatrix(level, self)
self.levels[lev.id] = lev
if len(self.levels.keys()) == 0:
@@ -232,7 +238,7 @@ def __init__(self, name: str) -> None:
except RuntimeError as e:
raise Exception(
- f"Wrong attribute 'crs' ('{self.srs}') in '{self.path}', not recognize by OSR"
+ f"Wrong attribute 'crs' ('{self.srs}') in '{self.path}', not recognize by OSR. Trace : {e}"
)
def get_level(self, level_id: str) -> "TileMatrix":
@@ -249,4 +255,4 @@ def get_level(self, level_id: str) -> "TileMatrix":
@property
def sorted_levels(self) -> List[TileMatrix]:
- return sorted(self.levels.values(), key=lambda l: l.resolution)
+ return sorted(self.levels.values(), key=lambda level: level.resolution)
diff --git a/src/rok4/utils.py b/src/rok4/utils.py
index 004852b..40c7884 100644
--- a/src/rok4/utils.py
+++ b/src/rok4/utils.py
@@ -1,18 +1,23 @@
"""Provide functions to manipulate OGR / OSR entities
"""
+# -- IMPORTS --
+# standard library
import os
import re
-from typing import Dict, List, Tuple, Union
+from typing import Tuple
+# 3rd party
from osgeo import gdal, ogr, osr
+# package
+from rok4.enums import ColorFormat
+
+# -- GLOBALS --
ogr.UseExceptions()
osr.UseExceptions()
gdal.UseExceptions()
-from rok4.enums import ColorFormat
-
__SR_BOOK = {}
@@ -187,7 +192,7 @@ def compute_bbox(source_dataset: gdal.Dataset) -> Tuple:
transform_vector = source_dataset.GetGeoTransform()
if transform_vector is None:
raise Exception(
- f"No transform vector found in the dataset created from "
+ "No transform vector found in the dataset created from "
+ f"the following file : {source_dataset.GetFileList()[0]}"
)
width = source_dataset.RasterXSize
diff --git a/src/rok4/vector.py b/src/rok4/vector.py
index 2cdb0a2..268d181 100644
--- a/src/rok4/vector.py
+++ b/src/rok4/vector.py
@@ -6,17 +6,23 @@
"""
+# -- IMPORTS --
+
+# standard library
import os
import tempfile
+# 3rd party
from osgeo import ogr
+# package
+from rok4.storage import copy, get_osgeo_path
+
+# -- GLOBALS --
+
# Enable GDAL/OGR exceptions
ogr.UseExceptions()
-from rok4.exceptions import *
-from rok4.storage import copy, get_osgeo_path
-
class Vector:
"""A data vector
@@ -137,7 +143,7 @@ def from_file(cls, path: str, **kwargs) -> "Vector":
vrt_file += "" + tmp_path + ".csv\n"
vrt_file += "" + name_fich + "\n"
vrt_file += "" + srs + "\n"
- if column_wkt == None:
+ if column_wkt is None:
vrt_file += (
'