From 0e080b31f8ac27497d2f0773f97b86755618736a Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 25 Apr 2023 18:39:22 +0200 Subject: [PATCH 01/30] python3: use six.string_types not version-dependant types Apply "Use of `unicode` needed to be immediately handled" by Yann. Reserved "but a few checks relying on `str` could become insufficient in python2 with the larger usage of unicode strings." for applying optionally these not-needed later, because they could change behavior and if needed, they would belong to these other commits. And, after 21 commits from Yann and my work on to of that, that didn't appear. Co-authored-by: Yann Dirson Signed-off-by: Bernhard Kaindl --- xcp/net/mac.py | 3 ++- xcp/pci.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/xcp/net/mac.py b/xcp/net/mac.py index 47e586c0..1e153c9a 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -31,6 +31,7 @@ __author__ = "Andrew Cooper" import re +import six VALID_COLON_MAC = re.compile(r"^([\da-fA-F]{1,2}:){5}[\da-fA-F]{1,2}$") VALID_DASH_MAC = re.compile(r"^([\da-fA-F]{1,2}-){5}[\da-fA-F]{1,2}$") @@ -59,7 +60,7 @@ def __init__(self, addr): self.octets = [] self.integer = -1 - if isinstance(addr, (str, unicode)): + if isinstance(addr, six.string_types): res = VALID_COLON_MAC.match(addr) if res: diff --git a/xcp/pci.py b/xcp/pci.py index 1c8e081d..be2df8d8 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -24,6 +24,7 @@ import os.path import subprocess import re +import six _SBDF = (r"(?:(?P [\da-dA-F]{4}):)?" # Segment (optional) " (?P [\da-fA-F]{2}):" # Bus @@ -66,7 +67,7 @@ def __init__(self, addr): self.function = -1 self.index = -1 - if isinstance(addr, (str, unicode)): + if isinstance(addr, six.string_types): res = VALID_SBDFI.match(addr) if res: From ac499f837eda1590ce65dcb8b3d301a480e90fdf Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 18:07:43 +0200 Subject: [PATCH 02/30] python3: use "six.ensure_binary" and "six.ensure_text" for str/bytes conversion Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 7b2623fa..149cbcda 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -310,7 +310,7 @@ def _init_write_gz(self): self.__write(b"\037\213\010\010%s\002\377" % timestamp) if self.name.endswith(".gz"): self.name = self.name[:-3] - self.__write(self.name + NUL) + self.__write(six.ensure_binary(self.name) + NUL) def write(self, s): """Write string s to the stream. @@ -1433,7 +1433,7 @@ def extractall(self, path=".", members=None): # Set correct owner, mtime and filemode on directories. for cpioinfo in directories: - path = os.path.join(path, cpioinfo.name) + path = os.path.join(path, six.ensure_text(cpioinfo.name)) try: self.chown(cpioinfo, path) self.utime(cpioinfo, path) From 27c53965ed03caf311e6a8e86f6e3a91e78abd84 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 25 May 2007 20:17:15 +0000 Subject: [PATCH 03/30] Remove direct call's to file's constructor and replace them with calls to open() as ths is considered best practice. (cherry picked from cpython commit 6cef076ba5edbfa42239924951d8acbb087b3b19) Signed-off-by: Brett Cannon --- xcp/cpiofile.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 149cbcda..0ddaff6c 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -951,7 +951,7 @@ def __init__(self, name=None, mode="r", fileobj=None): self.mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode] if not fileobj: - fileobj = file(name, self.mode) + fileobj = bltn_open(name, self.mode) self._extfileobj = False else: if name is None and hasattr(fileobj, "name"): @@ -1109,7 +1109,7 @@ def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9): raise CompressionError("gzip module is not available") if fileobj is None: - fileobj = file(name, mode + "b") + fileobj = bltn_open(name, mode + "b") try: t = cls.cpioopen(name, mode, gzip.GzipFile(name, mode, compresslevel, fileobj)) @@ -1354,7 +1354,7 @@ def add(self, name, arcname=None, recursive=True): # Append the cpio header and data to the archive. if cpioinfo.isreg(): - f = file(name, "rb") + f = bltn_open(name, "rb") self.addfile(cpioinfo, f) f.close() @@ -1594,7 +1594,7 @@ def makefile(self, cpioinfo, targetpath): if extractinfo: source = self.extractfile(extractinfo) - target = file(targetpath, "wb") + target = bltn_open(targetpath, "wb") copyfileobj(source, target) source.close() target.close() @@ -1926,5 +1926,5 @@ def is_cpiofile(name): except CpioError: return False -def cpioOpen(*al, **ad): - return CpioFile.open(*al, **ad) +bltn_open = open +open = CpioFile.open From ab23ef7bc4730580067ece31cc4351900523ac96 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 18:15:22 +0200 Subject: [PATCH 04/30] python3: xcp.net.mac: use six.python_2_unicode_compatible for stringification Signed-off-by: Yann Dirson --- xcp/net/mac.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xcp/net/mac.py b/xcp/net/mac.py index 1e153c9a..c0d4fba0 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -37,6 +37,7 @@ VALID_DASH_MAC = re.compile(r"^([\da-fA-F]{1,2}-){5}[\da-fA-F]{1,2}$") VALID_DOTQUAD_MAC = re.compile(r"^([\da-fA-F]{1,4}\.){2}[\da-fA-F]{1,4}$") +@six.python_2_unicode_compatible class MAC(object): """ Mac address object for manipulation and comparison @@ -123,9 +124,6 @@ def is_local(self): def __str__(self): - return unicode(self).encode('utf-8') - - def __unicode__(self): return ':'.join([ "%0.2x" % x for x in self.octets]) def __repr__(self): From 4dbd2a2879b4100036549705ba34d2a622c547b8 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 18:21:17 +0200 Subject: [PATCH 05/30] xcp.net.ifrename.logic: use "logger.warning", "logger.warn" is deprecated Signed-off-by: Yann Dirson --- xcp/net/ifrename/logic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index e2a3484b..1aa534c0 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -289,9 +289,10 @@ def rename_logic( static_rules, # Check that the function still has the same number of nics if len(lastnics) != len(newnics): - LOG.warn("multi-nic function %s had %d nics but now has %d. " - "Defering all until later for renaming" - % (fn, len(lastnics), len(newnics))) + LOG.warning( + "multi-nic function %s had %d nics but now has %d. " + "Defering all until later for renaming", + fn, len(lastnics), len(newnics)) continue # Check that all nics are still pending a rename From 7ebf6ff1d10406847a0c474fb68e2a60a3d97f86 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 18:22:58 +0200 Subject: [PATCH 06/30] python3: use raw strings for regexps, fixes insufficient quoting Running tests on python3 did reveal some of them. Signed-off-by: Yann Dirson --- xcp/bootloader.py | 6 +++--- xcp/dom0.py | 2 +- xcp/net/ifrename/logic.py | 6 +++--- xcp/net/ifrename/static.py | 2 +- xcp/pci.py | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index a1d19709..81a61bd4 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -336,19 +336,19 @@ def create_label(title): try: for line in fh: l = line.strip() - menu_match = re.match("menuentry ['\"]([^']*)['\"](.*){", l) + menu_match = re.match(r"menuentry ['\"]([^']*)['\"](.*){", l) # Only parse unindented default and timeout lines to prevent # changing these lines in if statements. if l.startswith('set default=') and l == line.rstrip(): default = l.split('=')[1] - match = re.match("['\"](.*)['\"]$", default) + match = re.match(r"['\"](.*)['\"]$", default) if match: default = match.group(1) elif l.startswith('set timeout=') and l == line.rstrip(): timeout = int(l.split('=')[1]) * 10 elif l.startswith('serial'): - match = re.match("serial --unit=(\d+) --speed=(\d+)", l) + match = re.match(r"serial --unit=(\d+) --speed=(\d+)", l) if match: serial = { 'port': int(match.group(1)), diff --git a/xcp/dom0.py b/xcp/dom0.py index 086a683b..b8a46c3a 100644 --- a/xcp/dom0.py +++ b/xcp/dom0.py @@ -96,7 +96,7 @@ def default_memory(host_mem_kib): return default_memory_for_version(host_mem_kib, platform_version) -_size_and_unit_re = re.compile("^(-?\d+)([bkmg]?)$", re.IGNORECASE) +_size_and_unit_re = re.compile(r"^(-?\d+)([bkmg]?)$", re.IGNORECASE) def _parse_size_and_unit(s): m = _size_and_unit_re.match(s) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index 1aa534c0..41e74c02 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -52,9 +52,9 @@ from xcp.logger import LOG from xcp.net.ifrename.macpci import MACPCI -VALID_CUR_STATE_KNAME = re.compile("^(?:eth[\d]+|side-[\d]+-eth[\d]+)$") -VALID_ETH_NAME = re.compile("^eth([\d])+$") -VALID_IBFT_NAME = re.compile("^ibft([\d])+$") +VALID_CUR_STATE_KNAME = re.compile(r"^(?:eth[\d]+|side-[\d]+-eth[\d]+)$") +VALID_ETH_NAME = re.compile(r"^eth([\d])+$") +VALID_IBFT_NAME = re.compile(r"^ibft([\d])+$") # util needs to import VALID_ETH_NAME from xcp.net.ifrename import util diff --git a/xcp/net/ifrename/static.py b/xcp/net/ifrename/static.py index c1b9b100..bf503d54 100644 --- a/xcp/net/ifrename/static.py +++ b/xcp/net/ifrename/static.py @@ -89,7 +89,7 @@ class StaticRules(object): methods = ["mac", "pci", "ppn", "label", "guess"] validators = { "mac": VALID_MAC, "pci": VALID_PCI, - "ppn": re.compile("^(?:em\d+|p(?:ci)?\d+p\d+)$") + "ppn": re.compile(r"^(?:em\d+|p(?:ci)?\d+p\d+)$") } def __init__(self, path=None, fd=None): diff --git a/xcp/pci.py b/xcp/pci.py index be2df8d8..45f5ceb3 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -27,9 +27,9 @@ import six _SBDF = (r"(?:(?P [\da-dA-F]{4}):)?" # Segment (optional) - " (?P [\da-fA-F]{2}):" # Bus - " (?P [\da-fA-F]{2})\." # Device - " (?P[\da-fA-F])" # Function + r" (?P [\da-fA-F]{2}):" # Bus + r" (?P [\da-fA-F]{2})\." # Device + r" (?P[\da-fA-F])" # Function ) # Don't change the meaning of VALID_SBDF as some parties may be using it @@ -37,7 +37,7 @@ VALID_SBDFI = re.compile( r"^(?P%s)" - " (?:[[](?P[\d]{1,2})[]])?$" # Index (optional) + r" (?:[[](?P[\d]{1,2})[]])?$" # Index (optional) % _SBDF , re.X) From 42728b365193c5adf3518159c1ec5c977dc8f7d1 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 12:06:59 +0200 Subject: [PATCH 07/30] test_dom0: mock "open()" in a python3-compatible way Signed-off-by: Yann Dirson --- tests/test_dom0.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dom0.py b/tests/test_dom0.py index bf198341..440bb109 100644 --- a/tests/test_dom0.py +++ b/tests/test_dom0.py @@ -30,7 +30,7 @@ def mock_version(open_mock, version): (2*1024, 4*1024, 8*1024), # Above max ] - with patch("__builtin__.open") as open_mock: + with patch("xcp.dom0.open") as open_mock: for host_gib, dom0_mib, _ in test_values: mock_version(open_mock, '2.8.0') expected = dom0_mib * 1024; @@ -39,7 +39,7 @@ def mock_version(open_mock, version): open_mock.assert_called_with("/etc/xensource-inventory") - with patch("__builtin__.open") as open_mock: + with patch("xcp.dom0.open") as open_mock: for host_gib, _, dom0_mib in test_values: mock_version(open_mock, '2.9.0') expected = dom0_mib * 1024; From b4a790623e83aec0f8c86d70047e7984b4dcf6e7 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 15:05:25 +0200 Subject: [PATCH 08/30] ifrename: don't rely on dict ordering in tests There is no guaranty about ordering of dict elements, and tests compare results derived from enumerating a dict element. We could have used an OrderedDict to store the formulae and get a predictible output order, but just considering the output as a set seems better. Only applying this to rules expected to hold more than one element. Signed-off-by: Yann Dirson --- tests/test_ifrename_dynamic.py | 4 ++-- tests/test_ifrename_static.py | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_ifrename_dynamic.py b/tests/test_ifrename_dynamic.py index 1cc95e39..0948d254 100644 --- a/tests/test_ifrename_dynamic.py +++ b/tests/test_ifrename_dynamic.py @@ -125,10 +125,10 @@ def test_pci_matching_invert(self): MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", kname="eth1", ppn="", label="")]) - self.assertEqual(dr.rules,[ + self.assertEqual(set(dr.rules), set([ MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1"), MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0") - ]) + ])) def test_pci_missing(self): diff --git a/tests/test_ifrename_static.py b/tests/test_ifrename_static.py index 9b11e380..674decfe 100644 --- a/tests/test_ifrename_static.py +++ b/tests/test_ifrename_static.py @@ -375,10 +375,10 @@ def test_pci_matching(self): sr.generate(self.state) - self.assertEqual(sr.rules,[ - MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth1"), - MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth0") - ]) + self.assertEqual(set(sr.rules), set([ + MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth1"), + MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth0") + ])) def test_pci_matching_invert(self): @@ -389,10 +389,10 @@ def test_pci_matching_invert(self): sr.generate(self.state) - self.assertEqual(sr.rules,[ - MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1"), - MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0") - ]) + self.assertEqual(set(sr.rules), set([ + MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1"), + MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0") + ])) def test_pci_matching_mixed(self): @@ -403,10 +403,10 @@ def test_pci_matching_mixed(self): sr.generate(self.state) - self.assertEqual(sr.rules,[ - MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0"), - MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1") - ]) + self.assertEqual(set(sr.rules), set([ + MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0"), + MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1") + ])) def test_pci_missing(self): From b975afeda0f49d0817f08979a9f9e16a87a05f0e Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 16:18:39 +0200 Subject: [PATCH 09/30] test_cpio: ensure paths are handled as text Caught by extended test. Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 0ddaff6c..50852c09 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1420,7 +1420,7 @@ def extractall(self, path=".", members=None): # Extract directory with a safe mode, so that # all files below can be extracted as well. try: - os.makedirs(os.path.join(path, cpioinfo.name), 0o777) + os.makedirs(os.path.join(path, six.ensure_text(cpioinfo.name)), 0o777) except EnvironmentError: pass directories.append(cpioinfo) @@ -1462,7 +1462,7 @@ def extract(self, member, path=""): cpioinfo._link_path = path try: - self._extract_member(cpioinfo, os.path.join(path, cpioinfo.name)) + self._extract_member(cpioinfo, os.path.join(path, six.ensure_text(cpioinfo.name))) except EnvironmentError as e: if self.errorlevel > 0: raise From ff165f78d086c90e77a4952c1a43e250e23ee2b6 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 26 Jul 2022 17:16:30 +0200 Subject: [PATCH 10/30] cpiofile: migrate last "list.sort()" call still using a "cmp" argument This goes away in python3. Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 50852c09..fb4d96f7 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1428,7 +1428,7 @@ def extractall(self, path=".", members=None): self.extract(cpioinfo, path) # Reverse sort directories. - directories.sort(lambda a, b: cmp(a.name, b.name)) + directories.sort(key=lambda x: x.name) directories.reverse() # Set correct owner, mtime and filemode on directories. From b91f2837934c970359daa0c07dfc83cefa844735 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 25 Apr 2023 17:29:14 +0200 Subject: [PATCH 11/30] xcp.repository: switch from md5 to hashlib.md5 hashlib came with python 2.5, and old md5 module disappears in 3.0 Originally by Yann Dirson, changed to not add .encode() for md5 creation, because encoding changes are dealt with in other commits. Co-authored-by: Yann Dirson Signed-off-by: Bernhard Kaindl --- xcp/repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/repository.py b/xcp/repository.py index ca284647..7c1c0b3c 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -23,7 +23,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import md5 +from hashlib import md5 import os.path import xml.dom.minidom import ConfigParser @@ -246,7 +246,7 @@ def findRepositories(cls, access): def __init__(self, access, base, is_group = False): BaseRepository.__init__(self, access, base) self.is_group = is_group - self._md5 = md5.new() + self._md5 = md5() self.requires = [] self.packages = [] From e84c3a5fcf4da7ac6976063fbbdf22b38305bd54 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 12:06:18 +0200 Subject: [PATCH 12/30] Pylint complements: honor len-as-condition convention Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index c3543c89..c1c4cd37 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -14,7 +14,7 @@ def writeRandomFile(fn, size, start=b'', add=b'a'): with open(fn, 'wb') as f: m = md5() m.update(start) - assert(len(add) != 0) + assert add while size > 0: d = m.digest() if size < len(d): From 26dff48dea7f4c74c60e385d5b4ecdbc70c57a64 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:40:49 +0200 Subject: [PATCH 13/30] Pylint complements: whitespace in expressions Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- xcp/cmd.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index c1c4cd37..fdb34f40 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -18,7 +18,7 @@ def writeRandomFile(fn, size, start=b'', add=b'a'): while size > 0: d = m.digest() if size < len(d): - d=d[:size] + d = d[:size] f.write(d) size -= len(d) m.update(add) diff --git a/xcp/cmd.py b/xcp/cmd.py index bbd94656..aa8a847e 100644 --- a/xcp/cmd.py +++ b/xcp/cmd.py @@ -28,11 +28,11 @@ import xcp.logger as logger def runCmd(command, with_stdout = False, with_stderr = False, inputtext = None): - cmd = subprocess.Popen(command, bufsize = 1, - stdin = (inputtext and subprocess.PIPE or None), - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - shell = isinstance(command, str)) + cmd = subprocess.Popen(command, bufsize=1, + stdin=(inputtext and subprocess.PIPE or None), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=isinstance(command, str)) (out, err) = cmd.communicate(inputtext) rv = cmd.returncode From 870ff059f4ba680da7c14c6a06f4e2b390764d87 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 16:24:30 +0200 Subject: [PATCH 14/30] Pylint complements: test_ifrename_logic: disable "no-member" warning Reported under python3 for members created on-the-fly in `setUp()` Signed-off-by: Yann Dirson --- tests/test_ifrename_logic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index 3bb7a911..a2715334 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -518,6 +518,7 @@ def test_ibft_nic_to_ibft(self): class TestInputSanitisation(unittest.TestCase): + # pylint: disable=no-member def setUp(self): """ From 32e5268afc1579fb9db8da6bc32bc6aeac8f1a39 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 16:33:12 +0200 Subject: [PATCH 15/30] Pylint complements: avoid no-else-raise "refactor" issues With python3, pylint complains about `else: raise()` constructs. This rework avoids them and reduces cyclomatic complexity by using the error-out-first idiom. Signed-off-by: Yann Dirson --- xcp/net/mac.py | 32 ++++++++++++------------ xcp/pci.py | 66 ++++++++++++++++++++++++-------------------------- 2 files changed, 47 insertions(+), 51 deletions(-) diff --git a/xcp/net/mac.py b/xcp/net/mac.py index c0d4fba0..56ba4b7b 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -61,27 +61,25 @@ def __init__(self, addr): self.octets = [] self.integer = -1 - if isinstance(addr, six.string_types): - - res = VALID_COLON_MAC.match(addr) - if res: - self._set_from_str_octets(addr.split(":")) - return + if not isinstance(addr, six.string_types): + raise TypeError("String expected") - res = VALID_DASH_MAC.match(addr) - if res: - self._set_from_str_octets(addr.split("-")) - return + res = VALID_COLON_MAC.match(addr) + if res: + self._set_from_str_octets(addr.split(":")) + return - res = VALID_DOTQUAD_MAC.match(addr) - if res: - self._set_from_str_quads(addr.split(".")) - return + res = VALID_DASH_MAC.match(addr) + if res: + self._set_from_str_octets(addr.split("-")) + return - raise ValueError("Unrecognised MAC address '%s'" % addr) + res = VALID_DOTQUAD_MAC.match(addr) + if res: + self._set_from_str_quads(addr.split(".")) + return - else: - raise TypeError("String expected") + raise ValueError("Unrecognised MAC address '%s'" % addr) def _set_from_str_octets(self, octets): diff --git a/xcp/pci.py b/xcp/pci.py index 45f5ceb3..ffc52a33 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -67,48 +67,46 @@ def __init__(self, addr): self.function = -1 self.index = -1 - if isinstance(addr, six.string_types): - - res = VALID_SBDFI.match(addr) - if res: - groups = res.groupdict() + if not isinstance(addr, six.string_types): + raise TypeError("String expected") - if "segment" in groups and groups["segment"] is not None: - self.segment = int(groups["segment"], 16) - else: - self.segment = 0 + res = VALID_SBDFI.match(addr) + if res: + groups = res.groupdict() - self.bus = int(groups["bus"], 16) - if not ( 0 <= self.bus < 2**8 ): - raise ValueError("Bus '%d' out of range 0 <= bus < 256" - % (self.bus,)) + if "segment" in groups and groups["segment"] is not None: + self.segment = int(groups["segment"], 16) + else: + self.segment = 0 - self.device = int(groups["device"], 16) - if not ( 0 <= self.device < 2**5): - raise ValueError("Device '%d' out of range 0 <= device < 32" - % (self.device,)) + self.bus = int(groups["bus"], 16) + if not ( 0 <= self.bus < 2**8 ): + raise ValueError("Bus '%d' out of range 0 <= bus < 256" + % (self.bus,)) - self.function = int(groups["function"], 16) - if not ( 0 <= self.function < 2**3): - raise ValueError("Function '%d' out of range 0 <= device " - "< 8" % (self.function,)) + self.device = int(groups["device"], 16) + if not ( 0 <= self.device < 2**5): + raise ValueError("Device '%d' out of range 0 <= device < 32" + % (self.device,)) - if "index" in groups and groups["index"] is not None: - self.index = int(groups["index"]) - else: - self.index = 0 + self.function = int(groups["function"], 16) + if not ( 0 <= self.function < 2**3): + raise ValueError("Function '%d' out of range 0 <= device " + "< 8" % (self.function,)) - self.integer = (int(self.segment << 16 | - self.bus << 8 | - self.device << 3 | - self.function) << 8 | - self.index) - return + if "index" in groups and groups["index"] is not None: + self.index = int(groups["index"]) + else: + self.index = 0 - raise ValueError("Unrecognised PCI address '%s'" % addr) + self.integer = (int(self.segment << 16 | + self.bus << 8 | + self.device << 3 | + self.function) << 8 | + self.index) + return - else: - raise TypeError("String expected") + raise ValueError("Unrecognised PCI address '%s'" % addr) def __str__(self): From ebc43d5be65fbb9408c5c898fee2f22d3ff5c1b1 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 25 Apr 2023 18:53:20 +0200 Subject: [PATCH 16/30] WIP xcp.repository: switch from ConfigParser to configparser This is supposed to be just a module renaming to conform to PEP8, see https://docs.python.org/3/whatsnew/3.0.html#library-changes The SafeConfigParser class has been renamed to ConfigParser in Python 3.2, and backported as addon package. The `readfp` method now triggers a deprecation warning to replace it with `read_file`. Originally authored by Yann Dirson. Co-authored-by: Yann Dirson Signed-off-by: Bernhard Kaindl --- requirements-dev.txt | 3 +++ tests/test_accessor.py | 2 +- tests/test_repository.py | 2 +- xcp/repository.py | 19 ++++++++++++++----- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b2799686..84da09b5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,3 +8,6 @@ pytest-cov # dependencies also in setup.py until they can be used six future + +# python-2.7 only +configparser ; python_version < "3.0" diff --git a/tests/test_accessor.py b/tests/test_accessor.py index ade787e6..8c892845 100644 --- a/tests/test_accessor.py +++ b/tests/test_accessor.py @@ -4,7 +4,7 @@ class TestAccessor(unittest.TestCase): def test_http(self): - raise unittest.SkipTest("comment out if you really mean it") + #raise unittest.SkipTest("comment out if you really mean it") a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) a.start() self.assertTrue(a.access('.treeinfo')) diff --git a/tests/test_repository.py b/tests/test_repository.py index 833627d0..4768740d 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -6,7 +6,7 @@ class TestRepository(unittest.TestCase): def test_http(self): - raise unittest.SkipTest("comment out if you really mean it") + #raise unittest.SkipTest("comment out if you really mean it") a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) repo_ver = repository.BaseRepository.getRepoVer(a) self.assertEqual(repo_ver, Version([3, 2, 1])) diff --git a/xcp/repository.py b/xcp/repository.py index 7c1c0b3c..b10aa092 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -24,9 +24,11 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from hashlib import md5 +import io import os.path import xml.dom.minidom -import ConfigParser +import configparser +import sys import six @@ -179,10 +181,17 @@ def _getVersion(cls, access, category): access.start() try: - treeinfofp = access.openAddress(cls.TREEINFO_FILENAME) - treeinfo = ConfigParser.SafeConfigParser() - treeinfo.readfp(treeinfofp) - treeinfofp.close() + rawtreeinfofp = access.openAddress(cls.TREEINFO_FILENAME) + if sys.version_info < (3, 0) or isinstance(rawtreeinfofp, io.TextIOBase): + # e.g. with FileAccessor + treeinfofp = rawtreeinfofp + else: + # e.g. with HTTPAccessor + treeinfofp = io.TextIOWrapper(rawtreeinfofp, encoding='utf-8') + treeinfo = configparser.ConfigParser() + treeinfo.read_file(treeinfofp) + treeinfofp = None + rawtreeinfofp.close() if treeinfo.has_section('system-v1'): ver_str = treeinfo.get('system-v1', category_map[category]) else: From 25753550bdbdad60ebb2c95311a2f1f8c425baa1 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Mon, 24 Apr 2023 16:47:32 +0200 Subject: [PATCH 17/30] xcp.xmlunwrap: encode() only if type is unicode (only for Py2) xcp.xmlunwrap extracts XML Elements from XML, and for Python2, the unwrapped unicode is encoded into Py2:str(bytes). Python3 unwraps XML Text elements as the Py3:str type which is likewise Unicode, but since Py3:str is the native type, we don't want to encode the Py3:str to Py3:bytes as that would break the API for use on Python3. BEcause binary data is not legal XML content and XML Text elements are defined to be encoded text, UTF-8 is the standard encoding, which Python converts to. It this fine to only encode() to Py2:str(=bytes) on Python2 as a legacy operation which can be removed once we drop Python2. Signed-off-by: Bernhard Kaindl --- xcp/xmlunwrap.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 1487afab..0523bd12 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -34,7 +34,9 @@ def getText(nodelist): for node in nodelist.childNodes: if node.nodeType == node.TEXT_NODE: rc = rc + node.data - return rc.encode().strip() + if not isinstance(rc, str): # Python 2 only, otherwise it would return unicode + rc = rc.encode() + return rc.strip() def getElementsByTagName(el, tags, mandatory = False): matching = [] @@ -47,7 +49,9 @@ def getElementsByTagName(el, tags, mandatory = False): def getStrAttribute(el, attrs, default = '', mandatory = False): matching = [] for attr in attrs: - val = el.getAttribute(attr).encode() + val = el.getAttribute(attr) + if not isinstance(val, str): # Python 2 only, otherwise it would return unicode + val = val.encode() if val != '': matching.append(val) if len(matching) == 0: From 196a86175a9e8cf736472886b020272c96ca6bd6 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Mon, 24 Apr 2023 18:03:05 +0200 Subject: [PATCH 18/30] Un-futurize: future.stanard_library uses imp(to be removed in Python 3.12) Because future.standard_library also has not type markers, replacing future with six.moves (which is also much shorter) also makes fixing mypy easier. Signed-off-by: Bernhard Kaindl --- requirements-dev.txt | 1 - setup.py | 1 - xcp/accessor.py | 9 ++------- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 84da09b5..71f84af0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,6 @@ pytest pytest-cov # dependencies also in setup.py until they can be used six -future # python-2.7 only configparser ; python_version < "3.0" diff --git a/setup.py b/setup.py index 08c8200d..e7881222 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,5 @@ requires=[ 'branding', 'six', - 'future', ], ) diff --git a/xcp/accessor.py b/xcp/accessor.py index 6d057927..a20dc997 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -25,18 +25,13 @@ """accessor - provide common interface to access methods""" -# pylint: disable=wrong-import-position,wrong-import-order -from future import standard_library -standard_library.install_aliases() - import ftplib import os import tempfile -import urllib.request # pylint: disable=import-error -import urllib.error # pylint: disable=import-error -import urllib.parse # pylint: disable=import-error import errno +from six.moves import urllib # pyright: ignore + import xcp.mount as mount import xcp.logger as logger From 3a31bf10f833ebed631867480f430d43e53f5495 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 25 Apr 2023 17:47:35 +0200 Subject: [PATCH 19/30] accessor: Add mode and kwargs parameters and default to binary mode Fix issue #19 based on the description and progress from PR #24. Allows for opening text and binary files in text and binary modes. Mode, encoding and error handling can be set by passing the parameters "encoding" and "errors" using the kwargs parameters from openAddress() and writeFile() to open(mode, **kwargs) and ftp.makefile(mode, **kwargs). Signed-off-by: Bernhard Kaindl --- xcp/accessor.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index a20dc997..b0743c7d 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -66,7 +66,7 @@ def access(self, name): return True - def openAddress(self, address): + def openAddress(self, address, mode="", **kwargs): """should be overloaded""" pass @@ -94,9 +94,9 @@ def __init__(self, location, ro): super(FilesystemAccessor, self).__init__(ro) self.location = location - def openAddress(self, address): + def openAddress(self, address, mode="rb", **kwargs): try: - filehandle = open(os.path.join(self.location, address), 'r') + filehandle = open(os.path.join(self.location, address), mode, **kwargs) except OSError as e: if e.errno == errno.EIO: self.lastError = 5 @@ -160,9 +160,9 @@ def finish(self): os.rmdir(self.location) self.location = None - def writeFile(self, in_fh, out_name): + def writeFile(self, in_fh, out_name, mode="wb", **kwargs): logger.info("Copying to %s" % os.path.join(self.location, out_name)) - out_fh = open(os.path.join(self.location, out_name), 'w') + out_fh = open(os.path.join(self.location, out_name), mode, **kwargs) return self._writeFile(in_fh, out_fh) def __del__(self): @@ -215,9 +215,9 @@ def __init__(self, baseAddress, ro): super(FileAccessor, self).__init__(ro) self.baseAddress = baseAddress - def openAddress(self, address): + def openAddress(self, address, mode="rb", **kwargs): try: - file = open(os.path.join(self.baseAddress, address)) + file = open(os.path.join(self.baseAddress, address), mode, **kwargs) except IOError as e: if e.errno == errno.EIO: self.lastError = 5 @@ -235,9 +235,9 @@ def openAddress(self, address): return False return file - def writeFile(self, in_fh, out_name): + def writeFile(self, in_fh, out_name, mode="wb", **kwargs): logger.info("Copying to %s" % os.path.join(self.baseAddress, out_name)) - out_fh = open(os.path.join(self.baseAddress, out_name), 'w') + out_fh = open(os.path.join(self.baseAddress, out_name), mode, **kwargs) return self._writeFile(in_fh, out_fh) def __repr__(self): @@ -326,13 +326,13 @@ def access(self, path): self.lastError = 500 return False - def openAddress(self, address): + def openAddress(self, address, mode="rb", **kwargs): logger.debug("Opening "+address) self._cleanup() url = urllib.parse.unquote(address) self.ftp.voidcmd('TYPE I') - s = self.ftp.transfercmd('RETR ' + url).makefile('rb') + s = self.ftp.transfercmd('RETR ' + url).makefile(mode, **kwargs) self.cleanup = True return s @@ -368,8 +368,10 @@ def __init__(self, baseAddress, ro): self.baseAddress = rebuild_url(self.url_parts) - def openAddress(self, address): + def openAddress(self, address, mode="", **kwargs): + """Open an HTTP/S URL. Note urllib must return binary as encoding may be gzip""" try: + # pylint: disable-next=consider-using-with urlFile = urllib.request.urlopen(os.path.join(self.baseAddress, address)) except urllib.error.HTTPError as e: self.lastError = e.code From 4d47ee5e1c9a010fc648bb765dfe9b235fa43972 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 28 Apr 2023 15:10:00 +0200 Subject: [PATCH 20/30] accessor: Suppress pylint warnings for open(): encoding and use-with Because the upcoming commits enable pylint to be more accessible, fix the errors which would be shown around suggesting to pass encoding to open() calls. The previous patch changed the default for the mode parameter to "rb"/"wb" in order to fix reading from binary files with Python3. This is also conistent with HTTPAccessor only being able to read bytes because urllib cannot decompress HTTP gzip encodings on the fly, and a conversion from the decompressed data to unicode sting can only happen after decompression. Because of the need to default to binary mode, the pylint warning to pass encoding= to open() does not apply to these calls which are setup by their calling mode to use binary mode by default. The next commit will enforce binary mode unless encoding is specified. Signed-off-by: Bernhard Kaindl --- xcp/accessor.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index b0743c7d..dec6581a 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -29,6 +29,7 @@ import os import tempfile import errno +from abc import abstractmethod from six.moves import urllib # pyright: ignore @@ -66,9 +67,9 @@ def access(self, name): return True + @abstractmethod def openAddress(self, address, mode="", **kwargs): - """should be overloaded""" - pass + """must be overloaded, accept mode and kwargs and return a file handle""" def canEject(self): return False @@ -95,7 +96,7 @@ def __init__(self, location, ro): self.location = location def openAddress(self, address, mode="rb", **kwargs): - try: + try: # pylint: disable-next=unspecified-encoding,consider-using-with filehandle = open(os.path.join(self.location, address), mode, **kwargs) except OSError as e: if e.errno == errno.EIO: @@ -162,6 +163,7 @@ def finish(self): def writeFile(self, in_fh, out_name, mode="wb", **kwargs): logger.info("Copying to %s" % os.path.join(self.location, out_name)) + # pylint: disable-next=unspecified-encoding,consider-using-with out_fh = open(os.path.join(self.location, out_name), mode, **kwargs) return self._writeFile(in_fh, out_fh) @@ -216,7 +218,7 @@ def __init__(self, baseAddress, ro): self.baseAddress = baseAddress def openAddress(self, address, mode="rb", **kwargs): - try: + try: # pylint: disable-next=unspecified-encoding,consider-using-with file = open(os.path.join(self.baseAddress, address), mode, **kwargs) except IOError as e: if e.errno == errno.EIO: @@ -237,6 +239,7 @@ def openAddress(self, address, mode="rb", **kwargs): def writeFile(self, in_fh, out_name, mode="wb", **kwargs): logger.info("Copying to %s" % os.path.join(self.baseAddress, out_name)) + # pylint: disable-next=unspecified-encoding,consider-using-with out_fh = open(os.path.join(self.baseAddress, out_name), mode, **kwargs) return self._writeFile(in_fh, out_fh) From cd6caacc6a456f707e390c342dcd57fdd54c1d7c Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 28 Apr 2023 15:58:54 +0200 Subject: [PATCH 21/30] accessor: enforce binary mode unless encoding is specified The previous patch supresses the pylint warning that encoding= should be bassed to open calls because the default of open() is text mode, which means decoding on reads and encoding on writes. Until this commit, this means that it would be the responsibilty of each callers to specify an encoding and setup handling decoding errors when they want to use file-based accessors in text mode. To be safe against mis-use like `openAccess(file, mode="r")` (note: the mode parameter is new), we could ignore that request and enforce binary mode unless encoding was passed as well. Summary: Use bytes whenever possible as this avoids any errors around coding/conversion of bytes to Unicode strings and back to bytes. Signed-off-by: Bernhard Kaindl --- xcp/accessor.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/xcp/accessor.py b/xcp/accessor.py index dec6581a..92df7b00 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -96,6 +96,9 @@ def __init__(self, location, ro): self.location = location def openAddress(self, address, mode="rb", **kwargs): + # To be safe against mis-use like `openAccess(file, mode="r")` for text mode + # (note: the mode parameter is new), enforce "rb" unless encoding was given: + mode = mode if "encoding" in kwargs else "rb" try: # pylint: disable-next=unspecified-encoding,consider-using-with filehandle = open(os.path.join(self.location, address), mode, **kwargs) except OSError as e: @@ -163,6 +166,9 @@ def finish(self): def writeFile(self, in_fh, out_name, mode="wb", **kwargs): logger.info("Copying to %s" % os.path.join(self.location, out_name)) + # To be safe against mis-use like `writeFile(file, mode="w")` for text mode + # (note: the mode parameter is new), enforce "wb" unless encoding was given: + mode = mode if "encoding" in kwargs else "wb" # pylint: disable-next=unspecified-encoding,consider-using-with out_fh = open(os.path.join(self.location, out_name), mode, **kwargs) return self._writeFile(in_fh, out_fh) @@ -218,6 +224,9 @@ def __init__(self, baseAddress, ro): self.baseAddress = baseAddress def openAddress(self, address, mode="rb", **kwargs): + # To be safe against mis-use like `openAccess(file, mode="r")` for text mode + # (note: the mode parameter is new), enforce "rb" unless encoding was given: + mode = mode if "encoding" in kwargs else "rb" try: # pylint: disable-next=unspecified-encoding,consider-using-with file = open(os.path.join(self.baseAddress, address), mode, **kwargs) except IOError as e: @@ -238,6 +247,9 @@ def openAddress(self, address, mode="rb", **kwargs): return file def writeFile(self, in_fh, out_name, mode="wb", **kwargs): + # To be safe against mis-use like `writeFile(file, mode="w")` for text mode + # (note: the mode parameter is new), enforce "wb" unless encoding was given: + mode = mode if "encoding" in kwargs else "wb" logger.info("Copying to %s" % os.path.join(self.baseAddress, out_name)) # pylint: disable-next=unspecified-encoding,consider-using-with out_fh = open(os.path.join(self.baseAddress, out_name), mode, **kwargs) From baca0da9f477419c25093dc380525d142dfe1047 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 25 Apr 2023 18:19:12 +0200 Subject: [PATCH 22/30] test_accessor.py: Test reading binary data from http and file Add and enable a parameterized test to read binary data from a file- and an HTTP based accessor. Originally authored by Yann Dirson. Extracted, refactored and squashed for a single considated and concise commit by me. Co-authored-by: Yann Dirson Signed-off-by: Bernhard Kaindl --- requirements-dev.txt | 1 + tests/data/repo/boot/isolinux/mboot.c32 | Bin 0 -> 33628 bytes tests/test_accessor.py | 47 ++++++++++++++++++------ 3 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 tests/data/repo/boot/isolinux/mboot.c32 diff --git a/requirements-dev.txt b/requirements-dev.txt index 71f84af0..f5042a93 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,7 @@ diff_cover mock pytest pytest-cov +parameterized # dependencies also in setup.py until they can be used six diff --git a/tests/data/repo/boot/isolinux/mboot.c32 b/tests/data/repo/boot/isolinux/mboot.c32 new file mode 100644 index 0000000000000000000000000000000000000000..3f5b2fe5acf672f54af1000ae844076807abb0d1 GIT binary patch literal 33628 zcmbTf4SZC^xj%kRc9U$v!dW0d&?u`cl{J8gP@5%Dm&hiOW+A+6z;_W6c?l4cJxSoQ zka!Zza5z>AZEb6B?XA5Rue8N0LQtwnz-*w12_Vpx+DN&2mW>iXA#g4G|2}h0Lhz;c ze}DOW&Yqb$^YYB|JoC&m&pb0}x?1?&Enn&a{vN8^tJMjD_PSTZ@8%`zVzJ-e(j1Hz zQykHY+Ut}>dus}Q#gR&z)pNDtQf9OE7SF7{nr;)^ms=un7hctZF01y|V$`ua37OfG zUuF-fic^P7yYL%z^AaS_VO{!7w+cd$_Igh&7SrCMnmt!56Ju0w^os=#>d8onZEnQx z=Xe6_zvmavRl1$}OBO+>+xu}Xsy(J{eh%L>Wh;L5X4I$US(&d}Rnm~ae7gEp1_CNl zG8!%hSMEkp^s4)_Sd8`PT_XhHHL;;tj#t_&nfsalw6?j&_@s>zgob9-e_BpfeR`v< zEm4f#&NEy#GL-QQa;DJ|GBy^7szQymc|}q2bz7tCP2QSv`fMzwv*Cge`BR)Agnc~| zCr%X9P+h#N@{tF3RgPW!5?5 zIy@4&{HNJ8o8midM}8$s2wDkxLK!e1)&C527RBVFdMCh8(SCuIweqiew_6Y%d;oB% znsPHWdI(vf6>8uF>(#%Hs`2TEna#@n*!kaG>J$$tzLVC73~B@S#ekQ!p*~h_wL5R#njdhD0EcI#q=9Wx{SN`u64m{HU*w$1m!CoC<87K#^SgV_=QGx~Edz zccE^o&TB)@Bz-hhnCC#)q6`7XbdXM}-;Tls;6WB{`NRtN%Z4FWo2D#64dB5!XxBmY zQ+Li$z;_Zjlx>>wG;&1mR6||h415sO{1p8S00iQz^BroweVMphKw14we0&gP25ZVs zQHCQkx}GIi@*OM>fON4uTR6{7b;JNw*sioWfL@PJU;C=6P*)Y3P4#!Pf)v$%k`<)t zO$n&hA>rpWG+;1wEqEbbpF;7?R0hP-@5i9bY}ONy-9-99Ld;G7h{c)!si2j7{?Mb3 zJ+fC#bM2k0Ujz~B_U44l%z8cQdT4LoNOtJQ`1dq?2VK~4)~*I@y9K8pTx=DC397mT zH8Ov9=Ed4madHy7dg(*0D0a&+&3+H_>&)My`MZn2JVkF()2c#JNDqF;WXcgSS=W@) zkS;~~2+-hxM>CtF3zu0`zpe(lH6!TO7=+iPEoM&1Af%xLd+^oVhAmA420A()VP7{CqDlq=|+z6GU}12NWd z4Yg_dzwoMB{sl!b`U85{cN!%T>8{I)+y2`d?()@Hb56?>-p;`Hi>+39 zIP0wY2m)SEE{mftDj#<%ZB8|G8d%Tv>(Yi(kZO9qvy_)Q2V;P)Qcn@QP}niNs~uC&WL9*L8y z(zAVf<#&)S?JByxRJk&yYBau0C`?M~MYe!xem0^Xy<}H7|wcpP3Hk@A9-% zS!fXFr>ey^Pe%ntoi28I+A9ZHe5c(Vs{aI==Hz^N+T`N2u@EA`G_Y_?hR81hI*asK z7@x{MduDTF9iTHg8DBZye|!sP+29~e*?`=UOk`2kIfUSCgs+*_nB1iXkeTy$_ZZIh zlzBU4wo-3>T@Vme$gZI@<-cQ*-pg#L_0H3XxywBf2%I#uX%y%;K zSFoR2?F3F6E=u+9Bfof6+BH7|IVMTJ?^tHxf((P;?Ehd8fI|RDgsNUcRdaEpBs*EI z3k5P*ZW@(ixlX+9WVz!_`AFXYa%GlUoYHVO=)Ra|i<8HyGtq31CC{-;)SpM~G>%bw z{CU)#hT2)4orGfB_~<(TFESZPfU)5p5MhyvM7~hBIzO$S@T^x|?_i-GwZ_gu5dA6O z!bH}VF84(3s@#RQMb^{h)4W(}W(*>QJRvwjcsXDw==$E0b&Ekx5Hpefpol{+FX zLK0@ZA_u3XNywX=25m4Qj0Mcm<=|4HVYENyh*(GOj;w;BFc25cop@SLMzJsot0%NcFBSW3f4! zaw}+4q#weR^5|ocf5G?b#2nNzo1TQCb$e%31H)km=C8`-T6F7Cs8p}hfRb+mT9hmI zR*kU)x|K6dlspFo@41OWtsvYj)Fug4 zw@SO>hgebkIiWTd3;UiEgk+(#?n-P-b)r$g+9UuE`W8%u%=ZGR6Ca@=E%0Jy^X@^Y zpB=yOYS`E8)JfTEDw^h>8}{viW+bpn?0m}^Ybgu7#Lg?HBm`c5EuJ!OLGwapO!Kuf zUn6Q#{f$jf-~@da^qq^o9#MX*F4Rp`$;FVf!>fw2Lt83FDJ!X(~5EX1=W1%gP zn+Rv}3*iJR9L@tG=w+i9@YK?kjNuc`BU~am1(}6X^i|Y>V!&+xO&S|X^)69fQ%V|I zuu=`Z5YA8Kb>>4gcL?EpC)EKm=u~OGtEG1g%O4-kpFnj&)c{f;lmrNUslE|`aDIkS zd&Ds9?$`LAWK>ZOlz*<;` zC$Oo)>+x->j`#Q*tL{?Ypazt6m{o7gZlFx^t=SsEB;TO=8kw&?avvh9T&wg7)rp>S zl@{i&jd2ha-ksD)0c8Ni^<+~yCdINo+gDpkWGh$V+Z z|LZH90U;kuVCyC*L%i1yT%~)l{@xcfYKK=63-4fo?H9dPt5jbD?o(=Rz`tAhXyMtH=CI)b+?XD$az2(9$YrF6p)&?!a(b8p=b`=j1 zJD6{)>f51uUEnjKJo<-8zL}?ChT3y2Z4cUr7r@QT_p%z=V+l30&`W>{;sgQ$wbK~- z9cfit<5b^PC`wnM2SHz{M48fYF_E>Q1KW0cku$Sd4ZVEfRqCWTtCZ1P^0L`eP>cEr z%=C42HN+k7s{ArQC{=3-h|+OC7fSE`xgbA>wA!W{kR(J6(l$_MB z)b&2TP+jp~sp|_&i~V)!JYv@M3##iFQgR&f^y}c-Xx6oxPpRk@!8J%~c%{Ftc&XvV zYjp)5jyy~CNK(TFqex#LgZC3mPZ^0uJ;_qTLSB^mhkEAmhyhbWJ|fW(O-=w19p02m zFFR+nXnYg&H7u|9NxPmOhLX}QMhd7BY64oG0wN98l6LK)ui0|*TD!FC9mOVnBwy#P0!G{P$YVgH?kW^V>v$$EM zYUp>Rb?*z_QfNS$+|JIi^Tt?8#fUP;(jSps^=+@S379NOSxr&j`dYw9G6zwwEm$KYBs;>jDc~Cv^DyEX72*fT) zC{>>!B!d5S1PMI|o`y~5eABG}OkaiiqUi=0bzvF?YG9xCUlicO3ZvWCr24j@)bAp} zFTbweEYRIhf$H|IS_w%czsM*0%x0200H9fztA*N0=h{9(x1uSUvZaOyVX!XLZkF zI*QN&gi2N4F$5pvi9nR{FhXcaxK{5H)XUWgn*moag&^T*;_5~LzDK`nv|NbNeKe$* zS5qiOzXwZBLqTV`)1g}j6Y7!$2+lsCUR#rxAZ=-;RX$e>&%-`Yw>N7ApjW7x+8WoL z@(j)2s|7x90{R903&WIlm)V#a=v956>&H;8ubnJvXQb-uWw}$rxpv44ryA!9DeZAA zPJQId{a8AyyqV52YQ~9LVWZJk4*=C`LW&l#Ook;i(O z5|3)QF@v%f>ttO5{VD&u>=?7VpzB`@-loB=dK_>EsV%3d3e-bEMPyL$uS%~q zcp_>?>o)OSG3;)O9$>u>@@fIY2Qb~$?Y)ue>`>>q?03gvXJge_^@oE~-X<->-R3z| z6~_)~_5^J+S7+y>c{-~`c{-{S+@0(Y>rk$asj{(htL}m5$!vb;(Y>TFJqG}&k20_S zt>u8Md5qH4qi%+6K~Sc~1y`sQR!>{SLao5*IaG1CW{*?X*uo+xEV$d)Q`Rt%21YO_ zj&`=%#%9^s44aw@te9Qj9cp20l8pTnE-ja~#oUVR^A+ zHt$8Z_YznieoEbT0W3|~gl2%NW(nZ2Q|?wxt*4Ys;MT^Dcb@fJg=OVf=VvTa^=Q>z z&-u!Dt)iJ_rVnLBR{cKkj-ePRHEBX{aOPo6`EWB9GS7h;Msuu9Z1EflI`VR*op~{7 z=bD(iH~1OZT>#BoeGZ4^g=MIx&}9b^Yz9}772qL!zmtEj!S^>gw>CpSHZ<3a+D$QT zc+!+Cu!epNAEFkeJ!bG1Er~96Q$KT17A%wEf)Z+H9r{C>J+vnMV+0ZLf>5DP-j~$fzMgsb5Co<6T}Gz zD(3SV?TIEUv@%rY`y68u3kZhx@H+zxVfXZC3fNYtN>Pha*lchFXNPHE07rBf)~Hxy zfJ+cl`4^@~r>HqD*k&FEo>(ge#I4F_4tCjd2@HM?0H?UvXBS>(x=tccMXZBpr0lcx zjX|YbWbbC~-#C4+&}el@JA0*_QZnZk!QowAJqqTD;K6yPE zm$b84tyfN>uy`WYrTBGgq#MZy*dl*70*=U^jes-KX#~VbI|5pH2Rm1LIjP#QTt1qt zO;3Psd^GYqe95_8PE)aLK8oE8$R+kOq+NRCC@PJMJWJmvNe#>K=C)*Xn)(-!kuac2 ze+Q~0b-N?-5EZ#yYRIA-`BwFNOzvP|SZ~-gq?kyHe3x>S1@B#N&sh@8%vlz^C8sbt zEGH<{52i0`r22SzKP8WY88a?IwrpUShI8Z!WPZ$lI&z-gzSEI2c*AO@T!{l!DF>s- z)eshgr!k6SIV+|5D^@{RKP6|8RDTtk&ib2jih_^kERY($kFs@sy%we3riDjjFMe_s z2bbk64VFeG;%iQ&RDY7n49-~<9Gg=foERBN$+A@c!@g2EHBvn(w~;T=%-c|{gy=f6 zR$$SoCT1SS1_BFo0mHl*V1aT)evKdG@G7v;1}Cn!;T;R6LHQU}!riSP87q2k=YZZ>B3dc%2slr`UA=l_o zD)~AXqeHH*IqW-#s-fAmdwkuM3F%~&IS4&xTrE%Yl5aY@n*i4!oD3; z1lp80I7$h139E-*dgP=M=mKkX;Ugp|h;~x-cd-t304p)d&E#fwuI2I==HJIUlUjnq zV)DM2+^pXXv_9}a%(o|sl@yq)8Eq?FPAu+%BbBQ*d8q2!gTdCU20A<)Aqxv2L$k-X zXYFV*dr(W*Rtt3TIunqW*U-#-+X0>}C_(aEbwC|Lg5ukby@zJjUhvaPq3t3pJpx5oU< zs{f$JzqK+^m3LrM@G+FVA0_EKmd<7t-$9RWYvnlRZ;Z(=#N;goJ@gqER}Vg*$}erS zCaC@$jnlB~X#8p>FNX93j~ZN#)ic%YZ;UL*z{aW?c5vmRxQ6}mAklY1guwg@$gKum zwgj5ot=WM+>-22j!OCP+ez7gVnaAW8z0t#JVx#W@#t`4)(E=BuH)+*Ybrftj(BGON zTaB6MIPHht&n{*AJJuyBp^jJ(oYCR&A6=IVdcG9rOMqKQxdX$?UlUHNGjxw zo<@VxvvKGZH7eEja&A6q!JzkbyZzmKOrAs!eJ6;+__RU0wAl8L|Bi+t2T*`ZgB%EB zQy|<13FAMFoulAn<#Jrmqb6?jcQ@LA6Pnz((2Q)MhDO0)UdpCMheIAquB4^5vnx`6WeJT!_j#3 zIP);pf8d44&wGjfTprnY--W1zIH~YzXj9wt#25yB_rPg=u5SN3P3dzU$GjA>EfZqZ z*4X#B0|WMJu)X&Xv=To5E8VHcKpO*rZc`9qckXj_7IfXDa4_j@0w*&TwY>*rKJ##N zy%PFdsKMS;j~e(~^z{tI@5Q09DE9NZkdr1Zda>T5p5Q&4HlJ6gRRC<=8?cV*KeX;p9@<_j~3pcpN(-j&hn?NlLX$&Geuvn|T0FIfQYEQ?WGUGGFutG3pa z$fn+}t+9FzRgTcs*jR-%I=J>8L8}twc=kPTDV9rT!_b|;`(Z14ivS^Y-PBWQc@yiR zPWu4v(w+%kL9o0|W+A0_%=$!iofXXtHI|5BfqY}IVgUd3#O9Nt%a=#*@bdB0U08>) z-9FY1GX?dLI}Nx8absgV6}+48y6bV2f$JyaRbWE~$PW3#?Qyz}@U<_WVIIdkE&Zq& z#JF^V8kBYpuE$`Q#tLlGt`Rd3WJPx6@}TvJVXuwDedRU^+l$aX)^M~KR=P9v8#s`U zKZ5+n*n8SMm$lC(g6$6w$sMOzCo!vw9nYTP3TD*alPnKMvU4!34Og0OL1p@%u@cDJ zm=|kA^0id5W<~$VbY))#*Zk2YaQf=s2&d;B2C`o#GeT)N&y{Q#jBcg6o<%8XSDw`{ z&!$3O%$tCCoDsKE9Gi~`M$|^pDOfAd)#vhhe~Nlb!^(ExSQIpM6W;8AVLAFPDmG0D zo3|molsVMRl(Cc*#YsCiZ>79NunBK|4xyG)2&pe1BzAI)s#_?0064w2J71vx^+W2- zShUJ~`=C>NWrL~(%M)przq{q5WUY?2p^UAmn@fLnZD(rtwSB1sX;*v8N7lNC1KQ&< z@lAR`mrz6dV#ndbA~k#g7`CVAKZat+!kFdAMDQWf!pcef@{KCxC_;R%O6fp|{=&)$ zgb3N(g0AdAjP-J3vqGv**teas@lC63s(+idnV8WqV8yWTF^phGk;UA*k{ZH@(EgRQ z;SUIrsobF!Sk-bHc2||lg7nOfC^t3|C82sMw~b?`$WY+ifu_;^J+O>cjI55&mYb?1 z@z~J!>)Oanq8yJ!t>l9M#~6o@x!<&^qa*+l`b$M5ts6~}w^V^}+I7Fyu zB%av0KD=S8QfeR|(Mzg?+u#T@lrC`%VTay8OQ_IxA()6g`R9-eJVY2Zw!;RXqX%7r zaHJiv91vD7g|M9r>VjCtE>g1!b@Ui*RTpCVq|d)?`3O7M_$-KhFhy@S>lYf4aTA$G zg~dqWnzMne5tY_>uhMIi&$Ri@M%2I_HP5N&6Wv%3^k}}OWfQ^OdJ`6W;0n1GdzRaa zmWj}I)c};yCVd;&;K2v_a1AtF{WBq8`T~pw2xdWF3Of*Y1-jRWKY%!DN`kEZBLr@A zd>Cs`^u*Y1CFd=Q3SqAmj243&rWorXdk7i$_Dl7bkzhz-yZ#(+?bv)|Y16Djf$jn_ zBPs94)Jdve@EoYZ#B8Mvr3Mqdq!3d(&QN2NQZikRod90XxH|u*oUE|I%nef#Y(pwowl+Js?F2`~+*Ho$6?2g6q7e&2JviZ*Dkz<6~G_Q+vsf*N!II>4I79Om!8AYSi@_JBJu z(r4=-o?|a^ur}yOprXJ&K@Yxbkl;@d&VP1{L46x=gJy2Vn0LTg08?2Xza?rlHnqwY z4I>-3kQo-NVtov+y@YB%PF{GMccKpcBIw1L{HA5tH-N(myQ1&J$iR%#qyGaCEO?Oh zeoZPV-C6qk$EiOfs6OZmCUu(Z;Ch|39`-KHSH!x|GVBAijMDIQS4I+q?sS*yDS_#H znwsu%XtEufGMLE$I5u?R5J?tn__YeQzU^QHjBV2dY?g8jq7Zw~7pJ?Xh{1T}P%N4S zh9-}*cxhKp6dovkt6@6)h0f*WNz-)5ZL5h82_!phEH3jR*nR~;-NXB;?91XjJ6Dgu z+p%u%9rNG>q|X7p9^$&*5uQ34scMlO{&>n%QBGCYShXorJ#F$3%Q{$YtuVdA3?Frd z(I;SGa~XbB*n`U*48~X3-C=HpQNESTzmwEl6pr&8l3fYCm5AD~_ja6}yYQa|{Mi5Dr90rk>45oS7H{=RDj4AgBa)PqXDgF97HBoEp>$U)OK?S2yX z7;-_k@up{@X;|E2e7&+U4qljd;9 zVrto|UqYm=<(+T%Y{08@FVF)=2wOi#9vJ*cn7yp}?9BJlH3brFU~ew$+eQJn%qmx6 z!J%Ys-_k1WJS+ybk$C@Qs#e*fhcth)=G$lbJNx$N>F{== z5*jRn`YA*J$p6rb0D=DqPynMwdHw8A({yrhMS zZA{+o_HS28wttCjcTcPI^q)|C8SIcR>UBUWpphD&cB5-eTnE~!e>TC;i?@uRVIvB5%_+3~b-y+d&499g(#hIHM1yj-R)@&93Lbl)~qcCs7qUkE*g%8%u^% zVl&8N8NyI$wNvFVWLqID6ImRrM5ryM=G#;uJrPoga;&$h`CQF80c##MilJBJTEn?q znbLAW<{dDw*fTGh_GGwFqd_z`2-=~3`gU{$)Fnh3DNr+TB7tLe}H0~Wc}5mJL2VdyO>#9}BacFq0CKm8??P6dz3`vW;V71f zejNjip|sdI&*H7@OXEs{;b_o&h=`X^2;_XDqXDgUNPGYP?Px&r!g5~eYv2F+ypW*Y z^UqCE`fnlH>RM!!Em+!2v!41qO+)H*M;hk%p}gk#RC7DGcByz0PgHPcaKqRf?R@= zG;e01AX#Yy{nub406LxJjaQ3Z(EaH+kabxWrV?zky$KWu`X-11i2G~m5bwiKP0{g{ zCBP4uIGFaEdi3=m0+yVtw_m=|V#@}^bAge810q)~)fu$9s=OUq&qn)H0W)B7GFCpID^&=}9aER2DIrLdu}WyXAm^!m z7w^=OF6M*lCHMf$5q3%PZl@Xg)EIxoEJ?g*w)8`XAq zn;M$JTHwy$%TRsenQsD?ST5C1d?N%ceSdl%LPT+xWciw!hFUSn)!;*FO&avJ zdF)uj;nfM3KFr)7b+FEs%hqYil{?nQD<4bP$j$3jdK0TN)cI3zNFvo!?ZTNythjyS zV*ytE-1|q;}cMTU5ORD!g-u5FcFMyf}D))<-hO&C8Gmu4={H1 zThPdBL;~3QWF$Vg-#{Oi7f_bc(BQqmo+Iov$l>MWh~B1LajgH2T9czqn5z84S$!kl zBWmnC^6|{EQ(Wxwk&nf4*zR(`VTPrHqpDBGovuICPUzX?15JG=U~4AUY5)&`Vl326 z$D*ncQ)=vjRKJ2kI7^kR!1r2ec$Biyq0m8cf+4uDAI1;Dbf{Q+LgC3`p_5oj!BWT1 zzhuEbLS3K3LOO$+sISZ1-PaWH3t-N-omY0V^h`WeEWqC+73t(}f)8qu4Z2mjDIhLEg;6m(GEY;h*q*2+SxP1x~}F04rh;tEeZCg#B5T zmx@XvM~1?NzT&i%c1Ayvlor_Ey5Tqu@k1e2;EFnMM9i+N&~f4_M78ZT;|yk3Zv)8oi})!Zb8)|12_Qb;mG_MN6zL<0{Rcx(U!{|g?mNsLo|Y@LJk=Gf?D z81*4?FZlHms_VkBL!>=VY10OAwor?l=s$%2=@!tUOC%K=2pbv~{Emv9=$=L#JcRST zt-fLSnt05jIN07N|U&j125@=6Jykp$2= zX=f){O!A%N!$;7I0Ie(21(v4;ZJqe` zVY0*_lYL5PpAdYQD&pL88_ZZ(<2d0*>TXx(q{3UCy)TBj;k6T*a}U-B@<}WYOi+hI z_486LIQZy4!6e3!w9oB>On?(zw>uxrbt^SqNU!IIa5fElz37Ax7RbhK)ME>-F>tmL zSu!u?aoI9GBi2ZZ^?m)o-K?&mNo|zs^=9WfD+(a{Iv0hIZtFuM9- z9RGiXG4KsAKKK>h?!29z(zpW_9l@;nTBcBX(~6S5d1d?K`s?dIT%iaG!i2J2CIib)1N`@`BqQ> zGl>mIhbt|oQn@q`Te*-JC%{TUVIF|67ds%bh`mB5Sbj!00q}l(7}B%}t3GB$cp{eL zK(*ON^q&L$XU74jQtS=Hp;C zcS%Y-vf{K-qW6Y{8>r*2U@d8M{8z~O&pIB5j?+X2#PXh5;Q@k(g&g+^>0tJp0<2RK z0GtqgSW{A9Y~Y|3hV$ov}~5>XT?4K>4?Jh$Px+X;%m2`giH5m8X?h ztCLvkSCE+CQ&!{+3mD)W#OvjhR3Aj$oT*74<4oO(Q zyi3p@2s_O8yj$g5)_Rw~K0h0+#`WPg2#T7x4MDM*&lRgvBFF;8D*6G5I;cwf@8vR?$ zaSE^i>2!{5t zUN6la)0UL7;Og?oOemPB%^^R5?T(r_PpH4jKLI^v`vFur=X%l{W9N)2X-bjmkC}J~ z;gs3*nw6YP7Nyk!BE^AM;v%mi1KJtN+Jf;~O=7fCtFa7dtn6B2el#}wD~*xxI(OtO zoY}8{{8VoQv-oL9Gc0*~6x|ZN-#{9ivc4T?v{!NBx~kSIt%F)3i36*G&#;cujsAUN zCl0e!jnb5K5WZiD4htkYqBqTe1zxIm5KBTunYBcFN@Uf*>GWCU(Z7S&lzVQH>bGD> zlqzqRllnr3|9!+lK@tmK>Ry8x^UGL%pJ-#fbz$N~t-MSvcw8$gpRY8}D=Ow}qkoB! zJdhukQDtKYHu&)IWpX%o0aq?&ELL+L*XHGx&j;rz%?nTiL7=}mum+e zCGkj;Yo9EY>i+~8!jrH_B4H9=wYDyFlte^TCCxPc4(z|d1(+2%NFZh4!We9$qw82+ zu$ftEKxf4{DvlL|^~sQKbb$e`F6dATa23T-)xqEzO)Iu8H7^TBYBethh*$2xVaZ3~ zph^wb_O0_Hg#%Lnji8nnGI`&%5mkVl20otlVCz7$Z`deqoGnVbR%1}X8(GKcC38?& z(W!44RPSJqsA_AET%p-BOiPWHOIsMyE^<~fss|V1zKI7H(9ydmWMHm-_$PAqN+vnIJNc!R?=rKx}H zu=dCr3@>z?)l3Kb!N|nQ3&H}9gncv0Ymxh(2-dHoeEM}Lr%_7QHYj;?smYzZ?RIw$ z(V}>|r?qM({KAVTRHZ3zlgUgVXb)Y3wkLAXoRSJMJO@I>(zGI0{_^UjKok;05=5uO-kC^L2PP7C=CDG|r5{x8y7k zq40&zk73dZbH}}iil7oYiitv@#yQtOk!ehf`HvcRJZSki5Y!QQ0vUM)HstL9^aB;K ze3bq6Vf6uAv(w4%{Dg=DfjF05kOn3GJ2)}nJ^2>C2Nbh+4uNYN)8cd~Y z!g9%s0jTJr8rcO4cPq4VtkD0dPM)2Y8oWtUy8BTZPN1oQA9`qy{1pCdM(}!-4}gUQ zB8%C};DZEPa31gHEi{P*7kzsl|AbNyuh|7@!4v?l%G+5*sxc)+ZbOUT&=#%jqDioT z!GSbJ*GCZf(K;r=4pw7V{YU9=9IiNn6u*9{fD3pEa+DfawjD~==<117CeWT;pbFtFK)kGN}n6lc6!w8qKUhrC`<)}$;hPXUP`cdnQ zx>L1iRqRkM4}lvbQisd&Z<4)Oq!TKvXhPCe_>VxH3pHEi5l!$51w$UhnG~d%2Ft!3 z1GscSNKP_Lp|`0sU0@@pT13|3c$iybooaC!z59)>{U*;x`KC{?-taovI{%n~SuYIJ zGO6xABAhHqYxvYSN5zgQAz&dv2?@#aTKa{}em-no`Y%4=jy-q4bEqGq(Z|en+IJvS z$TodmPeUl%ml{0K#OZnu?H0g@9hg!zocoN4S8@N#%W7aejQP=F>2?>)`o~ceX(;cX z5cb)L5BhAieBTefzQO4Aa2Oopf`eI+)3nVGG_}+22h7q^{Wj`Tc^XNfUm-|~LzryW zOSC*t+LTh%KVgY0?m@3(Qv~SZs){D#|d%$`y7~BX^$f~6h~vG4wmV})S4Ut{3)MgoPwY=AQUhg1HaoIC&$G=g0~-t<01|ef z!nCb~-DP;IW8c}z%F}?(>$VoI;Spvk+Cp%IYiahO&TvGVXvCLH1JSn^s~c%Tg@3Vn zCzdmEyqfEc!kbaKnkpx$Pm@ZKp6eCcAKlvogv9vJs_Wi_3iWZ@X?!i82qMhGem%B4 zGY?~-H?L3-asjI2GAvTC>RVd|G;Bq$N?FRx-01tD$=!5L;|A0{fheZE{wB7IuL?ixA$9^SBLD*Y5Fl>&V)ZuiVI1ny9$w^fa&ao*B#C?u~$y$$` zLWdmK3l1ZjcX{iVcuiNaZHy&^*b6MALal=HiZ=aekzGzNbfs>b=eoPGD27)~qoWk# z{7zh8BC-SN$O{*d>6nMCtTTE-^BcEZ9_ea;_qw%Hk6r=|2jf!8u_-QW?OI)!=M8t; z(pX_xp*)6jT@os`Mn(hGMmN22Xdc63LM}dAtCJOUJ}z>Wmb6yaSl|$ycLpOE4&-QK zMXys(z^mDx$B0Xzp5pD`zH~=E(d>6w3V}*=1DPC}{myI*pvn}K;t3dqv)L8*fri8D z$8pBz$T=m#TeY9gBMdqZi3|q}F;Gm)85Vu5pT>hYjd6?wf?LaJ0Cfd*3m=l|FsiN5 z{`A1OwXtxj2v}0oO*~WjP?Cp9upN2gKniEXjz)mDWm-XOEat~{;7!5#{on%_GOmFr z7}YNYP%fnr#6i(q7v|KYw3zJ1&Nr5dbv2l z);-IDNTp*Ab^A!f_50x{-9`FcuadZ1J09%Z1_)B2N2pn|uzJ>}C;kr-;4c9GD{W>B zH*r7*zyiA+X9ikX4|oA&)ar-CXsk|mjVC7naJ1^1fL9J20q}CDzA1QlRXGcz5w_zT zyr8xf;+3JwMR-k7{qs1{*s&uYYgHrwrmOx1M^0%K6fU3>*VQ8WX~k;l zV;J<}v;y0aQ{s6L+71S@I+QT6oqg;)Q~|$fZOU^SM{b;nC1Dkoge^xP9^J@Re^?%Z z{R^uL==4TTVu)XCbx8HMLL5lD2njY^T-Vrvg)@jX#RGsVQ_z|S3Ruyc*_bVVz+wt% z#BIC<CuD$Nd5V+FGKRDq8vw9$@|aM*XE_7vU^Q~(7beV!}Jb4=RsD26s9 zStmwbPF0HX2V?m+NZU-yzsNBf`13fOu*%6qaDw}Q1qsM*lazPCb#8=!bFn>Ub>(hM zR;Rcchrs>&T;$*I(SspY83b@Nt44n!AOHS0u9Y5yh;*0tFLCHBd+1#v!P# z(S{M?^t8*LqOi?_3MwC>YTSsy(iU`iaVnUyG|m%6FNBUmK11L6h-9tN+X1TM8mN^w ze;b(WJTQ?0!rJ1h#%`P;QriXM#u;&q1=yFiFpTe3%f@6J11@vfEYV~5PMHO%-U8g9 z0CMxvK^uUNe1`rgR#)7{IdP3eago0v3S_g+bP!|XG_f&P+&C?+F*i=Du&^mEZJLEa zI&7R~Y0S0o1`I%`^C|3n^gXjF(u_Awvo_`u2SUjnq~UTKv{s9BEjJ#)jrYel&WMjz zTx|8qH}lB^R){XvIs;?IE*=AFz=qo+K@x+TxgCPs2a8g}KEpOy`0WKT+_pzHVM?z( zL{w<9B}ty?i@TE#DGYts z#=xN{V!s7%Z9U1_R5lez)i7zTv19&nDdBirjIzdx@OW((Y0a?LRya@ymcK0K&r$Xj zaZv~VnIas@s@wbHvD_aF$4Tt^T^*2G#&HsO*l1-qW>a9r&6-$$;;7y|9!K?vS)Q{M z_RPcg!w>q>pKoc6CVP%U!{~_~!+jJT=6M+S|Nn-n&PLS`p#4YA9^A!`pE!7ju+fC! ztNUO-hOZZn11+)PuqkNLFN%w)q?!pgX-`L0I(ygs0X$;e5p|?^hJh_w(Q*A*OewhE zk`6Si#4^UNt;DsT$mY3RmFB*TqT&@bLG0B_ujN}u`Iz44Ei+dy;J`k6E0kWi-2M1~ z;cI&xl)r?Tmmgd1bFZn62MbXYlOCMI$+{v$im-0L zK3|w3f*yts6O6IQ@Hbo#Pf55IUI{&ilLl(&IUEniAq6#2{|}l@5;SZy!gq2CHpiSj$HXFXDtZT-~v)g zR`(T)=4BG2|5ghQmUS_pA<)6*+trb<8pDC9g9W-&bjWLuxIkOhrnJGy z0$z4Vh1)?#Gf^G>`e#8T;w|*M=))|O#{GhlDo4AE?(ye94u~qn-)Yn zvCW}{PGh1Q0aoH}`dH(3P>w45g6Jn2_U~YlmUUW)KY|%jP4MJdWr<$^t)sHf;K>>N zn@0CRH^MW3Zuw7DZJq);&B7rg?k8y&X z8?d(NzkwYH@C4xWr4ED7egGa4I7e%PJ$KL_k?inX(H7}C=Qo3@F^b zirz=V>6MBr4mZ;P5iH|G)hF2b`9qpIQG`B2z~|7ZXu)Hn<>c7}W-??t*J`$tr^ z!tUv;wvzJBI>}oFyQzrWeBG7EsInaB$3c*wBa%lT-o@l@+%-}VJx=CO!Ufs-(&45A zIG}LO5Sjmkrv(?ak^!+jKRtAU?y}f|1B@2xO;38L-4n*XH8`Lhc?dlO`wsoRcX37p zIl&f{!{`ze+|7DIe;GYeA+q)nKb!!s7E%JPrxBHAV6T}lP;NDlnjJb?Uuu*APY{fBIWcdZML>}Y=@t~SP+%*!K z8Sa@U)~Y8l?XQ=f-U#GE!b_|LSKA%Po@tYwCQSgG7x|duy_xjg}x)qI!0yzH+$O?(W4UG&so^Ie5g`JB!kYAf3;@UbKG(>mL8yHckuzU~)N&(AA{ayHIE{OaW1_kc_eKz`Y*3)Mb zovXk0uQZ2Vf>j0=r{jPKj4wE!@f>S?0H+FWU{6?Qvt_uZ!jAjg*o>6DKU8l}pRlUS zY-)vFUE_#7kwQMQWxSIALM5zC|MTx5DCq{aubiTzjQ8=33>is%qU-d&M0@%h0FZ2t zH8TdfoomK1y#kk3Osa(=HXGjQt4BacJ_U649gBm65u(FPKiP1e6H#nR@-KFdBE=+&pe&mQpKS#IZ5YM`g(^v7h zwu|!qp$9)$Mqp2KX{_!a6st%4JDOpXgG1qI_6aBhYz2;0oC z6=Ay>jz`#Gh7%A@F~f-nr@i@52YDsZnQE0#x^H{S{7JJEdG%y*LcPB!0z z%(vZq4>sRJ%(p~urOmd-ib(7&CDn~a;0loo6BGa5#a8^2C;m%2lV_ZOWOtZZe@_QUnFr`#{^Xv||vicQxYWRGL2JZSuvWwv8vUE%o$G zJnU6^o$?vnI%>n=0)8!}O2ZwC_ydp_>#4N~F6`Fu?G#;KiG)jGGzN1yJC7WBl%uT^ zt1@q7*oyzz)2V0TC}O96zkH6>Z=QhR zm|J&+OxyCGOEH6{=Wvly$yI~??gjznbiRaj+8z#Wt zD|0_g7FO{?-24fr?#b4-wVX_haaKP{fT17q=D$I4_tGlKTU>sqK(-DNtDC2vz&t$-5=xau2S3@a&snf`FXl7}I@FDm9yt-1P5KOwX& zqEig!SEw9YF?uwGy481lGDZLV$CPm@GV&{fNw)oxP!Lu(Qwy}Ssh88J>&4b8*+89_ z+OUuif~k3pt#=mVy4u6kR|AzXyDR!gVK~y2$SC$F?k-u66|023it$4lwgmqN6+<L0~`3SN9+vYAKlvWJV!>#E0c@AoV;Z>N8c8sOXPrx&ckI{PnC9JAl6N;=cto* z9-CaxF)+D)H{mQY6NH9ok6lte>9Sx(Qa&C7YSN@)#jbG>_B~ko(xIa$mn~I(GTL(c z|Kz+cSzAAa>?VD2vLcmC%^;U*eSUKNs}rzC$<9kVFHWw1eF7ZLFj(vqgxw+f@1(9M zr_wOfq_~t*mf&ddN`7O@owLAT`#FPgrtut?BbUr;cT>0~6&VekG?IcJ{Xd$)$}Owl zSgO_05y8Dp0VnQg(|^Fz=Odk;h{VN|_0Ok*8F4i9;%4Ih9DHIOP&@R6u$9oA%5+0B z?tkX#`u;yqqA+pgDq&*fs*NWvXaHO2&Ru986gobOP^S^ zD(GCgvUHWRGFTxm3d$8F*JWH-R<>%TJ^sQX@1j^1e zFzTFMa9j`uJVPIl|UwyA!Ix0FHl+H3@#~g-nwuSB`q(L7e85Yt8;0ovxL{S zc!IN}s$?awT;cQ;=ArDhzj;fRRsv7KMN2>is^l8dE6Yn3O>jPe8W%4uEiI`4nJQMT zaOPLYPb3e-@VGUy^NH2LlFEVf>nF?7l}m$57nYgqKqLYA0xK)!^72&`!IH%T8Z0ea z_++KCvLrawIc{uCvalY&ilQj^IVa%*r;l@C(3w#)E+ebPIlc_NGQw_9%qg#2SOpFu z3?>V+s!A%#7M7P2>zPGOMgu|#56&ee3m2DE41}5(rl?^09OvSN!G+)WVNFTJDrZH> zlc3yJzgMmXHjq1J4yaQ-2zZZNfYMm&Ws>0bxp0z7k(!#!n1L|5t=FGVh!`<-2!%FUzy=iA9S` zN@>ErmNqT-zUjU^A#>85lfEk~UAbsgMFp_d2Q&C|(Q-jvxqRiS>Xpt#t5%c~#%RQ@ zC`^s08l$R{L_JtQA0Ro9ctX#5k@x)c76FCh1#mklkODYzY-SJIT_W{F4*0+=Z zA{SrlW?zYKNu>^#F061qu@IEFMj&ELr$L@;i6%~rHe4XHhWxx?1hw^nOKbp#?4vkNw@(Cm2 z7gG7h9r0EwZ%K*Ir1BZV;s;ZC`^fkfUjA`xlnAyWJPsknRw)cigmP{x$CDulNzH;I z5p%!oaXe1MU&5R6*W&Ra-i7Z_2y6@Rq$2*DFg!6E;Tk+yh<6A>6EQ8@mf#tWI1INo zRA6hwQ-F96%0rd0&BK!-jIjM3<Cp7vZ&dau7d({)|Gn49^6_oA9>d*@UMM@m`dNer%hAXBgW5HQLASlC1&{ zTtSldp?!Fr*`C02E8;KX4Sn18G@e|Tx*2N7+>ct#3dbY*zrWQ42m+=cH4(LQ{eY!F4Z?;ySnZ^HKm zJb8$BqkIy=xZBkRFTL+*?;<*L!f5RKPn5_=aG{jHgJHhh+9@vhOerVEvHR2w`51aH~ ziZ}#!(tqJi@G?C2BYqy`iJk#GBLs&nPBiE*BmP}{KVZ_o1o3f*|0mvruLe9m#Lu8S z(f<)V@Hb8RnMwZ;;&&tdXOsRb5Qm{N=}o+epENvv#IK+{(Vuu`IQqZWq<BlDh{{``E#E+TuUyeBVI%y}~#7`=o?<0N@ z<*7gO@r)Wk|7yg)hwq0>`ag;IZHWI4Z))!uJo$)!hVq2Z$M9g>CjF~P|MiI9i};5o z{Z}D=2jXwzP5kr&JhKqLYSRDP#(xXyC;C5*$BF#E!dt*oi^q%jNBBKiw|2>obrO5Lg zY~jrd{(ouGzi<5i z&7}V-`;LqNmW)XbUyWHN?Dy*iVUB#&lLq&8V$`zR8$@F*!42@@CjL zrf6JKGoERfmbr)fL;u-0(pbu@z{Zotl*VOab9GbS8K3wb#=FBhM_6Ntb&WEnN~C8v z(d@h~>567`M{|0rRc$ivx9tBJSn(+J`?j2q^M7x4VR1R|=N~M2&FSgOlNTG!hIgwS zJoIKdeTo?dm|VVa#2GnSbSu^I$6eC%jvp6U?O<7K8Fbr^Y|gB>B<}{nLU3q}-^6Ka zV)v;^dp&>7XBXh0Ue$dpI7&O%f`dZbmgtn1w(cN=enr}-OMl=OkF%_7z32EXSb}ZX z*_1w5+dFBO*B6C*!G3AG?g$a#^d)hHhV3Ma&h0Eq)=}mYuoFwvpk>JtPe|hAWc9RNRzZva2;+# z2cCev_MEs^;T5c5TZfmx)}4wg+@o*;YT!W~8ZZSn;4Un}8mz->*nm&)8NR|6Y{L%h L!VlPkeX#Nmn``P+ literal 0 HcmV?d00001 diff --git a/tests/test_accessor.py b/tests/test_accessor.py index 8c892845..0e8441a5 100644 --- a/tests/test_accessor.py +++ b/tests/test_accessor.py @@ -1,21 +1,46 @@ import unittest +import hashlib +from tempfile import NamedTemporaryFile + +from parameterized import parameterized_class import xcp.accessor +@parameterized_class([{"url": "file://tests/data/repo/"}, + {"url": "https://updates.xcp-ng.org/netinstall/8.2.1"}]) class TestAccessor(unittest.TestCase): - def test_http(self): - #raise unittest.SkipTest("comment out if you really mean it") - a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) - a.start() - self.assertTrue(a.access('.treeinfo')) + url = "" + + def setup_accessor(self, arg0): + result = xcp.accessor.createAccessor(self.url, True) + result.start() + self.assertTrue(result.access(arg0)) + return result + + def test_access(self): + a = self.setup_accessor('.treeinfo') self.assertFalse(a.access('no_such_file')) self.assertEqual(a.lastError, 404) a.finish() - def test_file(self): - a = xcp.accessor.createAccessor("file://tests/data/repo/", True) - a.start() - self.assertTrue(a.access('.treeinfo')) - self.assertFalse(a.access('no_such_file')) - self.assertEqual(a.lastError, 404) + def test_file_binfile(self): + binary_file_in_repo = "boot/isolinux/mboot.c32" + a = self.setup_accessor(binary_file_in_repo) + inf = a.openAddress(binary_file_in_repo) + with NamedTemporaryFile("wb") as outf: + while True: + data = inf.read() + if not data: # EOF + break + outf.write(data) + outf.flush() + hasher = hashlib.md5() + with open(outf.name, "rb") as bincontents: + while True: + data = bincontents.read() + if not data: # EOF + break + hasher.update(data) + csum = hasher.hexdigest() + self.assertEqual(csum, "eab52cebc3723863432dc672360f6dac") a.finish() From 97be8f736ff39a72628c3fd47f84c97146af94f2 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 25 Apr 2023 20:19:22 +0200 Subject: [PATCH 23/30] CI: also run tests with python3 Comment from Yann: diff-cover defaults to origin/main in new version, it seems. Originally by Yann Dirson, Updates by me: - Fix the GitHub action deprecation warning caused by the use of old action repo version which use the deprecated node.js 12 - Rename the variable "pyversion" to the upstream "python-version" - Remove the installation of pyliblzma which is not used. - Add branding.py to fix GitHub CI with Python3 Co-authored-by: Yann Dirson Signed-off-by: Bernhard Kaindl --- .github/workflows/main.yml | 39 ++++++++++++++++++++------------------ branding.py | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 18 deletions(-) create mode 100644 branding.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53462463..3c2fd0c5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,48 +3,51 @@ name: Unit tests on: [push, pull_request] jobs: - test_py2: - runs-on: ubuntu-20.04 + test: + strategy: + matrix: + include: + - python-version: '2.7' + os: ubuntu-20.04 + - python-version: '3' + os: ubuntu-latest + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python 2.7 - uses: actions/setup-python@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: '2.7' + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - # FIXME: branding.py still has no permanent home - curl https://gist.github.com/ydirson/3c36a7e19d762cc529a6c82340894ccc/raw/5ca39f621b1feab813e171f535c1aad1bd483f1d/branding.py -O -L - pip install pyliblzma pip install -e . - command -v xz - name: Test run: | pytest --cov -rP coverage xml - coverage html - coverage html -d htmlcov-tests --include="tests/*" - diff-cover --html-report coverage-diff.html coverage.xml + coverage html -d htmlcov-${{ matrix.python-version }} + coverage html -d htmlcov-tests-${{ matrix.python-version }} --include="tests/*" + diff-cover --compare-branch=origin/master --html-report coverage-diff-${{ matrix.python-version }}.html coverage.xml - name: Pylint run: | pylint --version pylint --exit-zero xcp/ tests/ setup.py pylint --exit-zero --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" xcp/ tests/ setup.py > pylint.txt - diff-quality --violations=pylint --html-report pylint-diff.html pylint.txt + diff-quality --compare-branch=origin/master --violations=pylint --html-report pylint-diff-${{ matrix.python-version }}.html pylint.txt - uses: actions/upload-artifact@v3 with: name: Coverage and pylint reports path: | - coverage-diff.html - pylint-diff.html - htmlcov - htmlcov-tests + coverage-diff-${{ matrix.python-version }}.html + pylint-diff-${{ matrix.python-version }}.html + htmlcov-${{ matrix.python-version }} + htmlcov-tests-${{ matrix.python-version }} diff --git a/branding.py b/branding.py new file mode 100644 index 00000000..88914002 --- /dev/null +++ b/branding.py @@ -0,0 +1,36 @@ +BRAND_CONSOLE_URL = "https://xcp-ng.org" +BRAND_CONSOLE = "XCP-ng Center" +BRAND_GUEST_SHORT = "VM" +BRAND_GUESTS_SHORT = "VMs" +BRAND_GUESTS = "Virtual Machines" +BRAND_GUEST = "Virtual Machine" +BRAND_SERVERS = "XCP-ng Hosts" +BRAND_SERVER = "XCP-ng Host" +BRAND_VDI = "" +COMPANY_DOMAIN = "xcp-ng.org" +COMPANY_NAME_LEGAL = "Open Source" +COMPANY_NAME = "Open Source" +COMPANY_NAME_SHORT = "Open Source" +COMPANY = "Open Source" +COMPANY_PRODUCT_BRAND = "XCP-ng" +COMPANY_WEBSITE = "https://xcp-ng.org" +COPYRIGHT_YEARS = "2018-2022" +ISO_PV_TOOLS_COPYRIGHT = "XCP-ng" +ISO_PV_TOOLS_LABEL = "XCP-ng VM Tools" +ISO_PV_TOOLS_PUBLISHER = "XCP-ng" +PLATFORM_MAJOR_VERSION = "3" +PLATFORM_MICRO_VERSION = "1" +PLATFORM_MINOR_VERSION = "2" +PLATFORM_NAME = "XCP" +PLATFORM_ORGANISATION = "xen.org" +PLATFORM_VERSION = "3.2.1" +PLATFORM_WEBSITE = "www.xen.org" +PRODUCT_BRAND = "XCP-ng" +PRODUCT_BRAND_DASHED = "XCP-ng" +PRODUCT_MAJOR_VERSION = "8" +PRODUCT_MICRO_VERSION = "1" +PRODUCT_MINOR_VERSION = "2" +PRODUCT_NAME = "xenenterprise" +PRODUCT_VERSION = "8.2.1" +PRODUCT_VERSION_TEXT = "8.2" +PRODUCT_VERSION_TEXT_SHORT = "8.2" From b51d1794c0617f69a63ec2bdaec52bab60480099 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Tue, 25 Apr 2023 12:00:00 +0200 Subject: [PATCH 24/30] Add generated Python typing stubs to allow for better mypy checks Without these generated API descriptions, we'd have to disable all checks which use these APIs because we'd have to typing info for them. Signed-off-by: Bernhard Kaindl --- stubs/branding.pyi | 43 +++++++++++ stubs/parameterized/__init__.pyi | 1 + stubs/parameterized/parameterized.pyi | 76 ++++++++++++++++++++ stubs/parameterized/test.pyi | 100 ++++++++++++++++++++++++++ 4 files changed, 220 insertions(+) create mode 100644 stubs/branding.pyi create mode 100644 stubs/parameterized/__init__.pyi create mode 100644 stubs/parameterized/parameterized.pyi create mode 100644 stubs/parameterized/test.pyi diff --git a/stubs/branding.pyi b/stubs/branding.pyi new file mode 100644 index 00000000..0a3d72c4 --- /dev/null +++ b/stubs/branding.pyi @@ -0,0 +1,43 @@ +""" +python type stup for automatic type checking using mypy, prright et al. + +Generated from tests/branding.py using mypy's stubgen tool: +stubgen tests/branding.py +https://mypy.readthedocs.io/en/stable/stubgen.html?highlight=paths#automatic-stub-generation-stubgen +""" +BRAND_CONSOLE_URL: str +BRAND_CONSOLE: str +BRAND_GUEST_SHORT: str +BRAND_GUESTS_SHORT: str +BRAND_GUESTS: str +BRAND_GUEST: str +BRAND_SERVERS: str +BRAND_SERVER: str +BRAND_VDI: str +COMPANY_DOMAIN: str +COMPANY_NAME_LEGAL: str +COMPANY_NAME: str +COMPANY_NAME_SHORT: str +COMPANY: str +COMPANY_PRODUCT_BRAND: str +COMPANY_WEBSITE: str +COPYRIGHT_YEARS: str +ISO_PV_TOOLS_COPYRIGHT: str +ISO_PV_TOOLS_LABEL: str +ISO_PV_TOOLS_PUBLISHER: str +PLATFORM_MAJOR_VERSION: str +PLATFORM_MICRO_VERSION: str +PLATFORM_MINOR_VERSION: str +PLATFORM_NAME: str +PLATFORM_ORGANISATION: str +PLATFORM_VERSION: str +PLATFORM_WEBSITE: str +PRODUCT_BRAND: str +PRODUCT_BRAND_DASHED: str +PRODUCT_MAJOR_VERSION: str +PRODUCT_MICRO_VERSION: str +PRODUCT_MINOR_VERSION: str +PRODUCT_NAME: str +PRODUCT_VERSION: str +PRODUCT_VERSION_TEXT: str +PRODUCT_VERSION_TEXT_SHORT: str diff --git a/stubs/parameterized/__init__.pyi b/stubs/parameterized/__init__.pyi new file mode 100644 index 00000000..aed931d0 --- /dev/null +++ b/stubs/parameterized/__init__.pyi @@ -0,0 +1 @@ +from .parameterized import param as param, parameterized as parameterized, parameterized_class as parameterized_class diff --git a/stubs/parameterized/parameterized.pyi b/stubs/parameterized/parameterized.pyi new file mode 100644 index 00000000..23dba7d7 --- /dev/null +++ b/stubs/parameterized/parameterized.pyi @@ -0,0 +1,76 @@ +from _typeshed import Incomplete +from collections import OrderedDict as MaybeOrderedDict +from typing import NamedTuple + +# MaybeOrderedDict = dict + +class SkipTest(Exception): ... + +PY3: Incomplete +PY2: Incomplete +PYTEST4: Incomplete + +class InstanceType: ... + +lzip: Incomplete +text_type = str +string_types: Incomplete +bytes_type = bytes + +def make_method(func, instance, type): ... +def to_text(x): ... + +class CompatArgSpec(NamedTuple): + args: Incomplete + varargs: Incomplete + keywords: Incomplete + defaults: Incomplete + +def getargspec(func): ... +def skip_on_empty_helper(*a, **kw) -> None: ... +def reapply_patches_if_need(func): ... +def delete_patches_if_need(func) -> None: ... + +class _param(NamedTuple): + args: Incomplete + kwargs: Incomplete + +class param(_param): + def __new__(cls, *args, **kwargs): ... + @classmethod + def explicit(cls, args: Incomplete | None = ..., kwargs: Incomplete | None = ...): ... + @classmethod + def from_decorator(cls, args): ... + +class QuietOrderedDict(MaybeOrderedDict): ... # type: ignore + +def parameterized_argument_value_pairs(func, p): ... +def short_repr(x, n: int = ...): ... +def default_doc_func(func, num, p): ... +def default_name_func(func, num, p): ... +def set_test_runner(name) -> None: ... +def detect_runner(): ... + +class parameterized: + get_input: Incomplete + doc_func: Incomplete + skip_on_empty: Incomplete + def __init__(self, input, doc_func: Incomplete | None = ..., skip_on_empty: bool = ...) -> None: ... + def __call__(self, test_func): ... + def param_as_nose_tuple(self, test_self, func, num, p): ... + def assert_not_in_testcase_subclass(self) -> None: ... + @classmethod + def input_as_callable(cls, input): ... + @classmethod + def check_input_values(cls, input_values): ... + @classmethod + def expand(cls, input, name_func: Incomplete | None = ..., doc_func: Incomplete | None = ..., skip_on_empty: bool = ..., **legacy): ... + @classmethod + def param_as_standalone_func(cls, p, func, name): ... + @classmethod + def to_safe_name(cls, s): ... + +def parameterized_class(attrs, input_values: Incomplete | None = ..., class_name_func: Incomplete | None = ..., classname_func: Incomplete | None = ...): ... +def unwrap_mock_patch_func(f): ... +def get_class_name_suffix(params_dict): ... +def default_class_name_func(cls, num, params_dict): ... diff --git a/stubs/parameterized/test.pyi b/stubs/parameterized/test.pyi new file mode 100644 index 00000000..aab4e91f --- /dev/null +++ b/stubs/parameterized/test.pyi @@ -0,0 +1,100 @@ +from .parameterized import PY2 as PY2, PY3 as PY3, PYTEST4 as PYTEST4, SkipTest as SkipTest, detect_runner as detect_runner, param as param, parameterized as parameterized, parameterized_argument_value_pairs as parameterized_argument_value_pairs, parameterized_class as parameterized_class, short_repr as short_repr +from _typeshed import Incomplete +from unittest import TestCase + +def assert_contains(haystack, needle) -> None: ... + +runner: Incomplete +UNITTEST: Incomplete +NOSE2: Incomplete +PYTEST: Incomplete +SKIP_FLAGS: Incomplete +missing_tests: Incomplete + +def expect(skip, tests: Incomplete | None = ...) -> None: ... + +test_params: Incomplete + +def test_naked_function(foo, bar: Incomplete | None = ...) -> None: ... + +class TestParameterized: + def test_instance_method(self, foo, bar: Incomplete | None = ...) -> None: ... + +class TestSetupTeardown: + stack: Incomplete + actual_order: str + def setUp(self) -> None: ... + def tearDown(self) -> None: ... + def test_setup(self, count, *a) -> None: ... + +def custom_naming_func(custom_tag, kw_name): ... + +class TestParameterizedExpandWithMockPatchForClass(TestCase): + def test_one_function_patch_decorator(self, foo, mock_umask, mock_getpid) -> None: ... + def test_multiple_function_patch_decorator(self, foo, bar, mock_umask, mock_fdopen, mock_getpid) -> None: ... + +class TestParameterizedExpandWithNoExpand: + def test_patch_class_no_expand(self, foo, bar, mock_umask, mock_getpid) -> None: ... + +class TestParameterizedExpandWithNoMockPatchForClass(TestCase): + def test_one_function_patch_decorator(self, foo, mock_umask) -> None: ... + def test_multiple_function_patch_decorator(self, foo, bar, mock_umask, mock_fdopen) -> None: ... + +class TestParameterizedExpandWithNoMockPatchForClassNoExpand: + def test_patch_no_expand(self, foo, bar, mock_umask) -> None: ... + +def test_mock_patch_standalone_function(foo, mock_umask) -> None: ... + +class TestParamerizedOnTestCase(TestCase): + def test_on_TestCase(self, foo, bar: Incomplete | None = ...) -> None: ... + def test_on_TestCase2(self, foo, bar: Incomplete | None = ...) -> None: ... + +class TestParameterizedExpandDocstring(TestCase): + def test_custom_doc_func(self, foo, bar: Incomplete | None = ...) -> None: ... + def test_single_line_docstring(self, foo) -> None: ... + def test_empty_docstring(self, foo) -> None: ... + def test_multiline_documentation(self, foo) -> None: ... + def test_unicode_docstring(self, foo) -> None: ... + def test_default_values_get_correct_value(self, foo, bar: int = ...) -> None: ... + def test_with_leading_newline(self, foo, bar: int = ...) -> None: ... + +def test_warns_when_using_parameterized_with_TestCase() -> None: ... +def test_helpful_error_on_invalid_parameters() -> None: ... +def test_helpful_error_on_empty_iterable_input() -> None: ... +def test_skip_test_on_empty_iterable() -> None: ... +def test_helpful_error_on_empty_iterable_input_expand() -> None: ... +def test_wrapped_iterable_input(foo) -> None: ... +def test_helpful_error_on_non_iterable_input(): ... +def tearDownModule() -> None: ... +def test_old_style_classes() -> None: ... + +class TestOldStyleClass: + def test_old_style_classes(self, param) -> None: ... + +def test_parameterized_argument_value_pairs(func_params, p, expected) -> None: ... +def test_short_repr(input, expected, n: int = ...) -> None: ... +def test_with_docstring(input) -> None: ... + +cases_over_10: Incomplete + +def test_cases_over_10(input, expected) -> None: ... + +class TestParameterizedClass(TestCase): + def test_method_a(self) -> None: ... + def test_method_b(self) -> None: ... + def testCamelCaseMethodC(self) -> None: ... + +class TestNamedParameterizedClass(TestCase): + def test_method(self) -> None: ... + +class TestParameterizedClassDict(TestCase): + foo: int + bar: str + def setUp(self) -> None: ... + def tearDown(self) -> None: ... + def test_method(self) -> None: ... + +class TestUnicodeDocstring: + def test_with_docstring(self, param) -> None: ... + +def test_missing_argument_error() -> None: ... From 1161e8e9be929995d5e764690832ea4f0e011424 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 28 Apr 2023 18:06:02 +0200 Subject: [PATCH 25/30] accessor: fix pylint: funcs shall use matching argument names Signed-off-by: Bernhard Kaindl --- xcp/accessor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 92df7b00..27700ea4 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -54,11 +54,11 @@ def __init__(self, ro): self.read_only = ro self.lastError = 0 - def access(self, name): + def access(self, address): """ Return boolean determining where 'name' is an accessible object in the target. """ try: - f = self.openAddress(name) + f = self.openAddress(address) if not f: return False f.close() @@ -315,11 +315,11 @@ def finish(self): self.cleanup = False self.ftp = None - def access(self, path): + def access(self, address): try: - logger.debug("Testing "+path) + logger.debug("Testing " + address) self._cleanup() - url = urllib.parse.unquote(path) + url = urllib.parse.unquote(address) if self.ftp.size(url) is not None: return True From 79d4b837c79e580e1d29d8764d1400faf93687b4 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 28 Apr 2023 18:07:31 +0200 Subject: [PATCH 26/30] accessor: apply trivial simplification Signed-off-by: Bernhard Kaindl --- xcp/accessor.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 27700ea4..04a0a655 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -286,10 +286,8 @@ def _cleanup(self): def start(self): if self.start_count == 0: self.ftp = ftplib.FTP() - #self.ftp.set_debuglevel(1) - port = ftplib.FTP_PORT - if self.url_parts.port: - port = self.url_parts.port + port = self.url_parts.port or ftplib.FTP_PORT + assert self.url_parts.hostname self.ftp.connect(self.url_parts.hostname, port) username = self.url_parts.username password = self.url_parts.password From f7f1c3dff0dbe9ab4842a45638a799857274142e Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 28 Apr 2023 18:12:28 +0200 Subject: [PATCH 27/30] accessor: apply suggestion to join duplicated code Signed-off-by: Bernhard Kaindl --- xcp/accessor.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 04a0a655..e7d8cc59 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -277,6 +277,11 @@ def __init__(self, baseAddress, ro): self.ftp = None self.baseAddress = rebuild_url(self.url_parts) + def _parse_ftp_url(self, debug_message, ftp_url): + logger.debug(debug_message + " " + ftp_url) + self._cleanup() + return urllib.parse.unquote(ftp_url) + def _cleanup(self): if self.cleanup: # clean up after RETR @@ -315,10 +320,7 @@ def finish(self): def access(self, address): try: - logger.debug("Testing " + address) - self._cleanup() - url = urllib.parse.unquote(address) - + url = self._parse_ftp_url("Testing", address) if self.ftp.size(url) is not None: return True lst = self.ftp.nlst(os.path.dirname(url)) @@ -340,10 +342,7 @@ def access(self, address): return False def openAddress(self, address, mode="rb", **kwargs): - logger.debug("Opening "+address) - self._cleanup() - url = urllib.parse.unquote(address) - + url = self._parse_ftp_url("Opening", address) self.ftp.voidcmd('TYPE I') s = self.ftp.transfercmd('RETR ' + url).makefile(mode, **kwargs) self.cleanup = True From 24e7b0e172fb1261f899461ddf0ede28a13e33d3 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Thu, 27 Apr 2023 17:44:52 +0200 Subject: [PATCH 28/30] Add tox.ini, pyproject.toml & mypy. Add pylint to the GitHub Action view Toxify the CI for not having to push to run CI, just run tox locally Signed-off-by: Bernhard Kaindl --- .github/workflows/main.yml | 69 +++++++++----- gen_gh_pylint_report.py | 178 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 131 ++++++++++++++++++++++++++ requirements-dev.txt | 3 +- setup.py | 2 +- tests/__init__.py | 2 + tests/test_cpio.py | 6 +- tox.ini | 118 ++++++++++++++++++++++++ xcp/cpiofile.py | 2 +- xcp/net/ifrename/dynamic.py | 2 +- 10 files changed, 486 insertions(+), 27 deletions(-) create mode 100755 gen_gh_pylint_report.py create mode 100644 pyproject.toml create mode 100644 tests/__init__.py create mode 100644 tox.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3c2fd0c5..0802e82b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,53 +1,80 @@ +# actions can be run locally using act and docker, on Fedora 37 also with podman, using: +# https://github.com/nektos/act +# sudo dnf install -y act-cli podman-docker +# act --bind --container-daemon-socket $XDG_RUNTIME_DIR/podman/podman.sock -W .github/workflows/main.yml + name: Unit tests +# Checks can be skipped by adding "skip-checks: true" to a commit message, +# or requested by adding "request-checks: true" if disabled by default for pushes: +# https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks#skipping-and-requesting-checks-for-individual-commits on: [push, pull_request] +env: + PYTHONWARNINGS: "ignore" + PIP_ROOT_USER_ACTION: "ignore" # For local testing using act-cli + PIP_NO_WARN_SCRIPT_LOCATION: "0" # For local testing using act-cli jobs: test: strategy: + # max-parallel: 1 # See: https://github.com/xenserver/python-libs/pull/26#discussion_r1179482169 matrix: include: + - python-version: '3.11' + os: ubuntu-latest + - python-version: '3.10' + os: ubuntu-latest - python-version: '2.7' os: ubuntu-20.04 - - python-version: '3' + - python-version: '3.9' + os: ubuntu-latest + - python-version: '3.8' os: ubuntu-latest + - python-version: '3.7' + os: ubuntu-latest + - python-version: '3.6' + os: ubuntu-20.04 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 0 # Needed by diff-cover to get the changed lines: origin/master..HEAD - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Provide Pylint Summary and Report on the GitHub Action's info page + if: ${{ matrix.python-version == '3.10' }} run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - pip install -e . + pip install -q pylint pandas tabulate 2>&1|grep -v root ||: # Hide warning for root in local container + tree=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/tree/$GITHUB_REF_NAME + [ -n "$GITHUB_STEP_SUMMARY" ] || GITHUB_STEP_SUMMARY=pylint-summary-table.md + ./gen_gh_pylint_report.py xcp $GITHUB_STEP_SUMMARY $tree - name: Test run: | - pytest --cov -rP - coverage xml - coverage html -d htmlcov-${{ matrix.python-version }} - coverage html -d htmlcov-tests-${{ matrix.python-version }} --include="tests/*" - diff-cover --compare-branch=origin/master --html-report coverage-diff-${{ matrix.python-version }}.html coverage.xml + rm -f coverage.xml + pip -q install tox tox-gh-actions + tox --recreate - - name: Pylint - run: | - pylint --version - pylint --exit-zero xcp/ tests/ setup.py - pylint --exit-zero --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" xcp/ tests/ setup.py > pylint.txt - diff-quality --compare-branch=origin/master --violations=pylint --html-report pylint-diff-${{ matrix.python-version }}.html pylint.txt + - name: Upload coverage reports to Codecov + if: ${{ matrix.python-version == '3.11' }} + uses: codecov/codecov-action@v3 - uses: actions/upload-artifact@v3 + # Skip coverage upload during local actions testing using act-cli: + # For logging the contents of the context, see: + # https://docs.github.com/en/actions/learn-github-actions/contexts#example-printing-context-information-to-the-log + if: ${{ matrix.python-version == '3.11' && !startsWith(github.actor, 'nektos/act') }} with: name: Coverage and pylint reports path: | - coverage-diff-${{ matrix.python-version }}.html - pylint-diff-${{ matrix.python-version }}.html - htmlcov-${{ matrix.python-version }} - htmlcov-tests-${{ matrix.python-version }} + coverage.xml + .tox/py311-cov/log/htmlcov/ + .tox/py311-cov/log/htmlcov-tests/ + .tox/py311-cov/log/coverage-diff.html + .tox/py311-cov/log/pylint-diff.html + .tox/py311-cov/log/pylint.txt + .tox/py311-cov/log/.coverage diff --git a/gen_gh_pylint_report.py b/gen_gh_pylint_report.py new file mode 100755 index 00000000..7b60b450 --- /dev/null +++ b/gen_gh_pylint_report.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +import json +import os +import sys +from glob import glob +from io import StringIO, TextIOWrapper +from typing import List + +from pylint.lint import Run # type: ignore +from pylint.reporters import JSONReporter # type: ignore + +import pandas as pd + + +def del_dict_keys(r, *args): + for arg in args: + r.pop(arg, None) + + +def cleanup_results_dict(r, sym): + del_dict_keys( + r, + "module", + "column", + "endColumn", + "message-id", + "endLine", + "type", + "line", + ) + r["symbol"] = sym[:32] + r["message"] = r["message"][:96] + try: + dotpos = r["obj"].rindex(".") + 1 + except ValueError: + dotpos = 0 + r["obj"] = r["obj"][dotpos:][:16] + + +suppress_msg = ["Consi", "Unnec", "Unuse", "Use l", "Unkno", "Unrec", "Insta"] +suppress_sym = [ + "attribute-defined-outside-init", + "bare-except", + "broad-exception-raised", + # "duplicate-except", + "super-init-not-called", +] +notice_syms = [ + "fixme", + "no-member", + "unexpected-keyword-arg", + "assignment-from-no-return", +] + + +def filter_results(r): + msg = r["message"] + typ = r["type"] + if typ in ["convention", "refactor"] or not msg: + return None, None, None + sym = r["symbol"] + return ( + (None, None, None) + if sym in suppress_sym or msg[:5] in suppress_msg + else (typ, sym, msg) + ) + + +def pylint_project(module_path: str, branch_url: str, errorlog: TextIOWrapper): + pylint_options: List[str] = [] + pylint_overview = [] + pylint_results = [] + glob_pattern = os.path.join(module_path, "**", "*.py") + score_sum = 0.0 + smell_sum = 0 + for filepath in glob(glob_pattern, recursive=True): + filename = filepath.rsplit("/", maxsplit=1)[-1] + if filename in ["__init__.py", "pylintrc"]: + continue + reporter_buffer = StringIO() + results = Run( + [filepath] + pylint_options, + reporter=JSONReporter(reporter_buffer), + do_exit=False, + ) + score = results.linter.stats.global_note + file_results = json.loads(reporter_buffer.getvalue()) + if not file_results: + continue + filtered_file_results = [] + filtered_messages = {} + linktext = filename.split(".")[0] + for r in file_results: + typ, sym, msg = filter_results(r) + if msg is None: + continue + print(typ, sym, msg) + if sym in notice_syms: + typ = "notice" + else: + filtered_messages[sym] = 0 + errorlog.write( + f"::{typ} file={filepath},line={r['line']},endLine={r['endLine']}," + f"title={sym}::{msg}\n" + ) + r["path"] = f"[{linktext}]({branch_url}/{filepath}#L{r['line']})" + cleanup_results_dict(r, sym) + filtered_file_results.append(r) + + pylint_results.extend(filtered_file_results) + smells = len(filtered_file_results) + smell_sum += smells + score_sum += score + + pylint_overview.append( + { + "filepath": f"[{filepath[4:]}]({branch_url}/{filepath})", + "smells": smells, + "symbols": " ".join(filtered_messages.keys()), + "score": round(score, 3), + } + ) + avg_score = score_sum / len(pylint_overview) + pylint_overview.append( + { + "filepath": "total", + "smells": smell_sum, + "symbols": "", + "score": round(avg_score, 3), + } + ) + return pd.DataFrame(pylint_overview), pd.DataFrame(pylint_results) # , avg_score + + +def main(module_dir: str, output_file: str, branch_url: str, errorlog_file: str): + """Send pylint errors, warnings, notices to stdout. Github show show 10 of each + + Args: + module_dir (str): subdirectory of the module, e.g. "xcp" + output_file (str): output file path for the markdown summary table + branch_url (str): _url of the branch for file links in the summary table + """ + with open(errorlog_file, "a", encoding="utf-8") as errors: + panda_overview, panda_results = pylint_project(module_dir, branch_url, errors) + + # print("### Overview") + # print(panda_overview.to_markdown()) + # print("\n### Results") + # print(panda_results.to_markdown()) + + # Write the panda dable to a markdown output file: + summary_file = output_file or os.environ.get("GITHUB_STEP_SUMMARY") + scriptname = __file__.rsplit("/", maxsplit=1)[-1] + mylink = f"[{scriptname}]({branch_url}/{scriptname})" + if summary_file: + with open(summary_file, "w", encoding="utf-8") as fp: + fp.write( + f"### PyLint breakdown from {mylink} on **xcp/\\*\\*/*.py**\n" + ) + fp.write(panda_overview.to_markdown()) + + fp.write( + f"\n### PyLint results from {mylink} on **xcp/\\*\\*/*.py**\n" + ) + fp.write(panda_results.to_markdown()) + + +if __name__ == "__main__": + default_branch_url = "https://github.com/xenserver/python-libs/blob/master" + default_error_file = "/dev/stderr" + + py_module_dir = sys.argv[1] if len(sys.argv) > 1 else "xcp" + gh_output_file = sys.argv[2] if len(sys.argv) > 2 else "pylint-summary-table.md" + gh_branch_url = sys.argv[3] if len(sys.argv) > 3 else default_branch_url + errorlog_path = sys.argv[4] if len(sys.argv) > 4 else default_error_file + + print(py_module_dir, gh_output_file, gh_branch_url, errorlog_path) + main(py_module_dir, gh_output_file, gh_branch_url, errorlog_path) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..653412c9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,131 @@ +[project] +# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ +name = "python-libs" +version = "1.0.0" +description = "Common XCP-ng Python classes" +requires-python = ">=2.7" +license = {file = "LICENSE"} +keywords = ["xcp-ng", "xen-project", "libraries"] +authors = [ + {name = "Simon Rowe"}, + {name = "Andrew Cooper"}, + {name = "Yann Dirson"}, +] +maintainers = [ + {name = "Ross Lagerwall"}, + {name = "Pau Ruiz Safont"}, + {name = "Bernhard Kaindl"}, +] +readme = "README" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: System :: Hardware", + "Topic :: System :: Installation/Setup", + "Topic :: System :: Networking", + "Topic :: System :: Systems Administration", + "Topic :: Software Development :: Libraries :: Python Modules" +] + +[project.optional-dependencies] +coverage = ["coverage", "diff_cover", "pytest-cov"] + +[project.urls] +homepage = "https://github.com/xenserver/python-libs/" +repository = "https://github.com/xenserver/python-libs/" + +[build-system] +requires = ["setuptools >= 38.6.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.mypy] +pretty = true +show_error_context = true +error_summary = true +files = ["xcp", "tests/test_*.py", "stubs"] +python_version = "3.6" +warn_redundant_casts = true +warn_return_any = true +warn_unreachable = true +warn_unused_configs = true +disallow_any_unimported = true +disallow_any_explicit = true +disallow_any_generics = true +disallow_subclassing_any = true +show_error_codes = true +strict_equality = true +# Check the contents of untyped functions in all modules by default: +check_untyped_defs = true + +# xcp.cmd is to be fixed in PR #22: +[[tool.mypy.overrides]] +module = ["xcp.cmd"] +disable_error_code = ["operator", "comparison-overlap"] + +# xcp.accessor is be fixed with #24: +[[tool.mypy.overrides]] +module = ["xcp.accessor"] +disable_error_code = "union-attr" + +# To be fixed with #24: +[[tool.mypy.overrides]] +module = ["xcp.net.biosdevname", "xcp.net.ifrename.logic"] +disable_error_code = ["var-annotated", "no-any-return"] + +# xcp.net.ip should be is to be fixed in PR #22, but the ip output parser works anyway: +[[tool.mypy.overrides]] +module = ["xcp.net.ip"] +disable_error_code = ["arg-type", "comparison-overlap"] + +# The blame list of modules with various errors/warnings in their untyped defs, +# it shuts up 65 mypy errors, most are in cpiofile: +[[tool.mypy.overrides]] +module = [ + "xcp.pci", # xcp.pci should be fixed by PR #22 + "xcp.cpiofile", + "xcp.repository", + "xcp.bootloader", +] +check_untyped_defs = false # enable to see the blame list +disable_error_code = ["var-annotated", "unreachable"] + +# Most of these should be easily fixable by adding type annotations as comments(PEP484): + +[[tool.mypy.overrides]] +module = ["tests.test_pci"] +disable_error_code = ["no-any-return"] + +[[tool.mypy.overrides]] +module = ["tests.test_mac"] +disable_error_code = ["var-annotated"] + +[[tool.mypy.overrides]] +module = ["tests.test_ifrename_logic"] +disable_error_code = ["attr-defined", "no-any-return"] + +[[tool.mypy.overrides]] +module = ["tests.test_bootloader"] +disable_error_code = ["no-any-return", "union-attr"] + +[[tool.mypy.overrides]] +module = ["tests.test_ifrename_logic"] +disable_error_code = ["attr-defined", "no-any-return"] + +# Due to special cases for unicode handline in Python2 which are not reached by Python3: +[[tool.mypy.overrides]] +module = "xcp.xmlunwrap" +warn_unreachable = false + +[tool.coverage.run] +# The coverage-comment-action needs a .converage file with relative path names: +# https://github.com/py-cov-action/python-coverage-comment-action#setup +relative_files = true \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index f5042a93..99d92b25 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ # necessary dev/test tasks pylint -coverage +coverage[toml] diff_cover mock pytest @@ -11,3 +11,4 @@ six # python-2.7 only configparser ; python_version < "3.0" +pyliblzma ; python_version < "3.0" diff --git a/setup.py b/setup.py index e7881222..11e0fa48 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ -from distutils.core import setup +from setuptools import setup setup(name='python-libs', description='Common XenServer Python classes', diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..02b47bd0 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +# Needed for mypy to see the tests as module in which we can suppress warnings +# using configuration in pyproject.toml. diff --git a/tests/test_cpio.py b/tests/test_cpio.py index fdb34f40..8592baf2 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -44,8 +44,10 @@ def setUp(self): os.utime('archive/data', (0, 0)) os.utime('archive', (0, 0)) - check_call( - "find archive | cpio --reproducible -o -H newc > archive.cpio") + try: + check_call("find archive | cpio --reproducible -o -H newc > archive.cpio") + except: + raise unittest.SkipTest("cpio tool not available") check_call("gzip -c < archive.cpio > archive.cpio.gz") check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") try: diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..cded1d55 --- /dev/null +++ b/tox.ini @@ -0,0 +1,118 @@ +# Managing a Project's Virtualenvs with tox +# A comprehensive beginner's introduction to tox. +# Tox is a great tool for standardising and automating any development task +# that benefits from being run in an isolated virtualenv: +# https://www.seanh.cc/2018/09/01/tox-tutorial/ +# To run the tests for all supported and installed python versions, run: +# pip3 install 'py>=1.11.0' tox; tox + +# Installation of additional python versions: +# - On Fedora 37, `sudo dnf install tox` installs all Python versions, even 3.12a6. +# - On Ubuntu, the deadsnakes/ppa is broken(except for 3.12), so conda or pyenv has to be used: +# sudo apt-get install -y build-essential libssl-dev zlib1g-dev libbz2-dev +# libreadline-dev libsqlite3-dev xz-utils libffi-dev liblzma-dev +# curl https://pyenv.run | bash # Then install the displayed commands in your .bashrc. +# pyenv install 3.{6,7,8,9} && pyenv local 3.{6,7,8,9} # builds and add them to PATH. +# https://brandonrozek.com/blog/pyenvtox/; https://realpython.com/intro-to-pyenv/ +# +# - Note: virtualenv 20.22 broke support for the py27 venv with tox. (at least in some setups) +# As a workaround, downgrade it to 20.21 if that happens: pip3 install -U 'virtualenv<20.22' +# - For testing on newer Ubuntu hosts which have python2-dev, but not pip2, install pip2 this way: +# curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py;sudo python2 get-pip.py + +[tox] +skipsdist = true +isolated_build = true +envlist = py27-test, py36-test, py37-test, py38-test, py39-test, py310-test, py312-test, + py311-mypy, py311-cov +# This results in this list of environments tox -e py312-fox -av +# default environments: +# py27-test -> Run in a python2.7 virtualenv: pytest +# py36-test -> Run in a python3.6 virtualenv: pytest +# py37-test -> Run in a python3.7 virtualenv: pytest +# py38-test -> Run in a python3.8 virtualenv: pytest +# py39-test -> Run in a python3.9 virtualenv: pytest +# py310-test -> Run in a python3.10 virtualenv: pytest +# py312-test -> Run in a python3.12 virtualenv: pytest +# py311-mypy -> Run in a python3.11 virtualenv: mypy static analyis +# py311-cov -> Run in a python3.11 virtualenv: generate coverage html reports +# +# additional environments: +# py312-fox -> Run in a python3.12 virtualenv: generate coverage html reports and open them in firefox + +# TODO later: Enable the venv_update extension so that changes to requirements-dev.txt +# are automatically detected. +tox_pip_extensions_ext_venv_update = true + +[testenv] +description = + Run in a {basepython} virtualenv: + test: pytest + cov: generate coverage html reports + fox: generate coverage html reports and open them in firefox + mypy: mypy static analyis + covcp: copy the generated .converage and coverage.xml to the UPLOAD_DIR dir for upload +passenv = PYTEST_ADDOPTS + PYTEST_XDIST_WORKER_COUNT + covcp: UPLOAD_DIR + covcp: HOME + mypy: TERM + mypy: MYPY_FORCE_COLOR + mypy: MYPY_FORCE_TERMINAL_WIDTH + fox: DISPLAY + fox: XAUTHORITY + fox: DBUS_SESSION_BUS_ADDRESS +deps = + {cov,covcp,fox,test}: -r requirements-dev.txt + mypy: lxml + mypy: mypy + mypy: types-mock + mypy: types-simplejson + mypy: types-six +allowlist_externals = + {cov,covcp,fox}: cp + mypy: cat + fox: firefox +commands = + {cov,covcp,fox,test}: pytest --cov + {cov,covcp,fox}: {[cov]commands} + covcp: cp -av {envlogdir}/coverage.xml {env:UPLOAD_DIR:.} + mypy: mypy --txt-report . + mypy: cat index.txt + fox: {[fox]commands} + +[cov] +commands = + coverage xml -o {envlogdir}/coverage.xml + coverage html -d {envlogdir}/htmlcov + coverage html -d {envlogdir}/htmlcov-tests --include="tests/*" + diff-cover --compare-branch=origin/master \ + --html-report {envlogdir}/coverage-diff.html \ + {envlogdir}/coverage.xml + pylint --output {envlogdir}/pylint.txt --exit-zero \ + --msg-template='\{path\}:\{line\}: [\{msg_id\}(\{symbol\}), \{obj\}] \{msg\}' \ + xcp/ tests/ + diff-quality --compare-branch=origin/master --violations=pylint \ + --html-report {envlogdir}/pylint-diff.html {envlogdir}/pylint.txt + cp -av .coverage {envlogdir}/.coverage + cp -av {envlogdir}/coverage.xml . + +[fox] +commands = firefox {envlogdir}/htmlcov/index.html \ + {envlogdir}/htmlcov-tests/index.html \ + {envlogdir}/coverage-diff.html + +# Map the github python versions to environments to testenvs to run on them: +# See https://github.com/ymyzk/tox-gh-actions for details: +# https://github.com/ymyzk/tox-gh-actions#tox-gh-actions-configuration +# The benefit of using tox is that all versions can be run locally and +# the local venvs will be the same as the venvs created by tox on the GitHub runners: +[gh-actions] +python = + 2.7: py27-test + 3.6: py36-test + 3.7: py37-test + 3.8: py38-test + 3.9: py39-test + 3.10: py310-mypy + 3.11: py311-cov \ No newline at end of file diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index fb4d96f7..8b6ddf4b 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -64,7 +64,7 @@ try: import grp as GRP, pwd as PWD except ImportError: - GRP = PWD = None + GRP = PWD = None # type: ignore # from cpiofile import * __all__ = ["CpioFile", "CpioInfo", "is_cpiofile", "CpioError"] diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index 33040ffd..cc5a211b 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -39,7 +39,7 @@ import json except ImportError: try: - import simplejson as json + import simplejson as json # type: ignore # mypy sees the import above # The installer has no json. In the meantime, there is a workaround except ImportError: pass From 5f5658f122f1a2f22f82177ef76f51f0cb01da53 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Sat, 29 Apr 2023 15:51:54 +0200 Subject: [PATCH 29/30] tox.ini: py-cov-diff: Check pylint errors/warnings on changes lines Signed-off-by: Bernhard Kaindl --- tox.ini | 16 +++++++++++++++- xcp/cpiofile.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index cded1d55..41820182 100644 --- a/tox.ini +++ b/tox.ini @@ -62,8 +62,10 @@ passenv = PYTEST_ADDOPTS fox: DISPLAY fox: XAUTHORITY fox: DBUS_SESSION_BUS_ADDRESS +setenv = PYTHONWARNINGS=ignore:DEPRECATION deps = {cov,covcp,fox,test}: -r requirements-dev.txt + diff: diff-cov-lint mypy: lxml mypy: mypy mypy: types-mock @@ -71,6 +73,7 @@ deps = mypy: types-six allowlist_externals = {cov,covcp,fox}: cp + diff: sh mypy: cat fox: firefox commands = @@ -80,6 +83,7 @@ commands = mypy: mypy --txt-report . mypy: cat index.txt fox: {[fox]commands} + diff: {[diff]commands} [cov] commands = @@ -97,6 +101,16 @@ commands = cp -av .coverage {envlogdir}/.coverage cp -av {envlogdir}/coverage.xml . +[diff] +commands = + pylint --enable-all-extensions --disable=C,R,W0149,W0160,W0707 --exit-zero \ + --output {envlogdir}/pylint-warnings.txt xcp/ tests/ + sh -c "diff-cov-lint origin/master . \ + --lint_report={envlogdir}/pylint-warnings.txt \ + > {envlogdir}/pylint-warnings-on-changed-lines.txt" + sh -c 'if grep / {envlogdir}/pylint-warnings-on-changed-lines.txt;then \ + cat {envlogdir}/pylint-warnings-on-changed-lines.txt;exit 5;fi' + [fox] commands = firefox {envlogdir}/htmlcov/index.html \ {envlogdir}/htmlcov-tests/index.html \ @@ -115,4 +129,4 @@ python = 3.8: py38-test 3.9: py39-test 3.10: py310-mypy - 3.11: py311-cov \ No newline at end of file + 3.11: py311-cov-diff \ No newline at end of file diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 8b6ddf4b..24ea559d 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1927,4 +1927,4 @@ def is_cpiofile(name): return False bltn_open = open -open = CpioFile.open +open = CpioFile.open # pylint: disable=redefined-builtin From 349af90aaf0336a6963bc0d05082ba75a7906f3f Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 28 Apr 2023 12:00:00 +0200 Subject: [PATCH 30/30] Add README.md and set the new package version to 3.0.0 Signed-off-by: Bernhard Kaindl --- README | 5 -- README.md | 230 +++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 4 +- setup.py | 1 + tox.ini | 40 +-------- 5 files changed, 235 insertions(+), 45 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index 550a1ea8..00000000 --- a/README +++ /dev/null @@ -1,5 +0,0 @@ -This repository contains utility classes. - -To contribute bug fixes, email them to the XenServer development mailing list -(xs-devel@lists.xenserver.org). - diff --git a/README.md b/README.md new file mode 100644 index 00000000..99ddeedb --- /dev/null +++ b/README.md @@ -0,0 +1,230 @@ +Common XenServer/XCP-ng Python classes +====================================== + +The `xcp` directory contains the Common XenServer and XCP-ng Python packages. +They are intented for use in XenServer and XCP-ng Dom0 only and deal with logging, +Hardware/PCI, networking, and other Dom0 tasks. + +The pip package name is `python-libs` which is also the rpm package name in XenServer. +XCP-ng packages it as [xcp-python-libs](https://github.com/xcp-ng-rpms/xcp-python-libs) +([koji](https://koji.xcp-ng.org/packageinfo?packageID=400)). + +It supports Python 2.7 and is currently in progress to get further fixes for >= 3.6. +It depends on `six`, and on Python 2.7, also `configparser` and `pyliblzma`. + +Pylint results from GitHub CI in GitHub Actions page +---------------------------------------------------- +A step of the GitHub workflow produces a browser-friendly `pylint` report: +From the [Actions tab](https://github.com/xenserver/python-libs/actions), +open a recent workflow run the latest and scroll down until you see the tables! + +Testing locally and in GitHub CI using tox +------------------------------------------ + +`pytest` runs tests. Checks by `pylint` and `mypy`. With `tox`, developers can +run the full test suite for Python 2.7 and 3.x. Unit tests are passing but there are + many Python3 issues which it does not uncover yet. + +> Intro: Managing a Project's Virtualenvs with tox - +> A comprehensive beginner's introduction to tox. +> https://www.seanh.cc/2018/09/01/tox-tutorial/ + +To run the tests for all supported and installed python versions, run: +```yaml +# The latest versions of tox need 1.11.0, so sensure that you have the latest py-1.11: +pip3 install 'py>=1.11.0' tox; tox +``` +You can run tox with just the python versions you have using `tox -e py27-test -e py3.11-mypy`. +The syntax is `-e py-[-factor2]` The currently supported factors +are: +- `test`: runs pytest +- `cov`: runs pytest --cov and generate XML and HTML reports in `.tox/py-cov/logs/` +- `mypy`: runs mypy +- `fox`: runs like `cov` but then opens the HTML reports in Firefox! + +The list of `virtualenvs` can be shown using this command: `tox -av -e py312-fox` +```yaml +using tox-3.28.0 from /usr/lib/python3.11/site-packages/tox/__init__.py (pid 157772) +default environments: +py27-test -> Run in a python2.7 virtualenv: pytest +py36-test -> Run in a python3.6 virtualenv: pytest +py37-test -> Run in a python3.7 virtualenv: pytest +py38-test -> Run in a python3.8 virtualenv: pytest +py39-test -> Run in a python3.9 virtualenv: pytest +py310-test -> Run in a python3.10 virtualenv: pytest +py311-mypy -> Run in a python3.11 virtualenv: mypy static analyis +py311-cov -> Run in a python3.11 virtualenv: generate coverage html reports +py312-test -> Run in a python3.12 virtualenv: pytest + +additional environments: +py312-fox -> Run in a python3.12 virtualenv: generate coverage html reports and open them in firefox +``` +If you have just one version of Python3, that will be enough, just use `tox -e py-test`. + +Installation of additional python versions for testing different versions: +- Fedora 37: `sudo dnf install tox` installs all Python versions, even 3.12a7. +- On Ubuntu, the deadsnakes/ppa is broken(except for 3.12), so conda or pyenv has to be used. + For full instructions, see https://realpython.com/intro-to-pyenv/, E.g install on Ubuntu: + ```yaml + sudo apt-get install -y build-essential libssl-dev zlib1g-dev libbz2-dev + libreadline-dev libsqlite3-dev xz-utils libffi-dev liblzma-dev + curl https://pyenv.run | bash # and add the displayed commands to .bashrc + pyenv install 3.{6,7,8,9} && pyenv local 3.{6,7,8,9} # builds and adds them + ``` +- Note: `virtualenv-20.22` broke creating the `py27` venv with tox, at least in some setups. + As a workaround, downgrade it to 20.21 if that happens: `pip3 install -U 'virtualenv<20.22'` +- For testing on newer Ubuntu hosts which have `python2-dev`, but not `pip2`, install `pip2` this way: + ```json + curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py;sudo python2 get-pip.py + ``` + +Static analysis using mypy and pyright +-------------------------------------- +The preconditions for using static analysis with `mypy` (which passes now but has +only a few type comments) and `pyright` are present now and `mypy` is enabled in `tox` +which runs the tests in GitHub CI as well. But of course, because they code is largely +still not yet typed, no strict checks can be enabled so far. However, every checker +which is possible now, is enabled. + +Checking the contents of untyped functions is enabled for all but four modules which +would need more work. Look for `check_untyped_defs = false` in `pytproject.toml`. + +The goal or final benefit would be to have it to ensure internal type correctness +and code quality but also to use static analysis to check the interoperability with +the calling code. + +Type annotations: Use Type comments for now! +-------------------------------------------- +Python2.7 can't support the type annotation syntax, but until all users are migrated, +annotations in comments (type comments) can be used. They are supported by +tools like `mypy` and `pyright` (VS Code): + +Quoting from https://stackoverflow.com/questions/53306458/python-3-type-hints-in-python-2: + +> Function annotations were introduced in [PEP 3107](https://www.python.org/dev/peps/pep-3107/) for Python 3.0. The usage of annotations as type hints was formalized in in [PEP 484](https://www.python.org/dev/peps/pep-0484/) for Python 3.5+. +> +> Any version before 3.0 then will not support the syntax you are using for type hints at all. However, PEP 484 [offers a workaround](https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code), which some editors may choose to honor. In your case, the hints would look like this: +```py +def get_default_device(use_gpu=True): + # type: (bool) -> cl.Device + ... +``` +> or more verbosely, +```py +def get_default_device(use_gpu=True # type: bool + ): + # type: (...) -> cl.Device + ... +``` +> The PEP explicitly states that this form of type hinting should work for any version of Python. + +As proof, these examples show how the comment below triggers the checks: +```diff +--- a/xcp/xmlunwrap.py ++++ b/xcp/xmlunwrap.py +@@ -29,1 +29,2 @@ class XmlUnwrapError(Exception): + def getText(nodelist): ++ # type:(Element) -> str +``` +mypy: +```py +$ mypy xcp/xmlunwrap.py +xcp/xmlunwrap.py:31: error: Name "Element" is not defined +xcp/xmlunwrap.py:38: error: Incompatible return value type (got "bytes", expected "str") +``` +pyright (used by VS Code by default): +```py +$ pyright xcp/xmlunwrap.py|sed "s|$PWD/||" +... +pyright 1.1.295 +xcp/xmlunwrap.py + xcp/xmlunwrap.py:32:13 - error: "Element" is not defined (reportUndefinedVariable) + xcp/xmlunwrap.py:38:12 - error: Expression of type "Unknown | bytes" cannot be assigned to return type "str" +   Type "Unknown | bytes" cannot be assigned to type "str" +     "bytes" is incompatible with "str" (reportGeneralTypeIssues) + xcp/xmlunwrap.py:81:38 - error: Argument of type "Unknown | None" cannot be assigned to parameter "default" of type "str" in function "getStrAttribute" +   Type "Unknown | None" cannot be assigned to type "str" +     Type "None" cannot be assigned to type "str" (reportGeneralTypeIssues) +3 errors, 0 warnings, 0 informations +Completed in 0.604sec +``` +See https://github.com/xenserver/python-libs/pull/23 for the context of this example. + +Special open TODOs: +------------------- + +Charset encoding/string handling: +* With Python3, `read()` on files `open()`ed without specifying binary mode will attempt + to decode the data into the Python3 Unicode string type, which will fail for all + binary data. Thus all `open()` calls which might open binary files have to be converted + to binary mode by default unless the caller is sure he is opening an ASCII file, + even then, enabling an error handle to handle decoding errors is recommended. +* With Python3, the `stdin`, `stdout` and `stderr` pipes for `Popen()` default to + `bytes`(binary mode.) Binary mode is much safer because it foregoes the encode/decode. The existing users need to be able to enable text mode (when safe, it will attempt + to decode and encode!) or preferably be able to use bytes (which is the type behind Python2 strings too) instead. See these PRs for details: + * https://github.com/xenserver/python-libs/pull/22 + * https://github.com/xenserver/python-libs/pull/23 + * https://github.com/xenserver/python-libs/pull/24 + * What's more: When code is called from a xapi plugin (such as ACK), when such code + attempts to read text files like the `pciids` file, and there is a Unicode char + it int, and the locale is not set up to be UTF-8 (because xapi plugins are started + from xapi), the UTF-8 decoder has to be explicitly enabled for these files, + bese by adding `encoding="utf-8"` to the arguments of these specific `open()` calls, + to have valid Unicode text strings, e.g. `xcp.pci`, for regular text processing. + * TODO: More to be opened for all remaining `open()` and `Popen()` users, + as well as ensuring that users of `urllib` are able to work with they bytes + it returns (there is no option to use text mode, data may be gzip-encoded!) + +Users +----- + +* https://github.com/xenserver/host-installer + * /opt/xensource/installer/ (has copies of `cpiofile.py`, `repository.py` (with `accessor.py`) +* https://github.com/xcp-ng-rpms/host-upgrade-plugin ([koji](https://koji.xcp-ng.org/packageinfo?packageID=104)): + * /etc/xapi.d/plugins/prepare_host_upgrade.py +* https://github.com/xapi-project/xen-api (`xapi-core.rpm` and `xenopsd.rpm`) + * /etc/xapi.d/extensions/pool_update.apply + * /etc/xapi.d/extensions/pool_update.precheck + * /etc/xapi.d/plugins/disk-space + * /etc/xapi.d/plugins/install-supp-pack + * /opt/xensource/libexec/host-display + * /opt/xensource/libexec/mail-alarm + * /opt/xensource/libexec/usb_reset.py + * /opt/xensource/libexec/usb_scan.py + * /usr/libexec/xenopsd/igmp_query_injector.py +* xenserver-release-config/[xcp-ng-release-config](https://koji.xcp-ng.org/rpminfo?rpmID=10250) + * /opt/xensource/libexec/fcoe_driver + * /opt/xensource/libexec/xen-cmdline +* https://github.com/xcp-ng-rpms/interface-rename: + * /etc/sysconfig/network-scripts/interface-rename.py + * /opt/xensource/bin/interface-rename +* pvsproxy (Proprietary) + * /usr/libexec/xapi-storage-script/volume/org.xen.xapi.storage.tmpfs/memoryhelper.py +* https://github.com/xenserver/linux-guest-loader (not installed by default anymore) + * /opt/xensource/libexec/eliloader.py +* https://github.com/xcp-ng-rpms/vcputune + * /opt/xensource/bin/host-cpu-tune +* The ACK xenapi plugin see: https://github.com/xenserver/python-libs/pull/21 + +Verification: +```ps +# rpm -qf $(grep -r import /usr/libexec/ /usr/bin /etc/xapi.d/ /opt/xensource/|grep xcp|cut -d: -f1|grep -v Binary) --qf '%{name}\n'|sort -u|tee xcp-python-libs-importers.txt +host-upgrade-plugin +interface-rename +pvsproxy +vcputune +xapi-core +xenopsd +xenserver-release-config +# grep -s import $(rpm -ql xapi-core)|grep xcp|cut -d: -f1 +/etc/xapi.d/extensions/pool_update.apply +/etc/xapi.d/extensions/pool_update.precheck +/etc/xapi.d/plugins/disk-space +/etc/xapi.d/plugins/disk-space +/etc/xapi.d/plugins/install-supp-pack +/opt/xensource/libexec/host-display +/opt/xensource/libexec/mail-alarm +/opt/xensource/libexec/usb_reset.py +/opt/xensource/libexec/usb_scan.py +``` + diff --git a/pyproject.toml b/pyproject.toml index 653412c9..7b34526b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] # https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ name = "python-libs" -version = "1.0.0" +version = "3.0.0" description = "Common XCP-ng Python classes" requires-python = ">=2.7" license = {file = "LICENSE"} @@ -16,7 +16,7 @@ maintainers = [ {name = "Pau Ruiz Safont"}, {name = "Bernhard Kaindl"}, ] -readme = "README" +readme = "README.md" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: POSIX :: Linux", diff --git a/setup.py b/setup.py index 11e0fa48..6d4b82b6 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ from setuptools import setup setup(name='python-libs', + version='3.0.0', description='Common XenServer Python classes', packages=['xcp', 'xcp.net', diff --git a/tox.ini b/tox.ini index 41820182..65bb7fd1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,44 +1,8 @@ -# Managing a Project's Virtualenvs with tox -# A comprehensive beginner's introduction to tox. -# Tox is a great tool for standardising and automating any development task -# that benefits from being run in an isolated virtualenv: -# https://www.seanh.cc/2018/09/01/tox-tutorial/ -# To run the tests for all supported and installed python versions, run: -# pip3 install 'py>=1.11.0' tox; tox - -# Installation of additional python versions: -# - On Fedora 37, `sudo dnf install tox` installs all Python versions, even 3.12a6. -# - On Ubuntu, the deadsnakes/ppa is broken(except for 3.12), so conda or pyenv has to be used: -# sudo apt-get install -y build-essential libssl-dev zlib1g-dev libbz2-dev -# libreadline-dev libsqlite3-dev xz-utils libffi-dev liblzma-dev -# curl https://pyenv.run | bash # Then install the displayed commands in your .bashrc. -# pyenv install 3.{6,7,8,9} && pyenv local 3.{6,7,8,9} # builds and add them to PATH. -# https://brandonrozek.com/blog/pyenvtox/; https://realpython.com/intro-to-pyenv/ -# -# - Note: virtualenv 20.22 broke support for the py27 venv with tox. (at least in some setups) -# As a workaround, downgrade it to 20.21 if that happens: pip3 install -U 'virtualenv<20.22' -# - For testing on newer Ubuntu hosts which have python2-dev, but not pip2, install pip2 this way: -# curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py;sudo python2 get-pip.py - [tox] skipsdist = true isolated_build = true -envlist = py27-test, py36-test, py37-test, py38-test, py39-test, py310-test, py312-test, - py311-mypy, py311-cov -# This results in this list of environments tox -e py312-fox -av -# default environments: -# py27-test -> Run in a python2.7 virtualenv: pytest -# py36-test -> Run in a python3.6 virtualenv: pytest -# py37-test -> Run in a python3.7 virtualenv: pytest -# py38-test -> Run in a python3.8 virtualenv: pytest -# py39-test -> Run in a python3.9 virtualenv: pytest -# py310-test -> Run in a python3.10 virtualenv: pytest -# py312-test -> Run in a python3.12 virtualenv: pytest -# py311-mypy -> Run in a python3.11 virtualenv: mypy static analyis -# py311-cov -> Run in a python3.11 virtualenv: generate coverage html reports -# -# additional environments: -# py312-fox -> Run in a python3.12 virtualenv: generate coverage html reports and open them in firefox +envlist = py27-test, py36-test, py37-test, py38-test, py39-test, py310-test, + py311-mypy, py311-cov, py312-test # TODO later: Enable the venv_update extension so that changes to requirements-dev.txt # are automatically detected.