From 92c4403fd1b346762b27cabdbbac4ffc9a3f5ff8 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 29 Jul 2022 13:28:26 +0200 Subject: [PATCH 01/92] Coverage: test_cpio: recursive add to archive Adds more data to the archive (a copy of the same file, from another wd). Uses the "." special case for even more coverage. Coverage change: xcp.cpiofile: 50% -> 51.7% total: 38.4% -> 39% Signed-off-by: Yann Dirson --- tests/test_cpio.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 33084484..f63bc2e3 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -73,6 +73,10 @@ def archiveCreate(self, fn, fmt='w'): arc = CpioFile.open(fn, fmt) f = arc.getcpioinfo('archive/data') arc.addfile(f, open('archive/data')) + # test recursively add "." + os.chdir('archive') + arc.add(".") + os.chdir("..") # TODO add self crafted file arc.close() # special case for XZ, test check type (crc32) From 925e330afc18f948654323001970f560c0d8ebd6 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 16:10:21 +0200 Subject: [PATCH 02/92] Coverage: test_cpio: exercise "extractall()" and check its output Coverage change: xcp.cpiofile: 51.7% -> 58.1% total: 39% -> 41.1% Signed-off-by: Yann Dirson --- tests/test_cpio.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index f63bc2e3..85ff62d8 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -53,7 +53,7 @@ def setUp(self): self.doXZ = False def tearDown(self): - check_call("rm -rf archive archive.cpio*") + check_call("rm -rf archive archive.cpio* archive2") # TODO check with file (like 'r:*') # TODO use cat to check properly for pipes @@ -66,7 +66,15 @@ def archiveExtract(self, fn, fmt='r|*'): self.assertEqual(len(data), f.size) self.assertEqual(self.md5data, md5(data).hexdigest()) found = True + arc.close() self.assertTrue(found) + # extract with extractall and compare + arc = CpioFile.open(fn, fmt) + check_call("rm -rf archive2") + os.rename('archive', 'archive2') + arc.extractall() + check_call("diff -rq archive2 archive") + arc.close() def archiveCreate(self, fn, fmt='w'): os.unlink(fn) From d8fdc3dd824565d04eb955bb446212323f50274a Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 29 Jul 2022 14:03:36 +0200 Subject: [PATCH 03/92] Coverage: test_pci: new test from snippet in pci.py Coverage change: xcp.pci: 40.9% -> 76.3% total: 41.1% -> 43% Signed-off-by: Yann Dirson --- tests/data/lspci-mn | 36 +++++++++++++++++++++++++++++++ tests/data/pci.ids | 29 +++++++++++++++++++++++++ tests/test_pci.py | 52 ++++++++++++++++++++++++++++++++++++++++++++- xcp/pci.py | 16 -------------- 4 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 tests/data/lspci-mn create mode 100644 tests/data/pci.ids diff --git a/tests/data/lspci-mn b/tests/data/lspci-mn new file mode 100644 index 00000000..4450e6d3 --- /dev/null +++ b/tests/data/lspci-mn @@ -0,0 +1,36 @@ +00:00.0 "0600" "1022" "1630" "1022" "1630" +00:00.2 "0806" "1022" "1631" "1022" "1631" +00:01.0 "0600" "1022" "1632" "" "" +00:01.1 "0604" "1022" "1633" "" "" +00:02.0 "0600" "1022" "1632" "" "" +00:02.1 "0604" "1022" "1634" "" "" +00:02.2 "0604" "1022" "1634" "" "" +00:02.3 "0604" "1022" "1634" "" "" +00:08.0 "0600" "1022" "1632" "" "" +00:08.1 "0604" "1022" "1635" "" "" +00:08.2 "0604" "1022" "1635" "" "" +00:14.0 "0c05" "1022" "790b" -r51 "1462" "12ac" +00:14.3 "0601" "1022" "790e" -r51 "1462" "12ac" +00:18.0 "0600" "1022" "1448" "" "" +00:18.1 "0600" "1022" "1449" "" "" +00:18.2 "0600" "1022" "144a" "" "" +00:18.3 "0600" "1022" "144b" "" "" +00:18.4 "0600" "1022" "144c" "" "" +00:18.5 "0600" "1022" "144d" "" "" +00:18.6 "0600" "1022" "144e" "" "" +00:18.7 "0600" "1022" "144f" "" "" +01:00.0 "0604" "1002" "1478" -rc1 "" "" +02:00.0 "0604" "1002" "1479" "" "" +03:00.0 "0380" "1002" "7340" -rc1 "1462" "12ac" +03:00.1 "0403" "1002" "ab38" "1462" "12ac" +04:00.0 "0280" "8086" "2723" -r1a "8086" "0084" +05:00.0 "0200" "10ec" "8168" -r15 "1462" "12ac" +06:00.0 "0108" "144d" "a808" -p02 "144d" "a801" +07:00.0 "0300" "1002" "1636" -rc6 "1462" "12ac" +07:00.2 "1080" "1022" "15df" "1022" "15df" +07:00.3 "0c03" "1022" "1639" -p30 "1462" "12ac" +07:00.4 "0c03" "1022" "1639" -p30 "1462" "12ac" +07:00.5 "0480" "1022" "15e2" -r01 "1462" "12ac" +07:00.6 "0403" "1022" "15e3" "1462" "12ac" +08:00.0 "0106" "1022" "7901" -r81 -p01 "1462" "12ac" +08:00.1 "0106" "1022" "7901" -r81 -p01 "1462" "12ac" diff --git a/tests/data/pci.ids b/tests/data/pci.ids new file mode 100644 index 00000000..acaccbb1 --- /dev/null +++ b/tests/data/pci.ids @@ -0,0 +1,29 @@ +## Excerpt from /usr/share/hwdata/pci.ids + +# Vendors, devices and subsystems. Please keep sorted. + +# Syntax: +# vendor vendor_name +# device device_name <-- single tab +# subvendor subdevice subsystem_name <-- two tabs + +1002 Advanced Micro Devices, Inc. [AMD/ATI] + 1314 Wrestler HDMI Audio + 174b 1001 PURE Fusion Mini + 1636 Renoir + 7340 Navi 14 [Radeon RX 5500/5500M / Pro 5500M] + +# List of known device classes, subclasses and programming interfaces + +# Syntax: +# C class class_name +# subclass subclass_name <-- single tab +# prog-if prog-if_name <-- two tabs + +C 03 Display controller + 00 VGA compatible controller + 00 VGA controller + 01 8514 controller + 01 XGA compatible controller + 02 3D controller + 80 Display controller diff --git a/tests/test_pci.py b/tests/test_pci.py index 65fcd796..59af4828 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -1,8 +1,10 @@ #!/usr/bin/env python import unittest, sys, os, os.path as path +import subprocess +from mock import patch, Mock -from xcp.pci import PCI +from xcp.pci import PCI, PCIIds, PCIDevices class TestInvalid(unittest.TestCase): @@ -56,5 +58,53 @@ def test_equality(self): self.assertEqual(PCI("0000:00:00.0"), PCI("00:00.0")) +class TestPCIIds(unittest.TestCase): + def tests_nodb(self): + with patch("xcp.pci.os.path.exists") as exists_mock: + exists_mock.return_value = False + with self.assertRaises(Exception): + PCIIds.read() + exists_mock.assert_called_once_with("/usr/share/hwdata/pci.ids") + + def tests_videoclass(self): + with patch("xcp.pci.os.path.exists") as exists_mock, \ + patch("xcp.pci.open") as open_mock, \ + open("tests/data/pci.ids") as fake_data: + exists_mock.return_value = True + open_mock.return_value.__iter__ = Mock(return_value=iter(fake_data)) + ids = PCIIds.read() + exists_mock.assert_called_once_with("/usr/share/hwdata/pci.ids") + open_mock.assert_called_once_with("/usr/share/hwdata/pci.ids") + video_class = ids.lookupClass('Display controller') + self.assertEqual(video_class, ['03']) + + with patch("xcp.pci.subprocess.Popen") as popen_mock, \ + open("tests/data/lspci-mn") as fake_data: + popen_mock.return_value.stdout.__iter__ = Mock(return_value=iter(fake_data)) + devs = PCIDevices() + popen_mock.assert_called_once_with(['lspci', '-mn'], bufsize = 1, + stdout = subprocess.PIPE) + sorted_devices = sorted(devs.findByClass(video_class), + key=lambda x: x['id']) + self.assertEqual(len(sorted_devices), 2) + + for (video_dev, + num_functions, + vendor, + device, + ) in zip(sorted_devices, + (1, 5), + ("Advanced Micro Devices, Inc. [AMD/ATI]", + "Advanced Micro Devices, Inc. [AMD/ATI]"), + ("Navi 14 [Radeon RX 5500/5500M / Pro 5500M]", + "Renoir"), + ): + self.assertEqual(len(devs.findRelatedFunctions(video_dev['id'])), num_functions) + self.assertEqual(ids.findVendor(video_dev['vendor']), vendor) + self.assertEqual(ids.findDevice(video_dev['vendor'], video_dev['device']), device) + + self.assertEqual(len(devs.findRelatedFunctions('00:18.1')), 7) + + if __name__ == "__main__": sys.exit(unittest.main()) diff --git a/xcp/pci.py b/xcp/pci.py index c1fc90e3..8b371152 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Copyright (c) 2013, Citrix Inc. # All rights reserved. # @@ -314,17 +312,3 @@ def pci_sbdfi_to_nic(sbdfi, nics): raise Exception("Insufficient NICs with PCI SBDF %s (Found %d, wanted at least %d)" % (value, len(matching_nics), index)) return matching_nics[index] - - -if __name__ == "__main__": - IDS = PCIIds.read() - VIDEO_CLASS = IDS.lookupClass('Display controller') - - DEVS = PCIDevices() - for video_dev in DEVS.findByClass(VIDEO_CLASS): - print video_dev['id'], IDS.findVendor(video_dev['vendor']), \ - IDS.findDevice(video_dev['vendor'], video_dev['device']) - print DEVS.findRelatedFunctions(video_dev['id']) - print DEVS.findRelatedFunctions('00:1d.1') - - From 02468d43685f14cc3ea43cebd24dc85294683c9d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 29 Jul 2022 14:04:04 +0200 Subject: [PATCH 04/92] Coverage: new test_cmd from snippet in cmd.py Coverage change: xcp.cmd: 0% -> 78.0% total: 43% -> 44.2% Signed-off-by: Yann Dirson --- tests/test_cmd.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ xcp/cmd.py | 11 ---------- 2 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 tests/test_cmd.py diff --git a/tests/test_cmd.py b/tests/test_cmd.py new file mode 100644 index 00000000..c6304873 --- /dev/null +++ b/tests/test_cmd.py @@ -0,0 +1,54 @@ +import unittest +from mock import patch, Mock, DEFAULT + +from xcp.cmd import OutputCache + +class TestCache(unittest.TestCase): + def setUp(self): + self.c = OutputCache() + + def test_fileContents(self): + with patch("xcp.cmd.open") as open_mock: + open_mock.return_value.readlines = Mock(return_value=["line1\n", "line2\n"]) + + # uncached fileContents + data = self.c.fileContents('/tmp/foo') + open_mock.assert_called_once_with("/tmp/foo") + self.assertEqual(data, "line1\nline2\n") + + # rerun as cached + open_mock.reset_mock() + data = self.c.fileContents('/tmp/foo') + open_mock.assert_not_called() + self.assertEqual(data, "line1\nline2\n") + + # rerun after clearing cache + open_mock.reset_mock() + self.c.clearCache() + data = self.c.fileContents('/tmp/foo') + open_mock.assert_called_once_with("/tmp/foo") + self.assertEqual(data, "line1\nline2\n") + + def test_runCmd(self): + output_data = "line1\nline2\n" + with patch("xcp.cmd.subprocess.Popen") as popen_mock: + # mock Popen .communicate and .returncode for + # `output_data`on stdout, nothing on stderr, and exit + # value of 42 + communicate_mock = Mock(return_value=(output_data, "")) + popen_mock.return_value.communicate = communicate_mock + def communicate_side_effect(_input_text): + popen_mock.return_value.returncode = 42 + return DEFAULT + communicate_mock.side_effect = communicate_side_effect + + # uncached runCmd + data = self.c.runCmd(['ls', '/tmp'], True) + popen_mock.assert_called_once() + self.assertEqual(data, (42, output_data)) + + # rerun as cached + popen_mock.reset_mock() + data = self.c.runCmd(['ls', '/tmp'], True) + popen_mock.assert_not_called() + self.assertEqual(data, (42, output_data)) diff --git a/xcp/cmd.py b/xcp/cmd.py index b51eaa8a..ff039484 100644 --- a/xcp/cmd.py +++ b/xcp/cmd.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Copyright (c) 2013, Citrix Inc. # All rights reserved. # @@ -89,12 +87,3 @@ def runCmd(self, command, with_stdout = False, with_stderr = False, inputtext = def clearCache(self): self.cache.clear() - -if __name__ == '__main__': - c = OutputCache() - print c.fileContents('/tmp/foo') - print c.fileContents('/tmp/foo') - c.clearCache() - print c.fileContents('/tmp/foo') - print c.runCmd(['ls', '/tmp'], True) - print c.runCmd(['ls', '/tmp'], True) From b1a5152698deb9645b9c4efbcf1edb32b2790343 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 22 Jul 2022 17:28:03 +0200 Subject: [PATCH 05/92] Coverage: new test_xmlunwrap from snippet in xmlunwrap.py Coverage change: xcp.xmlunwrap: 0% -> 68.6%% total: 44.2% -> 45.2% Signed-off-by: Yann Dirson --- tests/test_xmlunwrap.py | 37 +++++++++++++++++++++++++++++++++++++ xcp/xmlunwrap.py | 27 --------------------------- 2 files changed, 37 insertions(+), 27 deletions(-) create mode 100644 tests/test_xmlunwrap.py diff --git a/tests/test_xmlunwrap.py b/tests/test_xmlunwrap.py new file mode 100644 index 00000000..f71d778f --- /dev/null +++ b/tests/test_xmlunwrap.py @@ -0,0 +1,37 @@ +import unittest +import xml.dom.minidom + +from xcp.xmlunwrap import (getElementsByTagName, getText, getMapAttribute, + getStrAttribute, XmlUnwrapError) + +class TestXmlUnwrap(unittest.TestCase): + def setUp(self): + a_text = """ + text1 + text2 + """ + xmldoc = xml.dom.minidom.parseString(a_text) + self.top_el = xmldoc.documentElement + + def test(self): + self.assertEqual(self.top_el.tagName, "installation") + + self.assertEqual([getText(el) + for el in getElementsByTagName(self.top_el, ["fred"])], + ["text1", "text2"]) + + x = getMapAttribute(self.top_el, ["mode"], [('test', 42), ('stuff', 77)]) + self.assertEqual(x, 42) + x = getMapAttribute(self.top_el, ["made"], [('test', 42), ('stuff', 77)], + default='stuff') + self.assertEqual(x, 77) + + x = getStrAttribute(self.top_el, ["mode"]) + self.assertEqual(x, "test") + x = getStrAttribute(self.top_el, ["made"]) + self.assertEqual(x, "") + x = getStrAttribute(self.top_el, ["made"], None) + self.assertEqual(x, None) + + with self.assertRaises(XmlUnwrapError): + x = getStrAttribute(self.top_el, ["made"], mandatory=True) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 3256cab3..654b84bf 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Copyright (c) 2013, Citrix Inc. # All rights reserved. # @@ -25,9 +23,6 @@ """xmlunwrap - general methods to unwrap XML elements & attributes""" -import xml.dom.minidom - - class XmlUnwrapError(Exception): pass @@ -87,25 +82,3 @@ def getMapAttribute(el, attrs, mapping, default = None): k_list = list(k) return v[k_list.index(key)] - -if __name__ == '__main__': - - a_text = """ - text1 - text2 - """ - xmldoc = xml.dom.minidom.parseString(a_text) - top_el = xmldoc.documentElement - - print top_el.tagName - - for el in getElementsByTagName(top_el, ["fred"]): - print getText(el) - - print getMapAttribute(top_el, ["mode"], [('test', 42), ('stuff', 77)]) - print getMapAttribute(top_el, ["made"], [('test', 42), ('stuff', 77)], default = 'stuff') - - print getStrAttribute(top_el, ["mode"]) - print getStrAttribute(top_el, ["made"]) - print getStrAttribute(top_el, ["made"], None) - print getStrAttribute(top_el, ["made"], mandatory = True) From cc0121e5f7b790bf607a864e6ab619f877e72d35 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 25 Jul 2022 11:15:27 +0200 Subject: [PATCH 06/92] Coverage: new test_bootloader We may want to do some real checks on the output, this first version only exercises the code. Includes ad-hoc testing of write/read methods for grub/extlinux from a grub2 configuration, for lack of representative versions of those deprecated config files. Ad-hoc test for extlinux reveals inconsistent typing, and is marked as failing for now: print("serial %d %d" % (self.serial['port'], > self.serial['baud']), file=fh) E TypeError: %d format: a number is required, not str Coverage change: xcp.bootloader: 0% -> 59.1%% total: 45.2% -> 53.4% Signed-off-by: Yann Dirson --- tests/data/grub.cfg | 35 ++++++++++++++++++++++++++++++++++ tests/test_bootloader.py | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/data/grub.cfg create mode 100644 tests/test_bootloader.py diff --git a/tests/data/grub.cfg b/tests/data/grub.cfg new file mode 100644 index 00000000..c6d2b4a1 --- /dev/null +++ b/tests/data/grub.cfg @@ -0,0 +1,35 @@ +serial --unit=0 --speed=115200 +terminal_input serial console +terminal_output serial console +set default=0 +set timeout=5 +menuentry 'XCP-ng' { + search --label --set root root-vgdorj + multiboot2 /boot/xen.gz dom0_mem=7584M,max:7584M watchdog ucode=scan dom0_max_vcpus=1-16 crashkernel=256M,below=4G console=vga vga=mode-0x0311 + module2 /boot/vmlinuz-4.19-xen root=LABEL=root-vgdorj ro nolvm hpet=disable console=hvc0 console=tty0 quiet vga=785 splash plymouth.ignore-serial-consoles + module2 /boot/initrd-4.19-xen.img +} +menuentry 'XCP-ng (Serial)' { + search --label --set root root-vgdorj + multiboot2 /boot/xen.gz com1=115200,8n1 console=com1,vga dom0_mem=7584M,max:7584M watchdog ucode=scan dom0_max_vcpus=1-16 crashkernel=256M,below=4G + module2 /boot/vmlinuz-4.19-xen root=LABEL=root-vgdorj ro nolvm hpet=disable console=tty0 console=hvc0 + module2 /boot/initrd-4.19-xen.img +} +menuentry 'XCP-ng in Safe Mode' { + search --label --set root root-vgdorj + multiboot2 /boot/xen.gz nosmp noreboot noirqbalance no-mce no-bootscrub no-numa no-hap no-mmcfg max_cstate=0 nmi=ignore allow_unsafe dom0_mem=7584M,max:7584M com1=115200,8n1 console=com1,vga + module2 /boot/vmlinuz-4.19-xen earlyprintk=xen root=LABEL=root-vgdorj ro nolvm hpet=disable console=tty0 console=hvc0 + module2 /boot/initrd-4.19-xen.img +} +menuentry 'XCP-ng (Xen 4.13.1 / Linux 4.19.0+1)' { + search --label --set root root-vgdorj + multiboot2 /boot/xen-fallback.gz dom0_mem=7584M,max:7584M watchdog ucode=scan dom0_max_vcpus=1-16 crashkernel=256M,below=4G + module2 /boot/vmlinuz-fallback root=LABEL=root-vgdorj ro nolvm hpet=disable console=hvc0 console=tty0 + module2 /boot/initrd-fallback.img +} +menuentry 'XCP-ng (Serial, Xen 4.13.1 / Linux 4.19.0+1)' { + search --label --set root root-vgdorj + multiboot2 /boot/xen-fallback.gz com1=115200,8n1 console=com1,vga dom0_mem=7584M,max:7584M watchdog ucode=scan dom0_max_vcpus=1-16 crashkernel=256M,below=4G + module2 /boot/vmlinuz-fallback root=LABEL=root-vgdorj ro nolvm hpet=disable console=tty0 console=hvc0 + module2 /boot/initrd-fallback.img +} diff --git a/tests/test_bootloader.py b/tests/test_bootloader.py new file mode 100644 index 00000000..35a20f26 --- /dev/null +++ b/tests/test_bootloader.py @@ -0,0 +1,41 @@ +import unittest +import os +import subprocess +from tempfile import NamedTemporaryFile + +from xcp.bootloader import Bootloader + +class TestBootloader(unittest.TestCase): + def test_grub2(self): + bl = Bootloader.readGrub2("tests/data/grub.cfg") + with NamedTemporaryFile("w") as temp: + bl.writeGrub2(temp.name) + # get a diff + proc = subprocess.Popen(["diff", "tests/data/grub.cfg", temp.name], + stdout = subprocess.PIPE, + universal_newlines=True) + for line in proc.stdout: + # FIXME: check is entirely ad-hoc, should we have a diff at all ? + self.assertRegexpMatches(line, r"^(5a6,13$|>)") + + proc.stdout.close() + proc.wait() + + +class TestBootloaderAdHoc(unittest.TestCase): + def setUp(self): + self.bl = Bootloader.readGrub2("tests/data/grub.cfg") + + def test_grub(self): + with NamedTemporaryFile("w", delete=False) as temp: + self.bl.writeGrub(temp) + bl2 = Bootloader.readGrub(temp.name) + os.unlink(temp.name) + + @unittest.expectedFailure + def test_extlinux(self): + # FIXME that one triggers a format error, real inconsistency bug ? + with NamedTemporaryFile("w", delete=False) as temp: + self.bl.writeExtLinux(temp) + bl2 = Bootloader.readExtLinux(temp.name) + os.unlink(temp.name) From 7c609a4952ea3afe6892712156fa7ab74e655b33 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 25 Jul 2022 15:38:07 +0200 Subject: [PATCH 07/92] xcp.bootloader: relax format strings for serial config File format parsers do not all agree whether to store some parameters as int or str. This relax the format strings to accept both. Fixes creating an extlinux config from a grub2 config. Coverage change: xcp.bootloader: 59.1% -> 72.8% total: 53.4% -> 55.3% Signed-off-by: Yann Dirson --- tests/test_bootloader.py | 2 -- xcp/bootloader.py | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_bootloader.py b/tests/test_bootloader.py index 35a20f26..ca48204e 100644 --- a/tests/test_bootloader.py +++ b/tests/test_bootloader.py @@ -32,9 +32,7 @@ def test_grub(self): bl2 = Bootloader.readGrub(temp.name) os.unlink(temp.name) - @unittest.expectedFailure def test_extlinux(self): - # FIXME that one triggers a format error, real inconsistency bug ? with NamedTemporaryFile("w", delete=False) as temp: self.bl.writeExtLinux(temp) bl2 = Bootloader.readExtLinux(temp.name) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index e1af85f6..51e362e8 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -466,10 +466,10 @@ def writeExtLinux(self, dst_file = None): if self.serial: if self.serial.get('flow', None) is None: - print >> fh, "serial %d %d" % (self.serial['port'], + print >> fh, "serial %s %s" % (self.serial['port'], self.serial['baud']) else: - print >> fh, "serial %d %d %s" % (self.serial['port'], + print >> fh, "serial %s %s %s" % (self.serial['port'], self.serial['baud'], self.serial['flow']) if self.default: @@ -507,7 +507,7 @@ def writeGrub(self, dst_file = None): print >> fh, "# location " + self.location if self.serial: - print >> fh, "serial --unit=%d --speed=%s" % (self.serial['port'], + print >> fh, "serial --unit=%s --speed=%s" % (self.serial['port'], self.serial['baud']) print >> fh, "terminal --timeout=10 console serial" else: @@ -540,7 +540,7 @@ def writeGrub2(self, dst_file = None): fh = open(dst_file, 'w') if self.serial: - print >> fh, "serial --unit=%d --speed=%s" % (self.serial['port'], + print >> fh, "serial --unit=%s --speed=%s" % (self.serial['port'], self.serial['baud']) print >> fh, "terminal_input serial console" print >> fh, "terminal_output serial console" From 8333f8b0cb372705f8ffeb9694b9328ce8dad231 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 26 Jul 2022 16:08:49 +0200 Subject: [PATCH 08/92] Coverage: new tests for xcp.accessor and xcp.repository Only FileAccessor is actually covered by these tests. HTTPAccessor tests are provided, but are skipped by default, as they currently need network access to xcp-ng.org repository. We will want at some point to mock the HTTP access. Coverage change: xcp.accessor: 0% -> 30.0% xcp.repository: 0% -> 72.2% total: 55.3% -> 62.7% Signed-off-by: Yann Dirson --- tests/data/repo/.treeinfo | 34 ++++++++++++++++++ tests/data/repo/XS-PACKAGES | 0 tests/data/repo/XS-REPOSITORY | 14 ++++++++ tests/data/repo/XS-REPOSITORY-LIST | 0 tests/data/repo/repodata/repomd.xml | 55 +++++++++++++++++++++++++++++ tests/test_accessor.py | 21 +++++++++++ tests/test_repository.py | 25 +++++++++++++ 7 files changed, 149 insertions(+) create mode 100644 tests/data/repo/.treeinfo create mode 100644 tests/data/repo/XS-PACKAGES create mode 100644 tests/data/repo/XS-REPOSITORY create mode 100644 tests/data/repo/XS-REPOSITORY-LIST create mode 100644 tests/data/repo/repodata/repomd.xml create mode 100644 tests/test_accessor.py create mode 100644 tests/test_repository.py diff --git a/tests/data/repo/.treeinfo b/tests/data/repo/.treeinfo new file mode 100644 index 00000000..e49cb4fc --- /dev/null +++ b/tests/data/repo/.treeinfo @@ -0,0 +1,34 @@ +[platform] +name = XCP +version = 3.2.1 + +[branding] +name = XCP-ng +version = 8.2.1 + +[build] +number = release/yangtze/master/58 + +[keys] +key1 = RPM-GPG-KEY-CH-8 +key2 = RPM-GPG-KEY-CH-8-LCM +key3 = RPM-GPG-KEY-Platform-V1 + +[general] +name = XCP-ng-8.2.1 +family = XCP-ng +timestamp = 1645700813.00 +variant = +version = 8.2.1 +packagedir = +arch = x86_64 + +[images-x86_64] +kernel = boot/pxelinux/mboot.c32 +initrd = boot/vmlinuz +boot.iso = boot/xen.gz + +[images-xen] +kernel = boot/pxelinux/mboot.c32 +initrd = boot/vmlinuz + diff --git a/tests/data/repo/XS-PACKAGES b/tests/data/repo/XS-PACKAGES new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/repo/XS-REPOSITORY b/tests/data/repo/XS-REPOSITORY new file mode 100644 index 00000000..8213385d --- /dev/null +++ b/tests/data/repo/XS-REPOSITORY @@ -0,0 +1,14 @@ + + + + Description + + + + diff --git a/tests/data/repo/XS-REPOSITORY-LIST b/tests/data/repo/XS-REPOSITORY-LIST new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/repo/repodata/repomd.xml b/tests/data/repo/repodata/repomd.xml new file mode 100644 index 00000000..43ef48d8 --- /dev/null +++ b/tests/data/repo/repodata/repomd.xml @@ -0,0 +1,55 @@ + + + 1645708616 + + f8454f26592a0669368825d673edc18a067d8a434f2d3ceaa03cdb45e72572e8 + 6f2d69e90edbe7b3eb83654995dc07c6e577d8fc2f9ee7ec7f4f206ed486218b + + 1645708616 + 208220 + 1449213 + + + 4e678f6ce4fc197faef2c75645d00c79ee8b70d3210aa2f692fe432ed223c695 + cea7a040a15876aba2238f3b8910cd48ded102ff0d4976df81903a3bc4ae8586 + + 1645708616 + 265582 + 2704673 + + + a19f12f68c74459bc9fc7887376408b4c8dc9e84876ebbf3301afec79f110a7b + 46a83a4100e33249a95462d7ed54cb71ee4a178db8f2346b83f1ba5f1a5d47dc + + 1645708616 + 204512 + 1054253 + + + 34a89554a4214fde5b473ddcf1d9be796801e8e8676cda860a3d032a7e8b10d8 + 1fc99d5403906556e8fc91d2121816354cac667741f1a04f51988b5cc3e01b98 + + 1645708616 + 419050 + 1754112 + 10 + + + f4c416c2cf253741749420300109cf8a046e0cdd27c269e69795a9ecc092669b + 1de3d29150b9da80b10c95b913c91369f3b7d03bb2ed57eedf511038e4537c50 + + 1645708616 + 323942 + 1380352 + 10 + + + 4a192c51fff556d2f1fdbb8f90b5454558c910a50674e65b7cdfacd5f791fdb5 + 23dbdc6ce9ece7f1d3ac3d7b31cd61b4019e96ef01903975516fd5e3083de32a + + 1645708616 + 243295 + 988160 + 10 + + diff --git a/tests/test_accessor.py b/tests/test_accessor.py new file mode 100644 index 00000000..ade787e6 --- /dev/null +++ b/tests/test_accessor.py @@ -0,0 +1,21 @@ +import unittest + +import xcp.accessor + +class TestAccessor(unittest.TestCase): + def test_http(self): + raise unittest.SkipTest("comment out if you really mean it") + a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) + a.start() + self.assertTrue(a.access('.treeinfo')) + self.assertFalse(a.access('no_such_file')) + self.assertEqual(a.lastError, 404) + a.finish() + + def test_file(self): + a = xcp.accessor.createAccessor("file://tests/data/repo/", True) + a.start() + self.assertTrue(a.access('.treeinfo')) + self.assertFalse(a.access('no_such_file')) + self.assertEqual(a.lastError, 404) + a.finish() diff --git a/tests/test_repository.py b/tests/test_repository.py new file mode 100644 index 00000000..833627d0 --- /dev/null +++ b/tests/test_repository.py @@ -0,0 +1,25 @@ +import unittest + +import xcp.accessor +from xcp import repository +from xcp.version import Version + +class TestRepository(unittest.TestCase): + def test_http(self): + raise unittest.SkipTest("comment out if you really mean it") + a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) + repo_ver = repository.BaseRepository.getRepoVer(a) + self.assertEqual(repo_ver, Version([3, 2, 1])) + product_ver = repository.BaseRepository.getProductVersion(a) + self.assertEqual(product_ver, Version([8, 2, 1])) + repos = repository.BaseRepository.findRepositories(a) + self.assertEqual(len(repos), 1) + + def test_file(self): + a = xcp.accessor.createAccessor("file://tests/data/repo/", True) + repo_ver = repository.BaseRepository.getRepoVer(a) + self.assertEqual(repo_ver, Version([3, 2, 1])) + product_ver = repository.BaseRepository.getProductVersion(a) + self.assertEqual(product_ver, Version([8, 2, 1])) + repos = repository.BaseRepository.findRepositories(a) + self.assertEqual(len(repos), 1) From a0ebbfd5515b539b06b0134be2d4efa80439df2c Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 28 Jul 2022 17:22:01 +0200 Subject: [PATCH 09/92] Prepare for futurize pass: fix "except" clause This is a common error documented in https://docs.python.org/2.7/whatsnew/2.6.html#pep-3110-exception-handling-changes Signed-off-by: Yann Dirson --- xcp/bootloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 51e362e8..4653caa5 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -431,7 +431,7 @@ def create_label(title): # If this fails, it is probably a string, so leave it unchanged. try: default = menu_order[int(default)] - except ValueError, KeyError: + except (ValueError, KeyError): pass finally: fh.close() From a8606c80d566acf1bfe839330b256229c317a0b9 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 11 Aug 2022 16:21:13 +0200 Subject: [PATCH 10/92] Futurize: first pass, using -wn1 Note there are indentation issues introduced in xcp.repository. Signed-off-by: Yann Dirson --- requirements-dev.txt | 2 + setup.py | 1 + tests/test_cpio.py | 3 +- tests/test_ifrename_logic.py | 29 +++++----- tests/test_pci.py | 2 +- xcp/accessor.py | 4 +- xcp/bootloader.py | 109 ++++++++++++++++++----------------- xcp/cpiofile.py | 101 ++++++++++++++++---------------- xcp/dom0.py | 3 +- xcp/environ.py | 4 +- xcp/mount.py | 5 +- xcp/net/ifrename/dynamic.py | 18 +++--- xcp/net/ifrename/static.py | 12 ++-- xcp/net/mac.py | 2 +- xcp/pci.py | 14 ++--- xcp/repository.py | 23 ++++---- xcp/xmlunwrap.py | 9 +-- 17 files changed, 176 insertions(+), 165 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 37123724..ecda27a2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,3 +5,5 @@ diff_cover mock pytest pytest-cov +# dependencies also in setup.py until they can be used +future diff --git a/setup.py b/setup.py index 9fb2bd83..0fdb3f1b 100644 --- a/setup.py +++ b/setup.py @@ -35,5 +35,6 @@ ], requires=[ 'branding', + 'future', ], ) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 84763b26..4cb13825 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import shutil import subprocess @@ -113,7 +114,7 @@ def test_bz2(self): def test_xz(self): if not self.doXZ: raise unittest.SkipTest("lzma package or xz tool not available") - print 'Running test for XZ' + print('Running test for XZ') self.doArchive('archive.cpio.xz', 'xz') # CpioFileCompat testing diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index 9fa9255c..c74522eb 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -1,3 +1,4 @@ +from __future__ import print_function import logging import sys import unittest @@ -43,15 +44,15 @@ def tearDown(self): self.siobuff.close() def debug_state(self, ts): - print >>sys.stderr, "" - print >>sys.stderr, self.siobuff.getvalue() - print >>sys.stderr, "" + print("", file=sys.stderr) + print(self.siobuff.getvalue(), file=sys.stderr) + print("", file=sys.stderr) if len(ts): for (s,d) in ts: - print >>sys.stderr, "'%s' -> '%s'" % (s, d) + print("'%s' -> '%s'" % (s, d), file=sys.stderr) else: - print >>sys.stderr, "No transactions" - print >>sys.stderr, "" + print("No transactions", file=sys.stderr) + print("", file=sys.stderr) def test_newhw_norules_1eth(self): @@ -268,16 +269,16 @@ def tearDown(self): self.siobuff.close() def debug_state(self, ts): - print >>sys.stderr, "" - print >>sys.stderr, self.siobuff.getvalue() - print >>sys.stderr, "" + print("", file=sys.stderr) + print(self.siobuff.getvalue(), file=sys.stderr) + print("", file=sys.stderr) if len(ts): - print >>sys.stderr, "Transactions:" + print("Transactions:", file=sys.stderr) for (s,d) in ts: - print >>sys.stderr, "'%s' -> '%s'" % (s, d) + print("'%s' -> '%s'" % (s, d), file=sys.stderr) else: - print >>sys.stderr, "No transactions" - print >>sys.stderr, "" + print("No transactions", file=sys.stderr) + print("", file=sys.stderr) def test_usecase1(self): """ @@ -559,7 +560,7 @@ def assertNotRaises(self, excp, fn, *argl, **kwargs): """Because unittest.TestCase seems to be missing this functionality""" try: fn(*argl, **kwargs) - except excp, e: + except excp as e: self.fail("function raised %s unexpectedly: %s" % (excp, e)) diff --git a/tests/test_pci.py b/tests/test_pci.py index 736b55de..17991428 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -9,7 +9,7 @@ class TestInvalid(unittest.TestCase): def test_invalid_types(self): self.assertRaises(TypeError, PCI, 0) - self.assertRaises(TypeError, PCI, 0L) + self.assertRaises(TypeError, PCI, 0) self.assertRaises(TypeError, PCI, (0,)) self.assertRaises(TypeError, PCI, []) self.assertRaises(TypeError, PCI, {}) diff --git a/xcp/accessor.py b/xcp/accessor.py index 22f9e24f..ecabb6c5 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -118,7 +118,7 @@ def openAddress(self, addr): class MountingAccessor(FilesystemAccessor): def __init__(self, mount_types, mount_source, mount_options = None): - ro = isinstance(mount_options, types.ListType) and 'ro' in mount_options + ro = isinstance(mount_options, list) and 'ro' in mount_options super(MountingAccessor, self).__init__(None, ro) self.mount_types = mount_types @@ -135,7 +135,7 @@ def start(self): try: opts = self.mount_options if fs == 'iso9660': - if isinstance(opts, types.ListType): + if isinstance(opts, list): if 'ro' not in opts: opts.append('ro') else: diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 4653caa5..441d3804 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -23,6 +23,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function import os import os.path import re @@ -455,47 +456,47 @@ def loadExisting(cls, root = '/'): elif os.path.exists(os.path.join(root, "boot/grub/menu.lst")): return cls.readGrub(os.path.join(root, "boot/grub/menu.lst")) else: - raise RuntimeError, "No existing bootloader configuration found" + raise RuntimeError("No existing bootloader configuration found") def writeExtLinux(self, dst_file = None): if hasattr(dst_file, 'name'): fh = dst_file else: fh = open(dst_file, 'w') - print >> fh, "# location " + self.location + print("# location " + self.location, file=fh) if self.serial: if self.serial.get('flow', None) is None: - print >> fh, "serial %s %s" % (self.serial['port'], - self.serial['baud']) + print("serial %s %s" % (self.serial['port'], + self.serial['baud']), file=fh) else: - print >> fh, "serial %s %s %s" % (self.serial['port'], + print("serial %s %s %s" % (self.serial['port'], self.serial['baud'], - self.serial['flow']) + self.serial['flow']), file=fh) if self.default: - print >> fh, "default " + self.default - print >> fh, "prompt 1" + print("default " + self.default, file=fh) + print("prompt 1", file=fh) if self.timeout: - print >> fh, "timeout %d" % self.timeout + print("timeout %d" % self.timeout, file=fh) for label in self.menu_order: - print >> fh, "\nlabel " + label + print("\nlabel " + label, file=fh) m = self.menu[label] if m.title: - print >> fh, " # " + m.title + print(" # " + m.title, file=fh) if m.tboot: - print >> fh, " kernel mboot.c32" - print >> fh, " append %s %s --- %s %s --- %s %s --- %s" % \ + print(" kernel mboot.c32", file=fh) + print(" append %s %s --- %s %s --- %s %s --- %s" % \ (m.tboot, m.tboot_args, m.hypervisor, m.hypervisor_args, - m.kernel, m.kernel_args, m.initrd) + m.kernel, m.kernel_args, m.initrd), file=fh) elif m.hypervisor: - print >> fh, " kernel mboot.c32" - print >> fh, " append %s %s --- %s %s --- %s" % \ - (m.hypervisor, m.hypervisor_args, m.kernel, m.kernel_args, m.initrd) + print(" kernel mboot.c32", file=fh) + print(" append %s %s --- %s %s --- %s" % \ + (m.hypervisor, m.hypervisor_args, m.kernel, m.kernel_args, m.initrd), file=fh) else: - print >> fh, " kernel " + m.kernel - print >> fh, " append " + m.kernel_args - print >> fh, " initrd " + m.initrd + print(" kernel " + m.kernel, file=fh) + print(" append " + m.kernel_args, file=fh) + print(" initrd " + m.initrd, file=fh) if not hasattr(dst_file, 'name'): fh.close() @@ -504,32 +505,32 @@ def writeGrub(self, dst_file = None): fh = dst_file else: fh = open(dst_file, 'w') - print >> fh, "# location " + self.location + print("# location " + self.location, file=fh) if self.serial: - print >> fh, "serial --unit=%s --speed=%s" % (self.serial['port'], - self.serial['baud']) - print >> fh, "terminal --timeout=10 console serial" + print("serial --unit=%s --speed=%s" % (self.serial['port'], + self.serial['baud']), file=fh) + print("terminal --timeout=10 console serial", file=fh) else: - print >> fh, "terminal console" + print("terminal console", file=fh) if self.default: for i in range(len(self.menu_order)): if self.menu_order[i] == self.default: - print >> fh, "default %d" % i + print("default %d" % i, file=fh) break if self.timeout: - print >> fh, "timeout %d" % (self.timeout / 10) + print("timeout %d" % (self.timeout / 10), file=fh) for label in self.menu_order: m = self.menu[label] - print >> fh, "\ntitle " + m.title + print("\ntitle " + m.title, file=fh) if m.hypervisor: - print >> fh, " kernel " + m.hypervisor + " " + m.hypervisor_args - print >> fh, " module " + m.kernel + " " + m.kernel_args - print >> fh, " module " + m.initrd + print(" kernel " + m.hypervisor + " " + m.hypervisor_args, file=fh) + print(" module " + m.kernel + " " + m.kernel_args, file=fh) + print(" module " + m.initrd, file=fh) else: - print >> fh, " kernel " + m.kernel + " " + m.kernel_args - print >> fh, " initrd " + m.initrd + print(" kernel " + m.kernel + " " + m.kernel_args, file=fh) + print(" initrd " + m.initrd, file=fh) if not hasattr(dst_file, 'name'): fh.close() @@ -540,19 +541,19 @@ def writeGrub2(self, dst_file = None): fh = open(dst_file, 'w') if self.serial: - print >> fh, "serial --unit=%s --speed=%s" % (self.serial['port'], - self.serial['baud']) - print >> fh, "terminal_input serial console" - print >> fh, "terminal_output serial console" + print("serial --unit=%s --speed=%s" % (self.serial['port'], + self.serial['baud']), file=fh) + print("terminal_input serial console", file=fh) + print("terminal_output serial console", file=fh) if self.default: for i in range(len(self.menu_order)): if self.menu_order[i] == self.default: - print >> fh, "set default=%d" % i + print("set default=%d" % i, file=fh) break else: - print >> fh, "set default='%s'" % str(self.default) + print("set default='%s'" % str(self.default), file=fh) if self.timeout: - print >> fh, "set timeout=%d" % (self.timeout / 10) + print("set timeout=%d" % (self.timeout / 10), file=fh) boilerplate = getattr(self, 'boilerplate', [])[:] boilerplate.reverse() @@ -563,41 +564,41 @@ def writeGrub2(self, dst_file = None): if boilerplate: text = boilerplate.pop() if text: - print >> fh, "\n".join(text) + print("\n".join(text), file=fh) extra = ' ' try: extra = m.extra except AttributeError: pass - print >> fh, "menuentry '%s'%s{" % (m.title, extra) + print("menuentry '%s'%s{" % (m.title, extra), file=fh) try: contents = "\n".join(m.contents) if contents: - print >> fh, contents + print(contents, file=fh) except AttributeError: pass if m.root: - print >> fh, "\tsearch --label --set root %s" % m.root + print("\tsearch --label --set root %s" % m.root, file=fh) if m.hypervisor: if m.tboot: - print >> fh, "\tmultiboot2 %s %s" % (m.tboot, m.tboot_args) - print >> fh, "\tmodule2 %s %s" % (m.hypervisor, m.hypervisor_args) + print("\tmultiboot2 %s %s" % (m.tboot, m.tboot_args), file=fh) + print("\tmodule2 %s %s" % (m.hypervisor, m.hypervisor_args), file=fh) else: - print >> fh, "\tmultiboot2 %s %s" % (m.hypervisor, m.hypervisor_args) + print("\tmultiboot2 %s %s" % (m.hypervisor, m.hypervisor_args), file=fh) if m.kernel: - print >> fh, "\tmodule2 %s %s" % (m.kernel, m.kernel_args) + print("\tmodule2 %s %s" % (m.kernel, m.kernel_args), file=fh) if m.initrd: - print >> fh, "\tmodule2 %s" % m.initrd + print("\tmodule2 %s" % m.initrd, file=fh) else: if m.kernel: - print >> fh, "\tlinux %s %s" % (m.kernel, m.kernel_args) + print("\tlinux %s %s" % (m.kernel, m.kernel_args), file=fh) if m.initrd: - print >> fh, "\tinitrd %s" % m.initrd - print >> fh, "}" + print("\tinitrd %s" % m.initrd, file=fh) + print("}", file=fh) if not hasattr(dst_file, 'name'): fh.close() @@ -642,9 +643,9 @@ def newDefault(cls, kernel_link_name, initrd_link_name, root = '/'): if b.menu[b.default].kernel != kernel_link_name: backup = [] if not os.path.exists(os.path.join(root, kernel_link_name[1:])): - raise RuntimeError, "kernel symlink not found" + raise RuntimeError("kernel symlink not found") if not os.path.exists(os.path.join(root, initrd_link_name[1:])): - raise RuntimeError, "initrd symlink not found" + raise RuntimeError("initrd symlink not found") old_kernel_link = b.menu[b.default].kernel old_ver = 'old' m = re.search(r'(-\d+\.\d+)-', old_kernel_link) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index a490aeff..a259efc3 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -33,6 +33,7 @@ Derived from Lars Gustäbel's tarfile.py """ +from __future__ import print_function __version__ = "0.1" __author__ = "Simon Rowe" @@ -55,7 +56,7 @@ # handling. In many places it is assumed a simple substitution of / by the # local os.path.sep is good enough to convert pathnames, but this does not # work with the mac rooted:path:name versus :nonrooted:path:name syntax - raise ImportError, "cpiofile does not work for platform==mac" + raise ImportError("cpiofile does not work for platform==mac") try: import grp as GRP, pwd as PWD @@ -78,26 +79,26 @@ #--------------------------------------------------------- # Bits used in the mode field, values in octal. #--------------------------------------------------------- -S_IFLNK = 0120000 # symbolic link -S_IFREG = 0100000 # regular file -S_IFBLK = 0060000 # block device -S_IFDIR = 0040000 # directory -S_IFCHR = 0020000 # character device -S_IFIFO = 0010000 # fifo - -TSUID = 04000 # set UID on execution -TSGID = 02000 # set GID on execution -TSVTX = 01000 # reserved - -TUREAD = 0400 # read by owner -TUWRITE = 0200 # write by owner -TUEXEC = 0100 # execute/search by owner -TGREAD = 0040 # read by group -TGWRITE = 0020 # write by group -TGEXEC = 0010 # execute/search by group -TOREAD = 0004 # read by other -TOWRITE = 0002 # write by other -TOEXEC = 0001 # execute/search by other +S_IFLNK = 0o120000 # symbolic link +S_IFREG = 0o100000 # regular file +S_IFBLK = 0o060000 # block device +S_IFDIR = 0o040000 # directory +S_IFCHR = 0o020000 # character device +S_IFIFO = 0o010000 # fifo + +TSUID = 0o4000 # set UID on execution +TSGID = 0o2000 # set GID on execution +TSVTX = 0o1000 # reserved + +TUREAD = 0o400 # read by owner +TUWRITE = 0o200 # write by owner +TUEXEC = 0o100 # execute/search by owner +TGREAD = 0o040 # read by group +TGWRITE = 0o020 # write by group +TGEXEC = 0o010 # execute/search by group +TOREAD = 0o004 # read by other +TOWRITE = 0o002 # write by other +TOEXEC = 0o001 # execute/search by other #--------------------------------------------------------- # Some useful functions @@ -253,7 +254,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.fileobj = fileobj self.bufsize = bufsize self.buf = "" - self.pos = 0L + self.pos = 0 self.closed = False if comptype == "gz": @@ -347,8 +348,8 @@ def close(self): # while the same crc on a 64-bit box may "look positive". # To avoid irksome warnings from the `struct` module, force # it to look positive on all boxes. - self.fileobj.write(struct.pack(" 1: - if self.hardlinks and self.inodes.has_key(cpioinfo.ino): + if self.hardlinks and cpioinfo.ino in self.inodes: # this inode has already been added cpioinfo.size = 0 self.inodes[cpioinfo.ino].append(cpioinfo.name) @@ -1418,7 +1419,7 @@ def extractall(self, path=".", members=None): # Extract directory with a safe mode, so that # all files below can be extracted as well. try: - os.makedirs(os.path.join(path, cpioinfo.name), 0777) + os.makedirs(os.path.join(path, cpioinfo.name), 0o777) except EnvironmentError: pass directories.append(cpioinfo) @@ -1436,7 +1437,7 @@ def extractall(self, path=".", members=None): self.chown(cpioinfo, path) self.utime(cpioinfo, path) self.chmod(cpioinfo, path) - except ExtractError, e: + except ExtractError as e: if self.errorlevel > 1: raise else: @@ -1462,7 +1463,7 @@ def extract(self, member, path=""): try: self._extract_member(cpioinfo, os.path.join(path, cpioinfo.name)) - except EnvironmentError, e: + except EnvironmentError as e: if self.errorlevel > 0: raise else: @@ -1470,7 +1471,7 @@ def extract(self, member, path=""): self._dbg(1, "cpiofile: %s" % e.strerror) else: self._dbg(1, "cpiofile: %s %r" % (e.strerror, e.filename)) - except ExtractError, e: + except ExtractError as e: if self.errorlevel > 1: raise else: @@ -1525,7 +1526,7 @@ def _extract_member(self, cpioinfo, cpiogetpath): if upperdirs and not os.path.exists(upperdirs): ti = CpioInfo() ti.name = upperdirs - ti.mode = S_IFDIR | 0777 + ti.mode = S_IFDIR | 0o777 ti.mtime = cpioinfo.mtime ti.uid = cpioinfo.uid ti.gid = cpioinfo.gid @@ -1567,7 +1568,7 @@ def makedir(self, cpioinfo, cpiogetpath): """ try: os.mkdir(cpiogetpath) - except EnvironmentError, e: + except EnvironmentError as e: if e.errno != errno.EEXIST: raise @@ -1578,7 +1579,7 @@ def makefile(self, cpioinfo, cpiogetpath): if cpioinfo.nlink == 1: extractinfo = cpioinfo else: - if self.inodes.has_key(cpioinfo.ino): + if cpioinfo.ino in self.inodes: # actual file exists, create link # FIXME handle platforms that don't support hardlinks os.link(os.path.join(cpioinfo._link_path, @@ -1738,7 +1739,7 @@ def next(self): cpioinfo = self.proc_member(cpioinfo) - except ValueError, e: + except ValueError as e: if self.offset == 0: raise ReadError("empty, unreadable or compressed " "file: %s" % e) @@ -1803,7 +1804,7 @@ def _load(self): members. """ while True: - cpioinfo = self.next() + cpioinfo = next(self) if cpioinfo is None: break self._loaded = True @@ -1829,7 +1830,7 @@ def _dbg(self, level, msg): """Write debugging output to sys.stderr. """ if level <= self.debug: - print >> sys.stderr, msg + print(msg, file=sys.stderr) # class CpioFile class CpioIter(object): @@ -1856,7 +1857,7 @@ def next(self): # happen that getmembers() is called during iteration, # which will cause CpioIter to stop prematurely. if not self.cpiofile._loaded: - cpioinfo = self.cpiofile.next() + cpioinfo = next(self.cpiofile) if not cpioinfo: self.cpiofile._loaded = True raise StopIteration diff --git a/xcp/dom0.py b/xcp/dom0.py index f2ca7097..66a23a9e 100644 --- a/xcp/dom0.py +++ b/xcp/dom0.py @@ -23,9 +23,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import re -import version +from . import version import sys def default_memory_v2(host_mem_kib): diff --git a/xcp/environ.py b/xcp/environ.py index 27e57323..c8901f6d 100644 --- a/xcp/environ.py +++ b/xcp/environ.py @@ -47,7 +47,7 @@ def readInventory(root = '/'): try: fh = open(os.path.join(root, 'etc/xensource-inventory')) - for line in ( x for x in ( y.strip() for y in fh.xreadlines() ) + for line in ( x for x in ( y.strip() for y in fh ) if not x.startswith('#') ): vals = line.split('=', 1) @@ -59,7 +59,7 @@ def readInventory(root = '/'): d[vals[0]] = vals[1].strip('"\'') - except IOError, e: + except IOError as e: raise InventoryError("Error reading from file '%s'" % (e,)) finally: diff --git a/xcp/mount.py b/xcp/mount.py index 60d191bc..647817fa 100644 --- a/xcp/mount.py +++ b/xcp/mount.py @@ -23,6 +23,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from future.utils import raise_ import os import os.path import tempfile @@ -52,13 +53,13 @@ def mount(dev, mountpoint, options = None, fstype = None, label = None): rc, out, err = xcp.cmd.runCmd(cmd, with_stdout=True, with_stderr=True) if rc != 0: - raise MountException, "out: '%s' err: '%s'" % (out, err) + raise_(MountException, "out: '%s' err: '%s'" % (out, err)) def bindMount(source, mountpoint): cmd = [ '/bin/mount', '--bind', source, mountpoint] rc, out, err = xcp.cmd.runCmd(cmd, with_stdout=True, with_stderr=True) if rc != 0: - raise MountException, "out: '%s' err: '%s'" % (out, err) + raise_(MountException, "out: '%s' err: '%s'" % (out, err)) def umount(mountpoint, force = False): # -d option also removes the loop device (if present) diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index 62844848..befab65a 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -104,7 +104,7 @@ def load_and_parse(self): LOG.error("No source of data to parse") return False - except IOError, e: + except IOError as e: LOG.error("IOError while reading file: %s" % (e,)) return False finally: @@ -137,7 +137,7 @@ def load_and_parse(self): if len(entry) != 3: raise ValueError("Expected 3 entries") macpci = MACPCI(entry[0], entry[1], tname=entry[2]) - except (TypeError, ValueError), e: + except (TypeError, ValueError) as e: LOG.warning("Invalid lastboot data entry: %s" % (e,)) continue @@ -149,7 +149,7 @@ def load_and_parse(self): if len(entry) != 3: raise ValueError("Expected 3 entries") macpci = MACPCI(entry[0], entry[1], tname=entry[2]) - except (TypeError, ValueError), e: + except (TypeError, ValueError) as e: LOG.warning("Invalid old data entry: %s" % (e,)) continue self.old.append(macpci) @@ -178,7 +178,7 @@ def generate(self, state): if nic.mac == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -199,7 +199,7 @@ def generate(self, state): if nic.ppn == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -213,7 +213,7 @@ def generate(self, state): try: nic = pci_sbdfi_to_nic(value, state) rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -226,7 +226,7 @@ def generate(self, state): if nic.label == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -257,7 +257,7 @@ def validate(entry): return False MACPCI(entry[0], entry[1], tname=entry[2]) return True - except Exception, e: + except Exception as e: LOG.warning("Failed to validate '%s' because '%s'" % (entry, e)) return False @@ -301,7 +301,7 @@ def save(self, header = True): LOG.error("No source of data to parse") return False - except IOError, e: + except IOError as e: LOG.error("IOError while reading file: %s" % (e,)) return False finally: diff --git a/xcp/net/ifrename/static.py b/xcp/net/ifrename/static.py index 89a77fda..615c2073 100644 --- a/xcp/net/ifrename/static.py +++ b/xcp/net/ifrename/static.py @@ -127,7 +127,7 @@ def load_and_parse(self): LOG.error("No source of data to parse") return False - except IOError, e: + except IOError as e: LOG.error("IOError while reading file: %s" % (e,)) return False finally: @@ -230,7 +230,7 @@ def generate(self, state): if nic.mac == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -251,7 +251,7 @@ def generate(self, state): if nic.ppn == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -265,7 +265,7 @@ def generate(self, state): try: nic = pci_sbdfi_to_nic(value, state) rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -278,7 +278,7 @@ def generate(self, state): if nic.label == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -340,7 +340,7 @@ def save(self, header = True): LOG.error("No source of data to parse") return False - except IOError, e: + except IOError as e: LOG.error("IOError while reading file: %s" % (e,)) return False finally: diff --git a/xcp/net/mac.py b/xcp/net/mac.py index a675a179..9b042c8f 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -57,7 +57,7 @@ def __init__(self, addr): """Constructor""" self.octets = [] - self.integer = -1L + self.integer = -1 if isinstance(addr, (str, unicode)): diff --git a/xcp/pci.py b/xcp/pci.py index 8b371152..367e7dc5 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -124,7 +124,7 @@ def __eq__(self, rhs): else: try: return self.integer == PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __ne__(self, rhs): @@ -133,7 +133,7 @@ def __ne__(self, rhs): else: try: return self.integer != PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __hash__(self): @@ -145,7 +145,7 @@ def __lt__(self, rhs): else: try: return self.integer < PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __le__(self, rhs): @@ -154,7 +154,7 @@ def __le__(self, rhs): else: try: return self.integer <= PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __gt__(self, rhs): @@ -163,7 +163,7 @@ def __gt__(self, rhs): else: try: return self.integer > PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __ge__(self, rhs): @@ -172,7 +172,7 @@ def __ge__(self, rhs): else: try: return self.integer >= PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented @@ -232,7 +232,7 @@ def read(cls): for f in ['/usr/share/hwdata/pci.ids']: if os.path.exists(f): return cls(f) - raise Exception, 'Failed to open PCI database' + raise Exception('Failed to open PCI database') def findVendor(self, vendor): return vendor in self.vendor_dict and self.vendor_dict[vendor] or None diff --git a/xcp/repository.py b/xcp/repository.py index 3da2f17b..5b23a6ba 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -23,6 +23,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from future.utils import raise_ import md5 import os.path import xml.dom.minidom @@ -187,8 +188,8 @@ def _getVersion(cls, access, category): ver_str = treeinfo.get(category, 'version') repo_ver = version.Version.from_string(ver_str) - except Exception, e: - raise RepoFormatError, "Failed to open %s: %s" % (cls.TREEINFO_FILENAME, str(e)) + except Exception as e: + raise_(RepoFormatError, "Failed to open %s: %s" % (cls.TREEINFO_FILENAME, str(e))) access.finish() return repo_ver @@ -230,8 +231,8 @@ def findRepositories(cls, access): for line in extra: package_list.append(line.strip()) extra.close() - except Exception, e: - raise RepoFormatError, "Failed to open %s: %s" % (cls.REPOLIST_FILENAME, str(e)) + except Exception as e: + raise_(RepoFormatError, "Failed to open %s: %s" % (cls.REPOLIST_FILENAME, str(e))) for loc in package_list: if cls.isRepo(access, loc): @@ -250,17 +251,17 @@ def __init__(self, access, base, is_group = False): try: repofile = access.openAddress(os.path.join(base, self.REPOSITORY_FILENAME)) - except Exception, e: + except Exception as e: access.finish() - raise NoRepository, e +raise_(NoRepository, e) self._parse_repofile(repofile) repofile.close() try: pkgfile = access.openAddress(os.path.join(base, self.PKGDATA_FILENAME)) - except Exception, e: + except Exception as e: access.finish() - raise NoRepository, e +raise_(NoRepository, e) self._parse_packages(pkgfile) pkgfile.close() @@ -290,7 +291,7 @@ def _parse_repofile(self, repofile): try: xmldoc = xml.dom.minidom.parseString(repofile_contents) except: - raise RepoFormatError, "%s not in XML" % self.REPOSITORY_FILENAME + raise_(RepoFormatError, "%s not in XML" % self.REPOSITORY_FILENAME) try: repo_node = xmlunwrap.getElementsByTagName(xmldoc, ['repository'], mandatory = True) @@ -314,7 +315,7 @@ def _parse_repofile(self, repofile): assert req['test'] in self.OPER_MAP self.requires.append(req) except: - raise RepoFormatError, "%s format error" % self.REPOSITORY_FILENAME + raise_(RepoFormatError, "%s format error" % self.REPOSITORY_FILENAME) self.identifier = "%s:%s" % (self.originator, self.name) ver_str = self.version @@ -333,7 +334,7 @@ def _parse_packages(self, pkgfile): try: xmldoc = xml.dom.minidom.parseString(pkgfile_contents) except: - raise RepoFormatError, "%s not in XML" % self.PKGDATA_FILENAME + raise_(RepoFormatError, "%s not in XML" % self.PKGDATA_FILENAME) for pkg_node in xmlunwrap.getElementsByTagName(xmldoc, ['package']): pkg = self._create_package(pkg_node) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 654b84bf..6f50988c 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -23,6 +23,7 @@ """xmlunwrap - general methods to unwrap XML elements & attributes""" +from future.utils import raise_ class XmlUnwrapError(Exception): pass @@ -39,7 +40,7 @@ def getElementsByTagName(el, tags, mandatory = False): for tag in tags: matching.extend(el.getElementsByTagName(tag)) if mandatory and len(matching) == 0: - raise XmlUnwrapError, "Missing mandatory element %s" % tags[0] + raise_(XmlUnwrapError, "Missing mandatory element %s" % tags[0]) return matching def getStrAttribute(el, attrs, default = '', mandatory = False): @@ -50,7 +51,7 @@ def getStrAttribute(el, attrs, default = '', mandatory = False): matching.append(val) if len(matching) == 0: if mandatory: - raise XmlUnwrapError, "Missing mandatory attribute %s" % attrs[0] + raise_(XmlUnwrapError, "Missing mandatory attribute %s" % attrs[0]) return default return matching[0] @@ -69,7 +70,7 @@ def getIntAttribute(el, attrs, default = None): try: int_val = int(val, 0) except: - raise XmlUnwrapError, "Invalid integer value for %s" % attrs[0] + raise_(XmlUnwrapError, "Invalid integer value for %s" % attrs[0]) return int_val def getMapAttribute(el, attrs, mapping, default = None): @@ -78,7 +79,7 @@ def getMapAttribute(el, attrs, mapping, default = None): key = getStrAttribute(el, attrs, default, mandatory) if key not in k: - raise XmlUnwrapError, "Unexpected key %s for attribute" % key + raise_(XmlUnwrapError, "Unexpected key %s for attribute" % key) k_list = list(k) return v[k_list.index(key)] From c25da360f14deedd2fa77de8efdfe7396cb0e464 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 27 Jul 2022 11:46:05 +0200 Subject: [PATCH 11/92] coverage: new test_environ: parse a sample "xensource-inventory" Coverage change: xcp.environ: 0% -> 80.0% total: 62.7% -> 63.3% Signed-off-by: Yann Dirson --- tests/data/inventory/etc/xensource-inventory | 23 ++++++++++++++++++++ tests/test_environ.py | 8 +++++++ 2 files changed, 31 insertions(+) create mode 100644 tests/data/inventory/etc/xensource-inventory create mode 100644 tests/test_environ.py diff --git a/tests/data/inventory/etc/xensource-inventory b/tests/data/inventory/etc/xensource-inventory new file mode 100644 index 00000000..950472c0 --- /dev/null +++ b/tests/data/inventory/etc/xensource-inventory @@ -0,0 +1,23 @@ +PRIMARY_DISK='/dev/disk/by-id/scsi-123456789012345678901234567899012' +PRODUCT_VERSION='8.2.1' +DOM0_VCPUS='16' +CONTROL_DOMAIN_UUID='3b1dccee-9da4-49c6-8250-e3725e0e4b2d' +DOM0_MEM='7584' +COMPANY_NAME_SHORT='Open Source' +MANAGEMENT_ADDRESS_TYPE='IPv4' +PARTITION_LAYOUT='ROOT,BACKUP,LOG,BOOT,SWAP,SR' +PRODUCT_VERSION_TEXT='8.2' +PRODUCT_BRAND='XCP-ng' +INSTALLATION_UUID='b8b63475-d28c-44d4-9d1e-c38298ad3a0d' +PRODUCT_VERSION_TEXT_SHORT='8.2' +BRAND_CONSOLE='XCP-ng Center' +PRODUCT_NAME='xenenterprise' +MANAGEMENT_INTERFACE='xapi1' +COMPANY_PRODUCT_BRAND='XCP-ng' +PLATFORM_VERSION='3.2.1' +BUILD_NUMBER='release/yangtze/master/58' +PLATFORM_NAME='XCP' +BRAND_CONSOLE_URL='https://xcp-ng.org' +BACKUP_PARTITION='/dev/disk/by-id/scsi-123456789012345678901234567899012-part2' +INSTALLATION_DATE='2021-08-03 16:39:24.077945' +COMPANY_NAME='Open Source' diff --git a/tests/test_environ.py b/tests/test_environ.py new file mode 100644 index 00000000..8b0da44e --- /dev/null +++ b/tests/test_environ.py @@ -0,0 +1,8 @@ +import unittest +import xcp.environ + +class TestEnviron(unittest.TestCase): + def test_read_inventory(self): + inventory = xcp.environ.readInventory(root="tests/data/inventory") + self.assertEqual(inventory["COMPANY_PRODUCT_BRAND"], 'XCP-ng') + self.assertEqual(inventory["COMPANY_NAME"], 'Open Source') From 3d91e2e193272e6a0c7c6d74d804b03364c516ed Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 28 Jul 2022 17:28:13 +0200 Subject: [PATCH 12/92] Futurize fixes: drop now-duplicated test Signed-off-by: Yann Dirson --- tests/test_pci.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pci.py b/tests/test_pci.py index 17991428..ac03c375 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -8,7 +8,6 @@ class TestInvalid(unittest.TestCase): def test_invalid_types(self): - self.assertRaises(TypeError, PCI, 0) self.assertRaises(TypeError, PCI, 0) self.assertRaises(TypeError, PCI, (0,)) self.assertRaises(TypeError, PCI, []) From c364a314dc03b3e66b17c827953ed065b3ee03c5 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 27 Jul 2022 15:13:22 +0200 Subject: [PATCH 13/92] tests: remove unused imports, direct test execution Direct test execution only brings us dependency on `sys` :) Signed-off-by: Yann Dirson --- tests/test_biosdevname.py | 6 +----- tests/test_cpio.py | 11 ++++------- tests/test_dom0.py | 9 ++------- tests/test_ifrename_dynamic.py | 9 ++------- tests/test_ifrename_logic.py | 9 +++------ tests/test_ifrename_static.py | 9 ++------- tests/test_mac.py | 7 +------ tests/test_pci.py | 8 +------- 8 files changed, 16 insertions(+), 52 deletions(-) diff --git a/tests/test_biosdevname.py b/tests/test_biosdevname.py index 07133bec..5616afb0 100644 --- a/tests/test_biosdevname.py +++ b/tests/test_biosdevname.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python +import unittest -import unittest, sys, os, os.path as path from xcp.net.biosdevname import has_ppn_quirks class TestQuirks(unittest.TestCase): @@ -24,6 +23,3 @@ def test_ppn_true(self): {"SMBIOS Instance": 1} ])) - -if __name__ == "__main__": - sys.exit(unittest.main()) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 85ff62d8..16a4a890 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -1,10 +1,10 @@ -#!/usr/bin/env python - -import unittest, sys, os, os.path as path +import os +import shutil +import subprocess +import unittest import warnings from xcp.cpiofile import CpioFile, CpioInfo -import subprocess, shutil try: from hashlib import md5 @@ -115,6 +115,3 @@ def test_xz(self): raise unittest.SkipTest("lzma package or xz tool not available") print 'Running test for XZ' self.doArchive('archive.cpio.xz', 'xz') - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_dom0.py b/tests/test_dom0.py index 2fc1746d..bf198341 100644 --- a/tests/test_dom0.py +++ b/tests/test_dom0.py @@ -1,9 +1,7 @@ -#!/usr/bin/env python - -import unittest, sys, os +import unittest +from mock import patch, Mock from xcp.dom0 import default_memory, parse_mem, default_vcpus -from mock import patch, Mock class TestDom0(unittest.TestCase): @@ -112,6 +110,3 @@ def test_default_vcpus(self): for host_pcpus, mem, expected in test_values: calculated = default_vcpus(host_pcpus, mem) self.assertEqual(calculated, expected) - -if __name__ == "__main__": - sys.exit(unittest.main()) diff --git a/tests/test_ifrename_dynamic.py b/tests/test_ifrename_dynamic.py index 2048f762..1eb28a74 100644 --- a/tests/test_ifrename_dynamic.py +++ b/tests/test_ifrename_dynamic.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python - -import unittest, sys, os, os.path as path, logging import json +import logging +import unittest from copy import deepcopy try: @@ -175,7 +174,3 @@ def test_one_ibft_lastboot(self): self.assertEqual(json.loads(dr.write(False)), {'lastboot': [], 'old': []}) - - -if __name__ == "__main__": - sys.exit(unittest.main()) diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index 326f8acb..9fa9255c 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python - -import unittest, sys, os, os.path as path, logging +import logging +import sys +import unittest from copy import deepcopy try: @@ -651,6 +651,3 @@ def test_oldstate_input(self): self.assertNotRaises(OldStateError, rename, [], [], [], [self.s123]) - -if __name__ == "__main__": - sys.exit(unittest.main()) diff --git a/tests/test_ifrename_static.py b/tests/test_ifrename_static.py index 8b72b607..16909142 100644 --- a/tests/test_ifrename_static.py +++ b/tests/test_ifrename_static.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python - -import unittest, sys, os, os.path as path, logging +import logging +import unittest from copy import deepcopy try: @@ -473,7 +472,3 @@ def test_two_valid(self): ) self.assertEqual(sr.write(False), desired_result) - - -if __name__ == "__main__": - sys.exit(unittest.main()) diff --git a/tests/test_mac.py b/tests/test_mac.py index 77ef2e79..310e17e3 100644 --- a/tests/test_mac.py +++ b/tests/test_mac.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python - -import unittest, sys, os, os.path as path +import unittest from xcp.net.mac import MAC @@ -235,6 +233,3 @@ def test_keys(self): self.assertEqual(d[m1], "zero") self.assertEqual(d[m2], "ascending") self.assertEqual(d[m3], "random") - -if __name__ == "__main__": - sys.exit(unittest.main()) diff --git a/tests/test_pci.py b/tests/test_pci.py index 59af4828..736b55de 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -1,7 +1,5 @@ -#!/usr/bin/env python - -import unittest, sys, os, os.path as path import subprocess +import unittest from mock import patch, Mock from xcp.pci import PCI, PCIIds, PCIDevices @@ -104,7 +102,3 @@ def tests_videoclass(self): self.assertEqual(ids.findDevice(video_dev['vendor'], video_dev['device']), device) self.assertEqual(len(devs.findRelatedFunctions('00:18.1')), 7) - - -if __name__ == "__main__": - sys.exit(unittest.main()) From b06612054a1c94e881a66765ed617496bc14d833 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 12 Jul 2022 11:50:45 +0200 Subject: [PATCH 14/92] Futurize fixes: fix broken indentations Signed-off-by: Yann Dirson --- xcp/repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/repository.py b/xcp/repository.py index 5b23a6ba..63b1277a 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -253,7 +253,7 @@ def __init__(self, access, base, is_group = False): repofile = access.openAddress(os.path.join(base, self.REPOSITORY_FILENAME)) except Exception as e: access.finish() -raise_(NoRepository, e) + raise_(NoRepository, e) self._parse_repofile(repofile) repofile.close() @@ -261,7 +261,7 @@ def __init__(self, access, base, is_group = False): pkgfile = access.openAddress(os.path.join(base, self.PKGDATA_FILENAME)) except Exception as e: access.finish() -raise_(NoRepository, e) + raise_(NoRepository, e) self._parse_packages(pkgfile) pkgfile.close() From 57a35b0cc2697012fe79daecc257093aa5cf5764 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 27 Jul 2022 15:12:00 +0200 Subject: [PATCH 15/92] Coverage: test_biosdevname: add a test of all_devices_all_names() Coverage change: xcp.net.biosdevname: 41.5% to 87.8% total: 63.3% -> 63.8% Signed-off-by: Yann Dirson --- tests/data/all_ethN.biosdevname | 68 +++++++++++++++++++++++++++++++++ tests/data/physical.biosdevname | 68 +++++++++++++++++++++++++++++++++ tests/test_biosdevname.py | 25 +++++++++++- 3 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tests/data/all_ethN.biosdevname create mode 100644 tests/data/physical.biosdevname diff --git a/tests/data/all_ethN.biosdevname b/tests/data/all_ethN.biosdevname new file mode 100644 index 00000000..63bdc506 --- /dev/null +++ b/tests/data/all_ethN.biosdevname @@ -0,0 +1,68 @@ +BIOS device: eth0 +Kernel name: eth0 +Permanent MAC: EC:F4:BB:C3:AF:A8 +Assigned MAC : EC:F4:BB:C3:AF:A8 +Driver: ixgbe +Driver version: 5.9.4 +Firmware version: 0x8000095c, 19.5.12 +Bus Info: 0000:01:00.0 +PCI name : 0000:01:00.0 +PCI Slot : embedded +SMBIOS Device Type: Ethernet +SMBIOS Instance: 1 +SMBIOS Label: Integrated NIC 1 +sysfs Label: NIC1 +Embedded Index: 1 + +BIOS device: eth1 +Kernel name: eth1 +Permanent MAC: EC:F4:BB:C3:AF:AA +Assigned MAC : EC:F4:BB:C3:AF:AA +Driver: ixgbe +Driver version: 5.9.4 +Firmware version: 0x8000095c, 19.5.12 +Bus Info: 0000:01:00.1 +PCI name : 0000:01:00.1 +PCI Slot : embedded +SMBIOS Device Type: Ethernet +SMBIOS Instance: 2 +SMBIOS Label: Integrated NIC 2 +sysfs Label: NIC2 +Embedded Index: 2 + +BIOS device: eth2 +Kernel name: eth2 +Permanent MAC: EC:F4:BB:C3:AF:AC +Assigned MAC : EC:F4:BB:C3:AF:AC +Driver: igb +Driver version: 5.3.5.20 +Firmware version: 1.67, 0x80000fab, 19.5.12 +Bus Info: 0000:07:00.0 +PCI name : 0000:07:00.0 +PCI Slot : embedded +SMBIOS Device Type: Ethernet +SMBIOS Instance: 3 +SMBIOS Label: Integrated NIC 3 +sysfs Label: NIC3 +VPD Port: 3 +VPD Index: 1 +VPD PCI master: 0000:07:00.0 + +BIOS device: eth3 +Kernel name: eth3 +Permanent MAC: EC:F4:BB:C3:AF:AD +Assigned MAC : EC:F4:BB:C3:AF:AD +Driver: igb +Driver version: 5.3.5.20 +Firmware version: 1.67, 0x80000fab, 19.5.12 +Bus Info: 0000:07:00.1 +PCI name : 0000:07:00.1 +PCI Slot : embedded +SMBIOS Device Type: Ethernet +SMBIOS Instance: 4 +SMBIOS Label: Integrated NIC 4 +sysfs Label: NIC4 +VPD Port: 4 +VPD Index: 1 +VPD PCI master: 0000:07:00.1 + diff --git a/tests/data/physical.biosdevname b/tests/data/physical.biosdevname new file mode 100644 index 00000000..3e0611d7 --- /dev/null +++ b/tests/data/physical.biosdevname @@ -0,0 +1,68 @@ +BIOS device: em1 +Kernel name: eth0 +Permanent MAC: EC:F4:BB:C3:AF:A8 +Assigned MAC : EC:F4:BB:C3:AF:A8 +Driver: ixgbe +Driver version: 5.9.4 +Firmware version: 0x8000095c, 19.5.12 +Bus Info: 0000:01:00.0 +PCI name : 0000:01:00.0 +PCI Slot : embedded +SMBIOS Device Type: Ethernet +SMBIOS Instance: 1 +SMBIOS Label: Integrated NIC 1 +sysfs Label: NIC1 +Embedded Index: 1 + +BIOS device: em2 +Kernel name: eth1 +Permanent MAC: EC:F4:BB:C3:AF:AA +Assigned MAC : EC:F4:BB:C3:AF:AA +Driver: ixgbe +Driver version: 5.9.4 +Firmware version: 0x8000095c, 19.5.12 +Bus Info: 0000:01:00.1 +PCI name : 0000:01:00.1 +PCI Slot : embedded +SMBIOS Device Type: Ethernet +SMBIOS Instance: 2 +SMBIOS Label: Integrated NIC 2 +sysfs Label: NIC2 +Embedded Index: 2 + +BIOS device: em3_1 +Kernel name: eth2 +Permanent MAC: EC:F4:BB:C3:AF:AC +Assigned MAC : EC:F4:BB:C3:AF:AC +Driver: igb +Driver version: 5.3.5.20 +Firmware version: 1.67, 0x80000fab, 19.5.12 +Bus Info: 0000:07:00.0 +PCI name : 0000:07:00.0 +PCI Slot : embedded +SMBIOS Device Type: Ethernet +SMBIOS Instance: 3 +SMBIOS Label: Integrated NIC 3 +sysfs Label: NIC3 +VPD Port: 3 +VPD Index: 1 +VPD PCI master: 0000:07:00.0 + +BIOS device: em4_1 +Kernel name: eth3 +Permanent MAC: EC:F4:BB:C3:AF:AD +Assigned MAC : EC:F4:BB:C3:AF:AD +Driver: igb +Driver version: 5.3.5.20 +Firmware version: 1.67, 0x80000fab, 19.5.12 +Bus Info: 0000:07:00.1 +PCI name : 0000:07:00.1 +PCI Slot : embedded +SMBIOS Device Type: Ethernet +SMBIOS Instance: 4 +SMBIOS Label: Integrated NIC 4 +sysfs Label: NIC4 +VPD Port: 4 +VPD Index: 1 +VPD PCI master: 0000:07:00.1 + diff --git a/tests/test_biosdevname.py b/tests/test_biosdevname.py index 5616afb0..75a3a266 100644 --- a/tests/test_biosdevname.py +++ b/tests/test_biosdevname.py @@ -1,6 +1,7 @@ import unittest +from mock import patch, Mock -from xcp.net.biosdevname import has_ppn_quirks +from xcp.net.biosdevname import has_ppn_quirks, all_devices_all_names class TestQuirks(unittest.TestCase): @@ -23,3 +24,25 @@ def test_ppn_true(self): {"SMBIOS Instance": 1} ])) +class TestDeviceNames(unittest.TestCase): + def test(self): + with patch("xcp.net.biosdevname.Popen") as popen_mock: + with open("tests/data/physical.biosdevname") as f: + fake_output_1 = f.read() + with open("tests/data/all_ethN.biosdevname") as f: + fake_output_2 = f.read() + communicate_mock = Mock(side_effect=iter([(fake_output_1, ""), + (fake_output_2, "")])) + popen_mock.return_value.communicate = communicate_mock + popen_mock.return_value.returncode = 0 + + devices = all_devices_all_names() + + # check after the fact that we mocked the proper calls + self.assertEqual(popen_mock.call_count, 2) + calls = popen_mock.call_args_list + self.assertEqual(calls[0].args[0], ['/sbin/biosdevname', '--policy', 'physical', '-d']) + self.assertEqual(calls[1].args[0], ['/sbin/biosdevname', '--policy', 'all_ethN', '-d']) + + self.assertEqual(devices['eth0']['BIOS device'], + {'all_ethN': 'eth0', 'physical': 'em1'}) From ea40073e7eb599fecbbfc45034feaafdefd71d60 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 12 Jul 2022 11:51:09 +0200 Subject: [PATCH 16/92] Futurize fixes: use "raise" not "raise_" when not reraising Signed-off-by: Yann Dirson --- xcp/mount.py | 5 ++--- xcp/xmlunwrap.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/xcp/mount.py b/xcp/mount.py index 647817fa..49b0a2e8 100644 --- a/xcp/mount.py +++ b/xcp/mount.py @@ -23,7 +23,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from future.utils import raise_ import os import os.path import tempfile @@ -53,13 +52,13 @@ def mount(dev, mountpoint, options = None, fstype = None, label = None): rc, out, err = xcp.cmd.runCmd(cmd, with_stdout=True, with_stderr=True) if rc != 0: - raise_(MountException, "out: '%s' err: '%s'" % (out, err)) + raise MountException("out: '%s' err: '%s'" % (out, err)) def bindMount(source, mountpoint): cmd = [ '/bin/mount', '--bind', source, mountpoint] rc, out, err = xcp.cmd.runCmd(cmd, with_stdout=True, with_stderr=True) if rc != 0: - raise_(MountException, "out: '%s' err: '%s'" % (out, err)) + raise MountException("out: '%s' err: '%s'" % (out, err)) def umount(mountpoint, force = False): # -d option also removes the loop device (if present) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 6f50988c..59404211 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -40,7 +40,7 @@ def getElementsByTagName(el, tags, mandatory = False): for tag in tags: matching.extend(el.getElementsByTagName(tag)) if mandatory and len(matching) == 0: - raise_(XmlUnwrapError, "Missing mandatory element %s" % tags[0]) + raise XmlUnwrapError("Missing mandatory element %s" % tags[0]) return matching def getStrAttribute(el, attrs, default = '', mandatory = False): @@ -51,7 +51,7 @@ def getStrAttribute(el, attrs, default = '', mandatory = False): matching.append(val) if len(matching) == 0: if mandatory: - raise_(XmlUnwrapError, "Missing mandatory attribute %s" % attrs[0]) + raise XmlUnwrapError("Missing mandatory attribute %s" % attrs[0]) return default return matching[0] @@ -79,7 +79,7 @@ def getMapAttribute(el, attrs, mapping, default = None): key = getStrAttribute(el, attrs, default, mandatory) if key not in k: - raise_(XmlUnwrapError, "Unexpected key %s for attribute" % key) + raise XmlUnwrapError("Unexpected key %s for attribute" % key) k_list = list(k) return v[k_list.index(key)] From 1f19cc9695d03118cebb5cef87ef5382ad66275f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 27 Jul 2022 15:25:00 +0200 Subject: [PATCH 17/92] xcp.net.biosname: remove dead code This function is private, not called, and if called would not do what the docstring says (does not pass "eth" as argument to the command called). Coverage change: xcp.net.biosdevname: 87.8% -> 94.6% total: 63.8% -> 63.9% Signed-off-by: Yann Dirson --- xcp/net/biosdevname.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/xcp/net/biosdevname.py b/xcp/net/biosdevname.py index e8166807..78995056 100644 --- a/xcp/net/biosdevname.py +++ b/xcp/net/biosdevname.py @@ -35,19 +35,6 @@ __ALL_POLICIES = [ "physical", "all_ethN" ] -def __run_single_device(eth, policy = "physical"): - """ - Run 'biosdevname -i eth' for a specified policy. - Return (stdout, stderr, returncode) tuple. - """ - - proc = Popen(["/sbin/biosdevname", "--policy", policy, - "-i"], stdout=PIPE, stderr=PIPE) - - stdout, stderr = proc.communicate() - - return ( stdout, stderr, proc.returncode ) - def __run_all_devices(policy = "physical"): """ Run 'biosdevname -d' for a specified policy. From 774641035d7bf6d57055e7d1d576df54d1b9adac Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:39:44 +0200 Subject: [PATCH 18/92] Un-futurize: replace future.utils.raise_ with six.raise_from Signed-off-by: Yann Dirson --- requirements-dev.txt | 2 +- setup.py | 2 +- xcp/repository.py | 25 ++++++++++++++----------- xcp/xmlunwrap.py | 7 ++++--- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ecda27a2..6bbf9447 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,4 @@ mock pytest pytest-cov # dependencies also in setup.py until they can be used -future +six diff --git a/setup.py b/setup.py index 0fdb3f1b..e7881222 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,6 @@ ], requires=[ 'branding', - 'future', + 'six', ], ) diff --git a/xcp/repository.py b/xcp/repository.py index 63b1277a..98b39683 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -23,12 +23,13 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from future.utils import raise_ import md5 import os.path import xml.dom.minidom import ConfigParser +import six + import xcp.version as version import xcp.xmlunwrap as xmlunwrap @@ -189,7 +190,8 @@ def _getVersion(cls, access, category): repo_ver = version.Version.from_string(ver_str) except Exception as e: - raise_(RepoFormatError, "Failed to open %s: %s" % (cls.TREEINFO_FILENAME, str(e))) + six.raise_from(RepoFormatError("Failed to open %s: %s" % + (cls.TREEINFO_FILENAME, str(e))), e) access.finish() return repo_ver @@ -232,7 +234,8 @@ def findRepositories(cls, access): package_list.append(line.strip()) extra.close() except Exception as e: - raise_(RepoFormatError, "Failed to open %s: %s" % (cls.REPOLIST_FILENAME, str(e))) + six.raise_from(RepoFormatError("Failed to open %s: %s" % + (cls.REPOLIST_FILENAME, str(e))), e) for loc in package_list: if cls.isRepo(access, loc): @@ -253,7 +256,7 @@ def __init__(self, access, base, is_group = False): repofile = access.openAddress(os.path.join(base, self.REPOSITORY_FILENAME)) except Exception as e: access.finish() - raise_(NoRepository, e) + six.raise_from(NoRepository(), e) self._parse_repofile(repofile) repofile.close() @@ -261,7 +264,7 @@ def __init__(self, access, base, is_group = False): pkgfile = access.openAddress(os.path.join(base, self.PKGDATA_FILENAME)) except Exception as e: access.finish() - raise_(NoRepository, e) + six.raise_from(NoRepository(), e) self._parse_packages(pkgfile) pkgfile.close() @@ -290,8 +293,8 @@ def _parse_repofile(self, repofile): # build xml doc object try: xmldoc = xml.dom.minidom.parseString(repofile_contents) - except: - raise_(RepoFormatError, "%s not in XML" % self.REPOSITORY_FILENAME) + except Exception as e: + six.raise_from(RepoFormatError("%s not in XML" % self.REPOSITORY_FILENAME), e) try: repo_node = xmlunwrap.getElementsByTagName(xmldoc, ['repository'], mandatory = True) @@ -314,8 +317,8 @@ def _parse_repofile(self, repofile): del req['build'] assert req['test'] in self.OPER_MAP self.requires.append(req) - except: - raise_(RepoFormatError, "%s format error" % self.REPOSITORY_FILENAME) + except Exception as e: + six.raise_from(RepoFormatError("%s format error" % self.REPOSITORY_FILENAME), e) self.identifier = "%s:%s" % (self.originator, self.name) ver_str = self.version @@ -333,8 +336,8 @@ def _parse_packages(self, pkgfile): # build xml doc object try: xmldoc = xml.dom.minidom.parseString(pkgfile_contents) - except: - raise_(RepoFormatError, "%s not in XML" % self.PKGDATA_FILENAME) + except Exception as e: + six.raise_from(RepoFormatError("%s not in XML" % self.PKGDATA_FILENAME), e) for pkg_node in xmlunwrap.getElementsByTagName(xmldoc, ['package']): pkg = self._create_package(pkg_node) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 59404211..1487afab 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -23,7 +23,8 @@ """xmlunwrap - general methods to unwrap XML elements & attributes""" -from future.utils import raise_ +import six + class XmlUnwrapError(Exception): pass @@ -69,8 +70,8 @@ def getIntAttribute(el, attrs, default = None): return default try: int_val = int(val, 0) - except: - raise_(XmlUnwrapError, "Invalid integer value for %s" % attrs[0]) + except Exception as e: + six.raise_from(XmlUnwrapError("Invalid integer value for %s" % attrs[0]), e) return int_val def getMapAttribute(el, attrs, mapping, default = None): From 935f9324572e7c02fb2e06b54d38d8c42e55ceb1 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 29 Jul 2022 15:06:34 +0200 Subject: [PATCH 19/92] Coverage: test_cpio: add a test for CpioFileCompat Coverage change: xcp.cpiofile: 58.1% -> 62.1% total: 63.9% -> 65.2% Signed-off-by: Yann Dirson --- tests/test_cpio.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 16a4a890..84763b26 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -4,7 +4,7 @@ import unittest import warnings -from xcp.cpiofile import CpioFile, CpioInfo +from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED try: from hashlib import md5 @@ -115,3 +115,43 @@ def test_xz(self): raise unittest.SkipTest("lzma package or xz tool not available") print 'Running test for XZ' self.doArchive('archive.cpio.xz', 'xz') + + # CpioFileCompat testing + + def archiveExtractCompat(self, fn, comp): + arc = CpioFileCompat(fn, mode="r", compression={"": CPIO_PLAIN, + "gz": CPIO_GZIPPED}[comp]) + found = False + for f in arc.namelist(): + info = arc.getinfo(f) + if info.isfile(): + data = arc.read(f) + self.assertEqual(len(data), info.size) + self.assertEqual(self.md5data, md5(data).hexdigest()) + found = True + arc.close() + self.assertTrue(found) + + def archiveCreateCompat(self, fn, comp): + if os.path.exists(fn): + os.unlink(fn) + arc = CpioFileCompat(fn, mode="w", compression={"": CPIO_PLAIN, + "gz": CPIO_GZIPPED}[comp]) + arc.write('archive/data') + arc.close() + self.archiveExtract(fn) + + def doArchiveCompat(self, fn, fmt): + self.archiveExtractCompat(fn, fmt) + + fn2 = "archive2" + fn[len("archive"):] + self.archiveCreateCompat(fn2, fmt) + self.archiveExtractCompat(fn2, fmt) + + def test_compat_plain(self): + self.doArchiveCompat('archive.cpio', '') + + def test_compat_gz(self): + # FIXME: this test exhibits "unclosed file" warnings when run + # under `-Wd` + self.doArchiveCompat('archive.cpio.gz', 'gz') From ce4863442c4b673f1bfa09f27389bc3d16ba8809 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 12 Jul 2022 11:51:15 +0200 Subject: [PATCH 20/92] Futurize fixes: remove now-unused imports Signed-off-by: Yann Dirson --- xcp/accessor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index ecabb6c5..7763cf96 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -28,7 +28,6 @@ import ftplib import os import tempfile -import types import urllib import urllib2 import urlparse From 8244c49d424fa897895f72ba6f662080a70d9372 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 9 Aug 2022 11:59:52 +0200 Subject: [PATCH 21/92] Coverage: add a test for Bootloader.newDefault Coverage change: xcp.bootloader: 72.8% -> 81.9% total: 65.4% -> 66.6% Signed-off-by: Yann Dirson --- tests/data/grub-linux.cfg | 5 +++++ tests/test_bootloader.py | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 tests/data/grub-linux.cfg diff --git a/tests/data/grub-linux.cfg b/tests/data/grub-linux.cfg new file mode 100644 index 00000000..e98e5d19 --- /dev/null +++ b/tests/data/grub-linux.cfg @@ -0,0 +1,5 @@ +set default=0 +menuentry 'linux' { + linux /boot/vmlinuz-1 ro + initrd /boot/initrd.img-1 +} diff --git a/tests/test_bootloader.py b/tests/test_bootloader.py index ca48204e..2e5700b5 100644 --- a/tests/test_bootloader.py +++ b/tests/test_bootloader.py @@ -1,7 +1,8 @@ import unittest import os +import shutil import subprocess -from tempfile import NamedTemporaryFile +from tempfile import NamedTemporaryFile, mkdtemp from xcp.bootloader import Bootloader @@ -21,6 +22,25 @@ def test_grub2(self): proc.stdout.close() proc.wait() +class TestLinuxBootloader(unittest.TestCase): + def setUp(self): + self.tmpdir = mkdtemp(prefix="testbl") + bootdir = os.path.join(self.tmpdir, "boot") + grubdir = os.path.join(bootdir, "grub") + os.makedirs(grubdir) + shutil.copyfile("tests/data/grub-linux.cfg", os.path.join(grubdir, "grub.cfg")) + with open(os.path.join(bootdir, "vmlinuz-1"), "w"): + pass + with open(os.path.join(bootdir, "vmlinuz-2"), "w"): + pass + with open(os.path.join(bootdir, "initrd.img-1"), "w"): + pass + with open(os.path.join(bootdir, "initrd.img-2"), "w"): + pass + def tearDown(self): + shutil.rmtree(self.tmpdir) + def test_grub2_newdefault(self): + Bootloader.newDefault("/boot/vmlinuz-2", "/boot/initrd.img-2", root=self.tmpdir) class TestBootloaderAdHoc(unittest.TestCase): def setUp(self): From 869fc72300beb1560c64e56eda661de19545c3fd Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:02:04 +0200 Subject: [PATCH 22/92] Futurize pylint complements: "print" continuation indentations, line lengths Signed-off-by: Yann Dirson --- xcp/bootloader.py | 22 +++++++++++----------- xcp/cpiofile.py | 3 +-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 441d3804..0602b822 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -468,11 +468,11 @@ def writeExtLinux(self, dst_file = None): if self.serial: if self.serial.get('flow', None) is None: print("serial %s %s" % (self.serial['port'], - self.serial['baud']), file=fh) + self.serial['baud']), file=fh) else: print("serial %s %s %s" % (self.serial['port'], - self.serial['baud'], - self.serial['flow']), file=fh) + self.serial['baud'], + self.serial['flow']), file=fh) if self.default: print("default " + self.default, file=fh) print("prompt 1", file=fh) @@ -486,13 +486,13 @@ def writeExtLinux(self, dst_file = None): print(" # " + m.title, file=fh) if m.tboot: print(" kernel mboot.c32", file=fh) - print(" append %s %s --- %s %s --- %s %s --- %s" % \ - (m.tboot, m.tboot_args, m.hypervisor, m.hypervisor_args, - m.kernel, m.kernel_args, m.initrd), file=fh) + print(" append %s %s --- %s %s --- %s %s --- %s" % + (m.tboot, m.tboot_args, m.hypervisor, m.hypervisor_args, + m.kernel, m.kernel_args, m.initrd), file=fh) elif m.hypervisor: print(" kernel mboot.c32", file=fh) - print(" append %s %s --- %s %s --- %s" % \ - (m.hypervisor, m.hypervisor_args, m.kernel, m.kernel_args, m.initrd), file=fh) + print(" append %s %s --- %s %s --- %s" % + (m.hypervisor, m.hypervisor_args, m.kernel, m.kernel_args, m.initrd), file=fh) else: print(" kernel " + m.kernel, file=fh) print(" append " + m.kernel_args, file=fh) @@ -508,8 +508,8 @@ def writeGrub(self, dst_file = None): print("# location " + self.location, file=fh) if self.serial: - print("serial --unit=%s --speed=%s" % (self.serial['port'], - self.serial['baud']), file=fh) + print("serial --unit=%s --speed=%s" % + (self.serial['port'], self.serial['baud']), file=fh) print("terminal --timeout=10 console serial", file=fh) else: print("terminal console", file=fh) @@ -542,7 +542,7 @@ def writeGrub2(self, dst_file = None): if self.serial: print("serial --unit=%s --speed=%s" % (self.serial['port'], - self.serial['baud']), file=fh) + self.serial['baud']), file=fh) print("terminal_input serial console", file=fh) print("terminal_output serial console", file=fh) if self.default: diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index a259efc3..79ffcd4e 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1306,8 +1306,7 @@ def list(self, verbose=True): print(filemode(cpioinfo.mode), end=' ') print("%d/%d" % (cpioinfo.uid, cpioinfo.gid), end=' ') if cpioinfo.ischr() or cpioinfo.isblk(): - print("%10s" % ("%d,%d" \ - % (cpioinfo.devmajor, cpioinfo.devminor)), end=' ') + print("%10s" % ("%d,%d" % (cpioinfo.devmajor, cpioinfo.devminor)), end=' ') else: print("%10d" % cpioinfo.size, end=' ') print("%d-%02d-%02d %02d:%02d:%02d" \ From 056238e032dadbe448d884525517e384d2305cf6 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 15:59:46 +0200 Subject: [PATCH 23/92] cpiofile: flag known issue in apparently-unused code Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index d6163849..a490aeff 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1158,6 +1158,7 @@ def xzopen(cls, name, mode="r", fileobj=None, compresslevel=6): if fileobj is not None: fileobj = _XZProxy(fileobj, mode) else: + # FIXME: not compatible with python3 API fileobj = lzma.LZMAFile(name, mode, options={'level': compresslevel, 'dict_size': 20 }) try: From 1e12b2f2350381de238e0ffcb8b92cbc57152d4f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 17:28:31 +0200 Subject: [PATCH 24/92] Migrate tests from StringIO.StringIO to io.StringIO This API change requires switching literals from str to unicode. Signed-off-by: Yann Dirson --- tests/test_ifrename_dynamic.py | 22 ++++----- tests/test_ifrename_logic.py | 12 ++--- tests/test_ifrename_static.py | 90 +++++++++++++++++----------------- 3 files changed, 59 insertions(+), 65 deletions(-) diff --git a/tests/test_ifrename_dynamic.py b/tests/test_ifrename_dynamic.py index 1eb28a74..6313b861 100644 --- a/tests/test_ifrename_dynamic.py +++ b/tests/test_ifrename_dynamic.py @@ -1,12 +1,10 @@ +from __future__ import unicode_literals import json import logging import unittest from copy import deepcopy -try: - import cStringIO as StringIO -except ImportError: - import StringIO +from io import StringIO from xcp.net.ifrename.dynamic import DynamicRules from xcp.net.ifrename.macpci import MACPCI @@ -16,7 +14,7 @@ class TestLoadAndParse(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): @@ -26,7 +24,7 @@ def tearDown(self): def test_null(self): - fd = StringIO.StringIO("") + fd = StringIO("") dr = DynamicRules(fd=fd) self.assertTrue(dr.load_and_parse()) @@ -36,7 +34,7 @@ def test_null(self): def test_empty(self): - fd = StringIO.StringIO( + fd = StringIO( '{"lastboot":[],"old":[]}' ) dr = DynamicRules(fd=fd) @@ -48,7 +46,7 @@ def test_empty(self): def test_one_invalid(self): - fd = StringIO.StringIO( + fd = StringIO( '{"lastboot":[["","",""]],"old":[]}' ) dr = DynamicRules(fd=fd) @@ -60,7 +58,7 @@ def test_one_invalid(self): def test_one_valid_lastboot(self): - fd = StringIO.StringIO( + fd = StringIO( '{"lastboot":[["01:23:45:67:89:0a","00:10.2","eth2"]],"old":[]}' ) dr = DynamicRules(fd=fd) @@ -74,7 +72,7 @@ def test_one_valid_lastboot(self): def test_one_valid_lastboot2(self): - fd = StringIO.StringIO( + fd = StringIO( '{"lastboot":[],"old":[["01:23:45:67:89:0a","00:10.2","eth2"]]}' ) dr = DynamicRules(fd=fd) @@ -88,7 +86,7 @@ def test_one_valid_lastboot2(self): class TestGenerate(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): @@ -149,7 +147,7 @@ def test_pci_missing(self): class TestSave(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index c74522eb..a1850c20 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -1,13 +1,11 @@ from __future__ import print_function +from __future__ import unicode_literals import logging import sys import unittest from copy import deepcopy -try: - import cStringIO as StringIO -except ImportError: - import StringIO +from io import StringIO from xcp.net.ifrename.logic import * from xcp.logger import LOG, openLog, closeLogs @@ -35,7 +33,7 @@ def apply_transactions(lst, trans): class TestSimpleLogic(unittest.TestCase): def setUp(self): - self.siobuff = StringIO.StringIO() + self.siobuff = StringIO() openLog(self.siobuff, logging.NOTSET) def tearDown(self): @@ -260,7 +258,7 @@ def test_1drule_1eth_already_complete(self): class TestUseCases(unittest.TestCase): def setUp(self): - self.siobuff = StringIO.StringIO() + self.siobuff = StringIO() openLog(self.siobuff, logging.NOTSET) def tearDown(self): @@ -534,7 +532,7 @@ def setUp(self): set to None and a tname set to the 'eth' """ - self.siobuff = StringIO.StringIO() + self.siobuff = StringIO() openLog(self.siobuff) diff --git a/tests/test_ifrename_static.py b/tests/test_ifrename_static.py index 16909142..9b11e380 100644 --- a/tests/test_ifrename_static.py +++ b/tests/test_ifrename_static.py @@ -1,11 +1,9 @@ +from __future__ import unicode_literals import logging import unittest from copy import deepcopy -try: - import cStringIO as StringIO -except ImportError: - import StringIO +from io import StringIO from xcp.net.ifrename.static import StaticRules from xcp.net.ifrename.macpci import MACPCI @@ -15,7 +13,7 @@ class TestLoadAndParse(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): @@ -37,7 +35,7 @@ def test_null(self): def test_empty(self): - fd = StringIO.StringIO("") + fd = StringIO("") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -46,7 +44,7 @@ def test_empty(self): def test_comment(self): - fd = StringIO.StringIO("#comment") + fd = StringIO("#comment") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -55,7 +53,7 @@ def test_comment(self): def test_comment_and_empty(self): - fd = StringIO.StringIO("\n # Another Comment\n ") + fd = StringIO("\n # Another Comment\n ") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -64,7 +62,7 @@ def test_comment_and_empty(self): def test_single_incorrect_mac(self): - fd = StringIO.StringIO('eth0:mac="foo"') + fd = StringIO('eth0:mac="foo"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -73,7 +71,7 @@ def test_single_incorrect_mac(self): def test_single_mac(self): - fd = StringIO.StringIO('eth0:mac="AB:CD:EF:AB:CD:EF"') + fd = StringIO('eth0:mac="AB:CD:EF:AB:CD:EF"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -82,7 +80,7 @@ def test_single_mac(self): def test_single_invalid_pci(self): - fd = StringIO.StringIO('eth0:pci="bar"') + fd = StringIO('eth0:pci="bar"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -91,7 +89,7 @@ def test_single_invalid_pci(self): def test_single_pci(self): - fd = StringIO.StringIO('eth0:pci="0000:00:00.1"') + fd = StringIO('eth0:pci="0000:00:00.1"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -100,7 +98,7 @@ def test_single_pci(self): def test_single_pci_0index(self): - fd = StringIO.StringIO('eth0:pci="0000:00:00.1[0]"') + fd = StringIO('eth0:pci="0000:00:00.1[0]"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -109,7 +107,7 @@ def test_single_pci_0index(self): def test_single_invalid_ppn(self): - fd = StringIO.StringIO('eth0:ppn="baz"') + fd = StringIO('eth0:ppn="baz"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -118,7 +116,7 @@ def test_single_invalid_ppn(self): def test_single_ppn_embedded(self): - fd = StringIO.StringIO('eth0:ppn="em2"') + fd = StringIO('eth0:ppn="em2"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -127,7 +125,7 @@ def test_single_ppn_embedded(self): def test_single_ppn_slot(self): - fd = StringIO.StringIO('eth0:ppn="p2p3"') + fd = StringIO('eth0:ppn="p2p3"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -136,7 +134,7 @@ def test_single_ppn_slot(self): def test_single_oldsytle_ppn_slot(self): # CA-82901 - Accept old-style PPNs but translate them to new-style - fd = StringIO.StringIO('eth0:ppn="pci2p3"') + fd = StringIO('eth0:ppn="pci2p3"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -145,7 +143,7 @@ def test_single_oldsytle_ppn_slot(self): def test_single_label(self): - fd = StringIO.StringIO('eth0:label="somestring"') + fd = StringIO('eth0:label="somestring"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -155,7 +153,7 @@ def test_single_label(self): class TestLoadAndParseGuess(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): @@ -165,7 +163,7 @@ def tearDown(self): def test_single_explicit_label(self): - fd = StringIO.StringIO("eth0=\"foo\"") + fd = StringIO("eth0=\"foo\"") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -174,7 +172,7 @@ def test_single_explicit_label(self): def test_single_implicit_label(self): - fd = StringIO.StringIO("eth0=foo") + fd = StringIO("eth0=foo") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -183,7 +181,7 @@ def test_single_implicit_label(self): def test_single_mac(self): - fd = StringIO.StringIO("eth0=00:00:00:00:00:00") + fd = StringIO("eth0=00:00:00:00:00:00") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -192,7 +190,7 @@ def test_single_mac(self): def test_single_pci(self): - fd = StringIO.StringIO("eth0=0000:00:00.0") + fd = StringIO("eth0=0000:00:00.0") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -201,7 +199,7 @@ def test_single_pci(self): def test_single_pci_index(self): - fd = StringIO.StringIO("eth0=0000:00:00.0[1]") + fd = StringIO("eth0=0000:00:00.0[1]") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -210,7 +208,7 @@ def test_single_pci_index(self): def test_single_ppn_embedded(self): - fd = StringIO.StringIO("eth0=em4") + fd = StringIO("eth0=em4") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -219,7 +217,7 @@ def test_single_ppn_embedded(self): def test_single_ppn_slot(self): - fd = StringIO.StringIO("eth0=p1p2") + fd = StringIO("eth0=p1p2") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -230,7 +228,7 @@ def test_single_ppn_slot(self): class TestGenerate(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) self.state = [ @@ -251,7 +249,7 @@ def tearDown(self): def test_null(self): - fd = StringIO.StringIO('eth0:label="somestring"') + fd = StringIO('eth0:label="somestring"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) sr.generate([]) @@ -260,7 +258,7 @@ def test_null(self): def test_single_not_matching_state(self): - fd = StringIO.StringIO('eth0:label="somestring"') + fd = StringIO('eth0:label="somestring"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) sr.generate(self.state) @@ -269,7 +267,7 @@ def test_single_not_matching_state(self): def test_single_mac_matching(self): - fd = StringIO.StringIO('eth0:mac="01:23:45:67:89:0a"') + fd = StringIO('eth0:mac="01:23:45:67:89:0a"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -281,7 +279,7 @@ def test_single_mac_matching(self): def test_single_pci_matching(self): - fd = StringIO.StringIO('eth0:pci="0000:00:10.0"') + fd = StringIO('eth0:pci="0000:00:10.0"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -293,7 +291,7 @@ def test_single_pci_matching(self): def test_single_ppn_embedded_matching(self): - fd = StringIO.StringIO('eth0:ppn="em1"') + fd = StringIO('eth0:ppn="em1"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -305,7 +303,7 @@ def test_single_ppn_embedded_matching(self): def test_single_ppn_slot_matching(self): - fd = StringIO.StringIO('eth0:ppn="p2p2"') + fd = StringIO('eth0:ppn="p2p2"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -317,7 +315,7 @@ def test_single_ppn_slot_matching(self): def test_single_label_matching(self): - fd = StringIO.StringIO('eth0:label="Ethernet1"') + fd = StringIO('eth0:label="Ethernet1"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -330,8 +328,8 @@ def test_single_label_matching(self): def test_ppn_quirks(self): # Test case taken from example on CA-75599 - fd = StringIO.StringIO('eth0:ppn="em1"\n' - 'eth1:ppn="em2"') + fd = StringIO('eth0:ppn="em1"\n' + 'eth1:ppn="em2"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -350,7 +348,7 @@ def test_ppn_quirks(self): class TestMultiplePCI(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) self.state = [ MACPCI("c8:cb:b8:d3:0c:ca", "0000:03:00.0", kname="eth0", @@ -370,8 +368,8 @@ def tearDown(self): def test_pci_matching(self): - fd = StringIO.StringIO('eth0:pci="0000:04:00.0"\n' - 'eth1:pci="0000:04:00.0[1]"') + fd = StringIO('eth0:pci="0000:04:00.0"\n' + 'eth1:pci="0000:04:00.0[1]"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -384,8 +382,8 @@ def test_pci_matching(self): def test_pci_matching_invert(self): - fd = StringIO.StringIO('eth0:pci="0000:04:00.0[1]"\n' - 'eth1:pci="0000:04:00.0[0]"') + fd = StringIO('eth0:pci="0000:04:00.0[1]"\n' + 'eth1:pci="0000:04:00.0[0]"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -398,8 +396,8 @@ def test_pci_matching_invert(self): def test_pci_matching_mixed(self): - fd = StringIO.StringIO('eth0:ppn="em3"\n' - 'eth1:pci="0000:04:00.0[1]"') + fd = StringIO('eth0:ppn="em3"\n' + 'eth1:pci="0000:04:00.0[1]"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -412,8 +410,8 @@ def test_pci_matching_mixed(self): def test_pci_missing(self): - fd = StringIO.StringIO('eth0:pci="0000:03:00.0"\n' - 'eth4:pci="0000:05:00.0"') + fd = StringIO('eth0:pci="0000:03:00.0"\n' + 'eth4:pci="0000:05:00.0"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -427,7 +425,7 @@ def test_pci_missing(self): class TestSave(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): From 3cc052741a073dfe922059a408d9efdb74c1ae83 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 29 Jul 2022 11:59:33 +0200 Subject: [PATCH 25/92] cpiofile: drop CpioFileCompat.write() This implementation of CpioFileCompat.write cannot work with python 2.7, as adding new attributes to a ZipInfo triggers an exception. `tarfile.py` in Python 2.7 has a fix, but this method is clearly not used, so there's no reason to keep it, especially as its use of StringIO should likely switch to BytesIO and we would need to write tests for this. Also note that in python3 the TarFileCompat class is removed. We could probably just drop our CpioFileCompat completely, if we're sure noone uses it, but it likely only exists because there was a TarFileCompat. Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 79ffcd4e..fa9282cd 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1905,16 +1905,7 @@ def read(self, name): return self.cpiofile.extractfile(self.cpiofile.getmember(name)).read() def write(self, filename, arcname=None, compress_type=None): self.cpiofile.add(filename, arcname) - def writestr(self, zinfo, bts): - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - import calendar - zinfo.name = zinfo.filename - zinfo.size = zinfo.file_size - zinfo.mtime = calendar.timegm(zinfo.date_time) - self.cpiofile.addfile(zinfo, StringIO(bts)) + # deleted writestr method def close(self): self.cpiofile.close() #class CpioFileCompat From 02e39ae86f481a50c360e9a2596b2f913f1d4cf2 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 17:37:04 +0200 Subject: [PATCH 26/92] Switch integer division to py3-compatible "//" ... on locations reported by futurize Signed-off-by: Yann Dirson --- xcp/bootloader.py | 5 +++-- xcp/dom0.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 0602b822..7ae55e2c 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -24,6 +24,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function +from __future__ import division import os import os.path import re @@ -519,7 +520,7 @@ def writeGrub(self, dst_file = None): print("default %d" % i, file=fh) break if self.timeout: - print("timeout %d" % (self.timeout / 10), file=fh) + print("timeout %d" % (self.timeout // 10), file=fh) for label in self.menu_order: m = self.menu[label] @@ -553,7 +554,7 @@ def writeGrub2(self, dst_file = None): else: print("set default='%s'" % str(self.default), file=fh) if self.timeout: - print("set timeout=%d" % (self.timeout / 10), file=fh) + print("set timeout=%d" % (self.timeout // 10), file=fh) boilerplate = getattr(self, 'boilerplate', [])[:] boilerplate.reverse() diff --git a/xcp/dom0.py b/xcp/dom0.py index 66a23a9e..086a683b 100644 --- a/xcp/dom0.py +++ b/xcp/dom0.py @@ -24,6 +24,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import +from __future__ import division import re from . import version @@ -40,7 +41,7 @@ def default_memory_v2(host_mem_kib): # # Add a bit extra to account for this. # - gb = (host_mem_kib + 256 * 1024) / 1024 / 1024 + gb = (host_mem_kib + 256 * 1024) // 1024 // 1024 if gb < 24: return 752 * 1024 @@ -62,7 +63,7 @@ def default_memory_v3(host_mem_kib): # # Add a bit extra to account for this. # - mb = (host_mem_kib + 256 * 1024) / 1024 + mb = (host_mem_kib + 256 * 1024) // 1024 # Give dom0 1 GiB + 5% of host memory, rounded to 16 MiB, limited to 8 GiB return min(1024 + int(mb * 0.05) & ~0xF, 8192) * 1024 From 2cf6d4cf4672060455318983d2844a95c77d6203 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 10:43:08 +0200 Subject: [PATCH 27/92] xcp.accessor: upgrade urllib usage (futurize) This is the only usage we have of `future`, where we used `six` instead everywhere else. At least `future` has the advantage of being really easy to drop when we cull python2 support. Signed-off-by: Yann Dirson --- requirements-dev.txt | 1 + setup.py | 1 + xcp/accessor.py | 34 ++++++++++++++++++---------------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6bbf9447..b2799686 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,3 +7,4 @@ pytest pytest-cov # dependencies also in setup.py until they can be used six +future diff --git a/setup.py b/setup.py index e7881222..08c8200d 100644 --- a/setup.py +++ b/setup.py @@ -36,5 +36,6 @@ requires=[ 'branding', 'six', + 'future', ], ) diff --git a/xcp/accessor.py b/xcp/accessor.py index 7763cf96..f2cf6631 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -25,11 +25,13 @@ """accessor - provide common interface to access methods""" +from future import standard_library +standard_library.install_aliases() import ftplib import os import tempfile -import urllib -import urllib2 +import urllib.request, urllib.parse, urllib.error +import urllib.request, urllib.error, urllib.parse import urlparse import errno @@ -280,12 +282,12 @@ def start(self): username = self.url_parts.username password = self.url_parts.password if username: - username = urllib.unquote(username) + username = urllib.parse.unquote(username) if password: - password = urllib.unquote(password) + password = urllib.parse.unquote(password) self.ftp.login(username, password) - directory = urllib.unquote(self.url_parts.path[1:]) + directory = urllib.parse.unquote(self.url_parts.path[1:]) if directory != '': logger.debug("Changing to " + directory) self.ftp.cwd(directory) @@ -305,7 +307,7 @@ def access(self, path): try: logger.debug("Testing "+path) self._cleanup() - url = urllib.unquote(path) + url = urllib.parse.unquote(path) if self.ftp.size(url) is not None: return True @@ -330,7 +332,7 @@ def access(self, path): def openAddress(self, address): logger.debug("Opening "+address) self._cleanup() - url = urllib.unquote(address) + url = urllib.parse.unquote(address) self.ftp.voidcmd('TYPE I') s = self.ftp.transfercmd('RETR ' + url).makefile('rb') @@ -339,7 +341,7 @@ def openAddress(self, address): def writeFile(self, in_fh, out_name): self._cleanup() - fname = urllib.unquote(out_name) + fname = urllib.parse.unquote(out_name) logger.debug("Storing as " + fname) self.ftp.storbinary('STOR ' + fname, in_fh) @@ -356,23 +358,23 @@ def __init__(self, baseAddress, ro): if self.url_parts.username: username = self.url_parts.username if username is not None: - username = urllib.unquote(self.url_parts.username) + username = urllib.parse.unquote(self.url_parts.username) password = self.url_parts.password if password is not None: - password = urllib.unquote(self.url_parts.password) - self.passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + password = urllib.parse.unquote(self.url_parts.password) + self.passman = urllib.request.HTTPPasswordMgrWithDefaultRealm() self.passman.add_password(None, self.url_parts.hostname, username, password) - self.authhandler = urllib2.HTTPBasicAuthHandler(self.passman) - self.opener = urllib2.build_opener(self.authhandler) - urllib2.install_opener(self.opener) + self.authhandler = urllib.request.HTTPBasicAuthHandler(self.passman) + self.opener = urllib.request.build_opener(self.authhandler) + urllib.request.install_opener(self.opener) self.baseAddress = rebuild_url(self.url_parts) def openAddress(self, address): try: - urlFile = urllib2.urlopen(os.path.join(self.baseAddress, address)) - except urllib2.HTTPError as e: + urlFile = urllib.request.urlopen(os.path.join(self.baseAddress, address)) + except urllib.error.HTTPError as e: self.lastError = e.code return False return urlFile From e546723ac1e3c231039688e8b3ccfb07b328032d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:17:42 +0200 Subject: [PATCH 28/92] xcp.accessor: more urllib upgrading These changes come from a `futurize -2` or `2to3` run, but are for some reason not applied by the `urllib` fix itself. Signed-off-by: Yann Dirson --- xcp/accessor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index f2cf6631..be98e545 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -32,7 +32,7 @@ import tempfile import urllib.request, urllib.parse, urllib.error import urllib.request, urllib.error, urllib.parse -import urlparse +import urllib.parse import errno import xcp.mount as mount @@ -252,14 +252,14 @@ def rebuild_url(url_parts): host = url_parts.hostname if url_parts.port: host += ':' + str(url_parts.port) - return urlparse.urlunsplit( + return urllib.parse.urlunsplit( (url_parts.scheme, host, url_parts.path, '', '')) class FTPAccessor(Accessor): def __init__(self, baseAddress, ro): super(FTPAccessor, self).__init__(ro) - self.url_parts = urlparse.urlsplit(baseAddress, allow_fragments=False) + self.url_parts = urllib.parse.urlsplit(baseAddress, allow_fragments=False) self.start_count = 0 self.cleanup = False self.ftp = None @@ -312,7 +312,7 @@ def access(self, path): if self.ftp.size(url) is not None: return True lst = self.ftp.nlst(os.path.dirname(url)) - return os.path.basename(url) in map(os.path.basename, lst) + return os.path.basename(url) in list(map(os.path.basename, lst)) except IOError as e: if e.errno == errno.EIO: self.lastError = 5 @@ -353,7 +353,7 @@ class HTTPAccessor(Accessor): def __init__(self, baseAddress, ro): assert ro super(HTTPAccessor, self).__init__(ro) - self.url_parts = urlparse.urlsplit(baseAddress, allow_fragments=False) + self.url_parts = urllib.parse.urlsplit(baseAddress, allow_fragments=False) if self.url_parts.username: username = self.url_parts.username @@ -391,7 +391,7 @@ def __repr__(self): } def createAccessor(baseAddress, *args): - url_parts = urlparse.urlsplit(baseAddress, allow_fragments=False) + url_parts = urllib.parse.urlsplit(baseAddress, allow_fragments=False) assert url_parts.scheme in SUPPORTED_ACCESSORS.keys() return SUPPORTED_ACCESSORS[url_parts.scheme](baseAddress, *args) From 1769f36e3b166549dec178e66139e2c6032a33fa Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 13:50:02 +0200 Subject: [PATCH 29/92] Use new-style classes even in python2 (2to3) Signed-off-by: Yann Dirson --- xcp/cmd.py | 2 +- xcp/repository.py | 2 +- xcp/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/cmd.py b/xcp/cmd.py index ff039484..bbd94656 100644 --- a/xcp/cmd.py +++ b/xcp/cmd.py @@ -56,7 +56,7 @@ def runCmd(command, with_stdout = False, with_stderr = False, inputtext = None): return rv, err return rv -class OutputCache: +class OutputCache(object): def __init__(self): self.cache = {} diff --git a/xcp/repository.py b/xcp/repository.py index 98b39683..7b4c1599 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -33,7 +33,7 @@ import xcp.version as version import xcp.xmlunwrap as xmlunwrap -class Package: +class Package(object): pass class BzippedPackage(Package): diff --git a/xcp/version.py b/xcp/version.py index aaa7fcd6..2daf65bf 100644 --- a/xcp/version.py +++ b/xcp/version.py @@ -25,7 +25,7 @@ """version - version comparison methods""" -class Version: +class Version(object): def __init__(self, ver, build = None): self.ver = ver self.build = build From d6d104c91f71982beb0658699f8ca48e2ced0939 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 13:59:03 +0200 Subject: [PATCH 30/92] futurize: deal with xrange() Changes driven by "futurize -f libfuturize.fixes.fix_xrange_with_import", but not using "from builtins import range", which only brings negligible performance degradation on python2. Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 6 +++--- xcp/net/mac.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index fa9282cd..88d06045 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -116,7 +116,7 @@ def copyfileobj(src, dst, length=None): bufsize = 16 * 1024 blocks, remainder = divmod(length, bufsize) - for _ in xrange(blocks): + for b in range(blocks): buf = src.read(bufsize) if len(buf) < bufsize: raise IOError("end of file reached") @@ -398,7 +398,7 @@ def seek(self, pos=0): """ if pos - self.pos >= 0: blocks, remainder = divmod(pos - self.pos, self.bufsize) - for _ in xrange(blocks): + for i in range(blocks): self.read(self.bufsize) self.read(remainder) else: @@ -1794,7 +1794,7 @@ def _getmember(self, name, cpioinfo=None): else: end = members.index(cpioinfo) - for i in xrange(end - 1, -1, -1): + for i in range(end - 1, -1, -1): if name == members[i].name: return members[i] diff --git a/xcp/net/mac.py b/xcp/net/mac.py index 9b042c8f..b512a565 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -89,7 +89,7 @@ def _set_from_str_octets(self, octets): self.octets = [ int(i, 16) for i in octets ] self.integer = long(sum(t[0] << t[1] for t in - zip(self.octets, xrange(40, -1, -8)))) + zip(self.octets, range(40, -1, -8)))) def _set_from_str_quads(self, quads): """Private helper""" @@ -101,7 +101,7 @@ def _set_from_str_quads(self, quads): self.octets.extend([(quad >> 8) & 0xff, quad & 0xff]) self.integer = long(sum(t[0] << t[1] for t in - zip(self.octets, xrange(40, -1, -8)))) + zip(self.octets, range(40, -1, -8)))) def is_unicast(self): """is this a unicast address?""" From 1c15aa3fc6ffdb9b373f0d57722f3851d1f4030d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 14:06:14 +0200 Subject: [PATCH 31/92] python3: remove casts from "int" to "long" There has not been any distinction at the python code level between `long` and `int` since 2.2. `2to3 -f long` would replace them with casts to `int` instead, which would just be a no-op. Signed-off-by: Yann Dirson --- xcp/net/mac.py | 8 ++++---- xcp/repository.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/xcp/net/mac.py b/xcp/net/mac.py index b512a565..47e586c0 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -88,8 +88,8 @@ def _set_from_str_octets(self, octets): raise ValueError("Expected 6 octets, got %d" % len(octets)) self.octets = [ int(i, 16) for i in octets ] - self.integer = long(sum(t[0] << t[1] for t in - zip(self.octets, range(40, -1, -8)))) + self.integer = sum(t[0] << t[1] for t in + zip(self.octets, range(40, -1, -8))) def _set_from_str_quads(self, quads): """Private helper""" @@ -100,8 +100,8 @@ def _set_from_str_quads(self, quads): for quad in ( int(i, 16) for i in quads ): self.octets.extend([(quad >> 8) & 0xff, quad & 0xff]) - self.integer = long(sum(t[0] << t[1] for t in - zip(self.octets, range(40, -1, -8)))) + self.integer = sum(t[0] << t[1] for t in + zip(self.octets, range(40, -1, -8))) def is_unicast(self): """is this a unicast address?""" diff --git a/xcp/repository.py b/xcp/repository.py index 7b4c1599..4bb98f8b 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -46,7 +46,7 @@ def __init__(self, repository, label, size, md5sum, optional, fname, root): self.optional, self.filename, self.destination - ) = ( repository, label, long(size), md5sum, (optional==True), fname, root ) + ) = ( repository, label, size, md5sum, (optional==True), fname, root ) def __repr__(self): return "" % self.label @@ -61,7 +61,7 @@ def __init__(self, repository, label, size, md5sum, optional, fname, options): self.optional, self.filename, self.options - ) = ( repository, label, long(size), md5sum, (optional==True), fname, options ) + ) = ( repository, label, size, md5sum, (optional==True), fname, options ) def __repr__(self): return "" % self.label @@ -76,7 +76,7 @@ def __init__(self, repository, label, size, md5sum, fname, kernel, options): self.filename, self.kernel, self.options - ) = ( repository, label, long(size), md5sum, fname, kernel, options ) + ) = ( repository, label, size, md5sum, fname, kernel, options ) def __repr__(self): return "" % (self.label, self.kernel) @@ -90,7 +90,7 @@ def __init__(self, repository, label, size, md5sum, fname, root): self.md5sum, self.filename, self.destination - ) = ( repository, label, long(size), md5sum, fname, root ) + ) = ( repository, label, size, md5sum, fname, root ) def __repr__(self): return "" % self.label @@ -103,7 +103,7 @@ def __init__(self, repository, label, size, md5sum, fname): self.size, self.md5sum, self.filename - ) = ( repository, label, long(size), md5sum, fname ) + ) = ( repository, label, size, md5sum, fname ) def __repr__(self): return "" % self.label From d8cac489f3c674c81cbf31661791d7200c90259e Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 14:37:13 +0200 Subject: [PATCH 32/92] python3: change float cast from "long" to "int" Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 88d06045..d070d3f7 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -303,7 +303,7 @@ def _init_write_gz(self): -self.zlib.MAX_WBITS, self.zlib.DEF_MEM_LEVEL, 0) - timestamp = struct.pack(" Date: Mon, 18 Jul 2022 14:34:57 +0200 Subject: [PATCH 33/92] 2to3: rename "next" methods to "__next__", use six.Iterator for portability Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index d070d3f7..b53b7bd4 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -51,6 +51,8 @@ import struct import copy +import six + if sys.platform == 'mac': # This module needs work for MacOS9, especially in the area of pathname # handling. In many places it is assumed a simple substitution of / by the @@ -917,7 +919,7 @@ def isdev(self): return (stat.S_ISCHR(self.mode) or stat.S_ISBLK(self.mode)) # class CpioInfo -class CpioFile(object): +class CpioFile(six.Iterator): """The CpioFile Class provides an interface to cpio archives. """ @@ -1697,7 +1699,7 @@ def utime(self, cpioinfo, cpiogetpath): raise ExtractError("could not change modification time") #-------------------------------------------------------------------------- - def next(self): + def __next__(self): """Return the next member of the archive as a CpioInfo object, when CpioFile is opened for reading. Return None if there is no more available. @@ -1832,7 +1834,7 @@ def _dbg(self, level, msg): print(msg, file=sys.stderr) # class CpioFile -class CpioIter(object): +class CpioIter(six.Iterator): """Iterator Class. for cpioinfo in CpioFile(...): @@ -1848,7 +1850,7 @@ def __iter__(self): """Return iterator object. """ return self - def next(self): + def __next__(self): """Return the next item using CpioFile's next() method. When all members have been read, set CpioFile as _loaded. """ From 5a4947e6eee51a9e2dd6f861c10a9ed8b42606af Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 15:46:55 +0200 Subject: [PATCH 34/92] futurize: convert "iteritems()" to "items()", all negligible perf impact Signed-off-by: Yann Dirson --- xcp/net/ifrename/dynamic.py | 2 +- xcp/net/ifrename/static.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index befab65a..aa9ff077 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -170,7 +170,7 @@ def generate(self, state): LOG.warning("Discovered physical policy naming quirks in provided " "state. Disabling 'method=ppn' generation") - for target, (method, value) in self.formulae.iteritems(): + for target, (method, value) in self.formulae.items(): if method == "mac": diff --git a/xcp/net/ifrename/static.py b/xcp/net/ifrename/static.py index 615c2073..be1e695d 100644 --- a/xcp/net/ifrename/static.py +++ b/xcp/net/ifrename/static.py @@ -175,7 +175,7 @@ def load_and_parse(self): # If we need to guess the method from the value if method == "guess": - for k, v in StaticRules.validators.iteritems(): + for k, v in StaticRules.validators.items(): if v.match(value) is not None: method = k break @@ -222,7 +222,7 @@ def generate(self, state): LOG.warning("Discovered physical policy naming quirks in provided " "state. Disabling 'method=ppn' generation") - for target, (method, value) in self.formulae.iteritems(): + for target, (method, value) in self.formulae.items(): if method == "mac": From fff9b3166da97cda4d5c43cfdff3b02ee30c436f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:29:51 +0200 Subject: [PATCH 35/92] Apply fixes from `2to3 -f map` that are useful even when suboptimal Only exclude a change to `xcp.accessor`. Signed-off-by: Yann Dirson --- xcp/bootloader.py | 4 ++-- xcp/cpiofile.py | 2 +- xcp/net/ifrename/logic.py | 24 +++++++++++------------- xcp/net/ifrename/util.py | 2 +- xcp/repository.py | 6 ++---- xcp/version.py | 2 +- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 7ae55e2c..819a417a 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -152,7 +152,7 @@ def readExtLinux(cls, src_file): # els[2] contains tboot args, hypervisor, # hypervisor args, kernel, # kernel args & initrd - args = map(lambda x: x.strip(), els[2].split('---')) + args = [x.strip() for x in els[2].split('---')] if len(args) == 4: hypervisor = args[1].split(None, 1) kernel = args[2].split(None, 1) @@ -168,7 +168,7 @@ def readExtLinux(cls, src_file): elif 'xen' in els[1]: # els[2] contains hypervisor args, kernel, # kernel args & initrd - args = map(lambda x: x.strip(), els[2].split('---')) + args = [x.strip() for x in els[2].split('---')] if len(args) == 3: kernel = args[1].split(None, 1) if len(kernel) == 2: diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index b53b7bd4..cccb4878 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1893,7 +1893,7 @@ def __init__(self, fpath, mode="r", compression=CPIO_PLAIN): m.file_size = m.size m.date_time = time.gmtime(m.mtime)[:6] def namelist(self): - return map(lambda m: m.name, self.infolist()) + return [m.name for m in self.infolist()] def infolist(self): return filter(lambda m: m.isreg(), self.cpiofile.getmembers()) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index 2328ca54..b4a52267 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -83,7 +83,7 @@ def __rename_nic(nic, name, transactions, cur_state): # Assert that name is valid assert VALID_ETH_NAME.match(name) is not None # Assert that name is not already taken in the current state - assert name not in map(lambda x: x.tname, cur_state) + assert name not in [x.tname for x in cur_state] # Given the previous assert, only un-renamed nics in the current state can # possibly alias the new name @@ -327,7 +327,7 @@ def rename_logic( static_rules, for fn in multinic_functions: newnics = util.get_nics_with_pci(filter(util.needs_renaming, cur_state), fn) - orders = sorted(map(lambda x: x.order, newnics)) + orders = sorted([x.order for x in newnics]) newnics.sort(key = lambda n: n.mac.integer) for nic, neworder in zip(newnics, orders): LOG.debug("NIC '%s' getting new order '%s'" % (nic, neworder)) @@ -336,10 +336,8 @@ def rename_logic( static_rules, # For completely new network cards which we have never seen before, work out # a safe new number to assign it ethnumbers = sorted( - map(lambda x: int(x[3:]), - filter(lambda x: VALID_ETH_NAME.match(x) is not None, - map(lambda x: x.tname or x.kname, - static_rules + cur_state + last_state)))) + [int(x[3:]) for x in filter(lambda x: VALID_ETH_NAME.match(x) is not None, + [x.tname or x.kname for x in static_rules + cur_state + last_state])]) if len(ethnumbers): nextethnum = ethnumbers[-1]+1 else: @@ -353,7 +351,7 @@ def rename_logic( static_rules, LOG.info("Renaming brand new nic '%s'" % (nic,)) if ( VALID_ETH_NAME.match(nic.kname) is not None and - nic.kname not in map(lambda x: x.tname, cur_state) ): + nic.kname not in [x.tname for x in cur_state] ): # User has been messing around with state files but not the udev # rules. If the eth name is still free, give it @@ -411,13 +409,13 @@ def rename( static_rules, "'eth'" % (e, )) # Verify no two static rules refer to the same eth name - _ = frozenset( map(lambda x: x.tname, static_rules) ) + _ = frozenset( [x.tname for x in static_rules] ) if len(_) != len(static_rules): raise StaticRuleError("Some static rules alias the same " "eth name") # Verify no two static rules refer to the same mac address - _ = frozenset( map(lambda x: x.mac, static_rules) ) + _ = frozenset( [x.mac for x in static_rules] ) if len(_) != len(static_rules): raise StaticRuleError("Some static rules alias the same MAC " "address") @@ -445,13 +443,13 @@ def rename( static_rules, # Verify no two entries of current state refer to the same eth name - _ = frozenset( map(lambda x: x.kname, cur_state) ) + _ = frozenset( [x.kname for x in cur_state] ) if len(_) != len(cur_state): raise CurrentStateError("Some entries of current state alias the " "same eth name") # Verify no two entries of current state refer to the same mac address - _ = frozenset( map(lambda x: x.mac, cur_state) ) + _ = frozenset( [x.mac for x in cur_state] ) if len(_) != len(cur_state): raise CurrentStateError("Some entries of current state alias the " "same MAC address") @@ -474,13 +472,13 @@ def rename( static_rules, # Verify no two entries of last state refer to the same eth name - _ = frozenset( map(lambda x: x.tname, last_state) ) + _ = frozenset( [x.tname for x in last_state] ) if len(_) != len(last_state): raise LastStateError("Some entries of last state alias the " "same eth name") # Verify no two entries of last state refer to the same mac address - _ = frozenset( map(lambda x: x.mac, last_state) ) + _ = frozenset( [x.mac for x in last_state] ) if len(_) != len(last_state): raise LastStateError("Some entries of last state alias the " "same MAC address") diff --git a/xcp/net/ifrename/util.py b/xcp/net/ifrename/util.py index 72c89fa3..3a7326f5 100644 --- a/xcp/net/ifrename/util.py +++ b/xcp/net/ifrename/util.py @@ -50,7 +50,7 @@ def get_nic_with_kname(nics, kname): def tname_free(nics, name): """Check that name is not taken by any nics""" - return name not in map(lambda x: x.tname, nics) + return name not in [x.tname for x in nics] def get_nic_with_mac(nics, mac): """Search for nic with mac""" diff --git a/xcp/repository.py b/xcp/repository.py index 4bb98f8b..32399a88 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -170,8 +170,7 @@ def __init__(self, access, base = ""): def isRepo(cls, access, base): """ Return whether there is a repository at base address 'base' accessible using accessor.""" - return False not in map(lambda x: access.access(os.path.join(base, x)), - [cls.TREEINFO_FILENAME, cls.REPOMD_FILENAME]) + return False not in [access.access(os.path.join(base, x)) for x in [cls.TREEINFO_FILENAME, cls.REPOMD_FILENAME]] @classmethod def _getVersion(cls, access, category): @@ -368,8 +367,7 @@ def _create_package(self, node): def isRepo(cls, access, base): """ Return whether there is a repository at base address 'base' accessible using accessor.""" - return False not in map(lambda x: access.access(os.path.join(base, x)), - [cls.REPOSITORY_FILENAME, cls.PKGDATA_FILENAME]) + return False not in [access.access(os.path.join(base, x)) for x in [cls.REPOSITORY_FILENAME, cls.PKGDATA_FILENAME]] @classmethod def getRepoVer(cls, access): diff --git a/xcp/version.py b/xcp/version.py index 2daf65bf..37f9e30a 100644 --- a/xcp/version.py +++ b/xcp/version.py @@ -52,7 +52,7 @@ def from_string(cls, ver_str): if '-' in ver_str: ver_str, build = ver_str.split('-', 1) - ver = map(cls.intify, ver_str.split('.')) + ver = list(map(cls.intify, ver_str.split('.'))) return cls(ver, build) From acc51cdad01637bdae323faff3493f0fae75d86a Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 17:21:50 +0200 Subject: [PATCH 36/92] test_cpio: always use hashlib, came with python 2.5 Signed-off-by: Yann Dirson --- tests/test_cpio.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 628c54ca..57c15188 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -1,4 +1,5 @@ from __future__ import print_function +from hashlib import md5 import os import shutil import subprocess @@ -7,11 +8,6 @@ from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED -try: - from hashlib import md5 -except: - from md5 import md5 - def writeRandomFile(fn, size, start='', add='a'): f = open(fn, 'wb') m = md5() From 07675d3e24313033925c83db26c0f4b901fc2ede Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 12:13:01 +0200 Subject: [PATCH 37/92] Apply all fixes from `2to3 -f filter` that are useful even when suboptimal Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 3 +-- xcp/net/ifrename/dynamic.py | 4 ++-- xcp/net/ifrename/logic.py | 10 ++++------ xcp/pci.py | 12 +++++------- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index cccb4878..d03a2ab7 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1895,8 +1895,7 @@ def __init__(self, fpath, mode="r", compression=CPIO_PLAIN): def namelist(self): return [m.name for m in self.infolist()] def infolist(self): - return filter(lambda m: m.isreg(), - self.cpiofile.getmembers()) + return [m for m in self.cpiofile.getmembers() if m.isreg()] def printdir(self): self.cpiofile.list() def testzip(self): diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index aa9ff077..0528baa7 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -262,8 +262,8 @@ def validate(entry): % (entry, e)) return False - lastboot = filter(validate, self.lastboot) - old = filter(validate, self.old) + lastboot = list(filter(validate, self.lastboot)) + old = list(filter(validate, self.old)) try: res += json.dumps({"lastboot": lastboot, "old": old}, diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index b4a52267..21cd628d 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -88,7 +88,7 @@ def __rename_nic(nic, name, transactions, cur_state): # Given the previous assert, only un-renamed nics in the current state can # possibly alias the new name aliased = util.get_nic_with_kname( - filter(lambda x: x.tname is None, cur_state), name) + [x for x in cur_state if x.tname is None], name) if aliased is None: # Using this rule will not alias another currently present NIC @@ -325,7 +325,7 @@ def rename_logic( static_rules, if len(multinic_functions): LOG.debug("New multi-nic logic - attempting to re-order") for fn in multinic_functions: - newnics = util.get_nics_with_pci(filter(util.needs_renaming, cur_state), + newnics = util.get_nics_with_pci(list(filter(util.needs_renaming, cur_state)), fn) orders = sorted([x.order for x in newnics]) newnics.sort(key = lambda n: n.mac.integer) @@ -336,8 +336,7 @@ def rename_logic( static_rules, # For completely new network cards which we have never seen before, work out # a safe new number to assign it ethnumbers = sorted( - [int(x[3:]) for x in filter(lambda x: VALID_ETH_NAME.match(x) is not None, - [x.tname or x.kname for x in static_rules + cur_state + last_state])]) + [int(x[3:]) for x in [x for x in [x.tname or x.kname for x in static_rules + cur_state + last_state] if VALID_ETH_NAME.match(x) is not None]]) if len(ethnumbers): nextethnum = ethnumbers[-1]+1 else: @@ -422,8 +421,7 @@ def rename( static_rules, if len(cur_state): # Filter out iBFT NICs - cur_state = filter(lambda x: VALID_IBFT_NAME.match(x.kname) is None, - cur_state) + cur_state = [x for x in cur_state if VALID_IBFT_NAME.match(x.kname) is None] # Verify types and properties of the list for e in cur_state: diff --git a/xcp/pci.py b/xcp/pci.py index 367e7dc5..85f82ec7 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -260,8 +260,7 @@ def __init__(self): stdout = subprocess.PIPE) for l in cmd.stdout: line = l.rstrip() - el = filter(lambda x: not x.startswith('-'), - line.replace('"','').split()) + el = [x for x in line.replace('"','').split() if not x.startswith('-')] self.devs[el[0]] = {'id': el[0], 'class': el[1][:2], 'subclass': el[1][2:], @@ -279,11 +278,11 @@ def findByClass(self, cls, subcls = None): [class1, class2, ... classN]""" if subcls: assert isinstance(cls, str) - return filter(lambda x: x['class'] == cls and - x['subclass'] == subcls, self.devs.values()) + return [x for x in self.devs.values() if x['class'] == cls and + x['subclass'] == subcls] else: assert isinstance(cls, list) - return filter(lambda x: x['class'] in cls, self.devs.values()) + return [x for x in self.devs.values() if x['class'] in cls] def findRelatedFunctions(self, dev): """ return other devices that share the same bus & slot""" @@ -291,8 +290,7 @@ def slot(dev): left, _ = dev.rsplit('.', 1) return left - return filter(lambda x: x != dev and slot(x) == slot(dev), - self.devs.keys()) + return [x for x in self.devs.keys() if x != dev and slot(x) == slot(dev)] def pci_sbdfi_to_nic(sbdfi, nics): From aee19c85eacb68b5d48b9b95cd1cb363b013ac47 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 16:45:37 +0200 Subject: [PATCH 38/92] test_cpio: use context manager to avoid forgetting to close some files Signed-off-by: Yann Dirson --- tests/test_cpio.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 57c15188..b43e1764 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -37,7 +37,8 @@ def setUp(self): shutil.rmtree('archive', True) os.mkdir('archive') writeRandomFile('archive/data', 10491) - self.md5data = md5(open('archive/data').read()).hexdigest() + with open('archive/data') as fd: + self.md5data = md5(fd.read()).hexdigest() check_call("find archive | cpio -o -H newc > archive.cpio") check_call("gzip -c < archive.cpio > archive.cpio.gz") check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") @@ -78,7 +79,8 @@ def archiveCreate(self, fn, fmt='w'): os.unlink(fn) arc = CpioFile.open(fn, fmt) f = arc.getcpioinfo('archive/data') - arc.addfile(f, open('archive/data')) + with open('archive/data') as fd: + arc.addfile(f, fd) # test recursively add "." os.chdir('archive') arc.add(".") From de9b4bfdbfa53f29086ce9f57c0e4b9ecf2aae0d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:45:52 +0200 Subject: [PATCH 39/92] Futurize fixes: use generator comprehensions when list ones are not needed 2to3 uses list comprehensions to replace `map` calls in many places where they're not needed, and would yield worse perf than `map`. Signed-off-by: Yann Dirson --- xcp/net/ifrename/logic.py | 28 +++++++++++++++------------- xcp/net/ifrename/util.py | 2 +- xcp/repository.py | 6 ++++-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index 21cd628d..19b5b5b7 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -83,12 +83,12 @@ def __rename_nic(nic, name, transactions, cur_state): # Assert that name is valid assert VALID_ETH_NAME.match(name) is not None # Assert that name is not already taken in the current state - assert name not in [x.tname for x in cur_state] + assert name not in (x.tname for x in cur_state) # Given the previous assert, only un-renamed nics in the current state can # possibly alias the new name aliased = util.get_nic_with_kname( - [x for x in cur_state if x.tname is None], name) + (x for x in cur_state if x.tname is None), name) if aliased is None: # Using this rule will not alias another currently present NIC @@ -325,9 +325,8 @@ def rename_logic( static_rules, if len(multinic_functions): LOG.debug("New multi-nic logic - attempting to re-order") for fn in multinic_functions: - newnics = util.get_nics_with_pci(list(filter(util.needs_renaming, cur_state)), - fn) - orders = sorted([x.order for x in newnics]) + newnics = util.get_nics_with_pci((x for x in cur_state if util.needs_renaming(x)), fn) + orders = sorted(x.order for x in newnics) newnics.sort(key = lambda n: n.mac.integer) for nic, neworder in zip(newnics, orders): LOG.debug("NIC '%s' getting new order '%s'" % (nic, neworder)) @@ -336,7 +335,10 @@ def rename_logic( static_rules, # For completely new network cards which we have never seen before, work out # a safe new number to assign it ethnumbers = sorted( - [int(x[3:]) for x in [x for x in [x.tname or x.kname for x in static_rules + cur_state + last_state] if VALID_ETH_NAME.match(x) is not None]]) + int(x[3:]) + for x in (x for x in (x.tname or x.kname + for x in static_rules + cur_state + last_state) + if VALID_ETH_NAME.match(x) is not None)) if len(ethnumbers): nextethnum = ethnumbers[-1]+1 else: @@ -350,7 +352,7 @@ def rename_logic( static_rules, LOG.info("Renaming brand new nic '%s'" % (nic,)) if ( VALID_ETH_NAME.match(nic.kname) is not None and - nic.kname not in [x.tname for x in cur_state] ): + nic.kname not in (x.tname for x in cur_state) ): # User has been messing around with state files but not the udev # rules. If the eth name is still free, give it @@ -408,13 +410,13 @@ def rename( static_rules, "'eth'" % (e, )) # Verify no two static rules refer to the same eth name - _ = frozenset( [x.tname for x in static_rules] ) + _ = frozenset(x.tname for x in static_rules) if len(_) != len(static_rules): raise StaticRuleError("Some static rules alias the same " "eth name") # Verify no two static rules refer to the same mac address - _ = frozenset( [x.mac for x in static_rules] ) + _ = frozenset(x.mac for x in static_rules) if len(_) != len(static_rules): raise StaticRuleError("Some static rules alias the same MAC " "address") @@ -441,13 +443,13 @@ def rename( static_rules, # Verify no two entries of current state refer to the same eth name - _ = frozenset( [x.kname for x in cur_state] ) + _ = frozenset(x.kname for x in cur_state) if len(_) != len(cur_state): raise CurrentStateError("Some entries of current state alias the " "same eth name") # Verify no two entries of current state refer to the same mac address - _ = frozenset( [x.mac for x in cur_state] ) + _ = frozenset(x.mac for x in cur_state) if len(_) != len(cur_state): raise CurrentStateError("Some entries of current state alias the " "same MAC address") @@ -470,13 +472,13 @@ def rename( static_rules, # Verify no two entries of last state refer to the same eth name - _ = frozenset( [x.tname for x in last_state] ) + _ = frozenset(x.tname for x in last_state) if len(_) != len(last_state): raise LastStateError("Some entries of last state alias the " "same eth name") # Verify no two entries of last state refer to the same mac address - _ = frozenset( [x.mac for x in last_state] ) + _ = frozenset(x.mac for x in last_state) if len(_) != len(last_state): raise LastStateError("Some entries of last state alias the " "same MAC address") diff --git a/xcp/net/ifrename/util.py b/xcp/net/ifrename/util.py index 3a7326f5..0aad71f1 100644 --- a/xcp/net/ifrename/util.py +++ b/xcp/net/ifrename/util.py @@ -50,7 +50,7 @@ def get_nic_with_kname(nics, kname): def tname_free(nics, name): """Check that name is not taken by any nics""" - return name not in [x.tname for x in nics] + return name not in (x.tname for x in nics) def get_nic_with_mac(nics, mac): """Search for nic with mac""" diff --git a/xcp/repository.py b/xcp/repository.py index 32399a88..e83e14f2 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -170,7 +170,8 @@ def __init__(self, access, base = ""): def isRepo(cls, access, base): """ Return whether there is a repository at base address 'base' accessible using accessor.""" - return False not in [access.access(os.path.join(base, x)) for x in [cls.TREEINFO_FILENAME, cls.REPOMD_FILENAME]] + return False not in (access.access(os.path.join(base, x)) + for x in [cls.TREEINFO_FILENAME, cls.REPOMD_FILENAME]) @classmethod def _getVersion(cls, access, category): @@ -367,7 +368,8 @@ def _create_package(self, node): def isRepo(cls, access, base): """ Return whether there is a repository at base address 'base' accessible using accessor.""" - return False not in [access.access(os.path.join(base, x)) for x in [cls.REPOSITORY_FILENAME, cls.PKGDATA_FILENAME]] + return False not in (access.access(os.path.join(base, x)) + for x in [cls.REPOSITORY_FILENAME, cls.PKGDATA_FILENAME]) @classmethod def getRepoVer(cls, access): From e7a3c3f69e75ce0365424ad7a92b6b194ed7e86b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 16:46:15 +0200 Subject: [PATCH 40/92] test_cpio: use context manager for other "open" calls for consistency Signed-off-by: Yann Dirson --- tests/test_cpio.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index b43e1764..5af86b92 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -9,18 +9,17 @@ from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED def writeRandomFile(fn, size, start='', add='a'): - f = open(fn, 'wb') - m = md5() - m.update(start) - assert(len(add) != 0) - while size > 0: - d = m.digest() - if size < len(d): - d=d[:size] - f.write(d) - size -= len(d) - m.update(add) - f.close() + with open(fn, 'wb') as f: + m = md5() + m.update(start) + assert(len(add) != 0) + while size > 0: + d = m.digest() + if size < len(d): + d=d[:size] + f.write(d) + size -= len(d) + m.update(add) def check_call(cmd): @@ -89,10 +88,9 @@ def archiveCreate(self, fn, fmt='w'): arc.close() # special case for XZ, test check type (crc32) if fmt.endswith('xz'): - f = open(fn, 'rb') - f.seek(6) - self.assertEqual(f.read(2), '\x00\x01') - f.close() + with open(fn, 'rb') as f: + f.seek(6) + self.assertEqual(f.read(2), '\x00\x01') self.archiveExtract(fn) def doArchive(self, fn, fmt=None): From bdc36179562d3ee77cc13c17414087f9f2ddc2b2 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:49:31 +0200 Subject: [PATCH 41/92] Futurize cleanups: use list comprehension not `list(map())` or `list(filter())` Looks like a bug/limitation in `2to3`. Signed-off-by: Yann Dirson --- xcp/net/ifrename/dynamic.py | 4 ++-- xcp/version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index 0528baa7..33040ffd 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -262,8 +262,8 @@ def validate(entry): % (entry, e)) return False - lastboot = list(filter(validate, self.lastboot)) - old = list(filter(validate, self.old)) + lastboot = [x for x in self.lastboot if validate(x)] + old = [x for x in self.old if validate(x)] try: res += json.dumps({"lastboot": lastboot, "old": old}, diff --git a/xcp/version.py b/xcp/version.py index 37f9e30a..e19d3c96 100644 --- a/xcp/version.py +++ b/xcp/version.py @@ -52,7 +52,7 @@ def from_string(cls, ver_str): if '-' in ver_str: ver_str, build = ver_str.split('-', 1) - ver = list(map(cls.intify, ver_str.split('.'))) + ver = [cls.intify(i) for i in ver_str.split('.')] return cls(ver, build) From a24f84ee9292da3d200fe9eeb39a579a6f47d6d9 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 27 Jul 2022 11:07:55 +0200 Subject: [PATCH 42/92] tests: avoid useless try/except Tests already fail on uncaught exception, and those except blocks are just adding dead code to tests. Signed-off-by: Yann Dirson --- tests/test_ifrename_dynamic.py | 5 +---- tests/test_ifrename_logic.py | 22 +++------------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/tests/test_ifrename_dynamic.py b/tests/test_ifrename_dynamic.py index 6313b861..1cc95e39 100644 --- a/tests/test_ifrename_dynamic.py +++ b/tests/test_ifrename_dynamic.py @@ -160,10 +160,7 @@ def test_one_invalid_lastboot(self): dr = DynamicRules() dr.lastboot = [["foo", "bar", "baz"]] - try: - json.loads(dr.write(False)) - except Exception: - self.fail() + json.loads(dr.write(False)) def test_one_ibft_lastboot(self): diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index a1850c20..670aaade 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -554,22 +554,8 @@ def tearDown(self): self.siobuff.close() - def assertNotRaises(self, excp, fn, *argl, **kwargs): - """Because unittest.TestCase seems to be missing this functionality""" - try: - fn(*argl, **kwargs) - except excp as e: - self.fail("function raised %s unexpectedly: %s" - % (excp, e)) - def test_srule_eth_unaliased(self): - - self.assertNotRaises(StaticRuleError, - rename, - [self.s111], - [], - [], - []) + rename([self.s111], [], [], []) def test_srule_eth_alias(self): """ @@ -636,8 +622,7 @@ def test_laststate_input(self): self.assertRaises(LastStateError, rename, [], [], [self.c123], []) - self.assertNotRaises(LastStateError, rename, - [], [], [self.s123], []) + rename([], [], [self.s123], []) def test_oldstate_input(self): """ @@ -648,5 +633,4 @@ def test_oldstate_input(self): self.assertRaises(OldStateError, rename, [], [], [], [self.c123]) - self.assertNotRaises(OldStateError, rename, - [], [], [], [self.s123]) + rename([], [], [], [self.s123]) From 0709e18d281a58aced9ce0990edcd9d8eff879b3 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 12:28:15 +0200 Subject: [PATCH 43/92] Futurize cleanups: xcp.net.ifrename.logic: simplify nested comprehensions Signed-off-by: Yann Dirson --- xcp/net/ifrename/logic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index 19b5b5b7..ce7fbcd1 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -336,9 +336,8 @@ def rename_logic( static_rules, # a safe new number to assign it ethnumbers = sorted( int(x[3:]) - for x in (x for x in (x.tname or x.kname - for x in static_rules + cur_state + last_state) - if VALID_ETH_NAME.match(x) is not None)) + for x in (x.tname or x.kname for x in static_rules + cur_state + last_state) + if VALID_ETH_NAME.match(x) is not None) if len(ethnumbers): nextethnum = ethnumbers[-1]+1 else: From 2b8f87c2e8c8a00cc8828741bc46a90b64392a29 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 27 Jul 2022 11:17:47 +0200 Subject: [PATCH 44/92] tests: use TestCase.assertRaises as context manager Makes for much more readable tests. Signed-off-by: Yann Dirson --- tests/test_ifrename_logic.py | 24 ++++++++++++------------ tests/test_mac.py | 30 ++++++++++++++++++++---------- tests/test_pci.py | 13 ++++++++----- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index 670aaade..3bb7a911 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -568,8 +568,8 @@ def test_srule_eth_alias(self): [self.s112, self.s222, self.s331] ] for i in srule_inputs: - self.assertRaises(StaticRuleError, rename, - i, [], [], []) + with self.assertRaises(StaticRuleError): + rename(i, [], [], []) def test_srule_mac_alias(self): """ @@ -582,8 +582,8 @@ def test_srule_mac_alias(self): [self.s211, self.s222, self.s133] ] for i in srule_inputs: - self.assertRaises(StaticRuleError, rename, - i, [], [], []) + with self.assertRaises(StaticRuleError): + rename(i, [], [], []) def test_curstate_eth_alias(self): """ @@ -596,8 +596,8 @@ def test_curstate_eth_alias(self): [self.c112, self.c222, self.c331] ] for i in curstate_inputs: - self.assertRaises(CurrentStateError, rename, - [], i, [], []) + with self.assertRaises(CurrentStateError): + rename([], i, [], []) def test_curstate_mac_alias(self): """ @@ -610,8 +610,8 @@ def test_curstate_mac_alias(self): [self.c211, self.c222, self.c133] ] for i in curstate_inputs: - self.assertRaises(CurrentStateError, rename, - [], i, [], []) + with self.assertRaises(CurrentStateError): + rename([], i, [], []) def test_laststate_input(self): """ @@ -619,8 +619,8 @@ def test_laststate_input(self): (xx:xx.x,xx:xx:xx:xx:xx:xx)->eth## """ - self.assertRaises(LastStateError, rename, - [], [], [self.c123], []) + with self.assertRaises(LastStateError): + rename([], [], [self.c123], []) rename([], [], [self.s123], []) @@ -630,7 +630,7 @@ def test_oldstate_input(self): (xx:xx.x,xx:xx:xx:xx:xx:xx)->eth## """ - self.assertRaises(OldStateError, rename, - [], [], [], [self.c123]) + with self.assertRaises(OldStateError): + rename([], [], [], [self.c123]) rename([], [], [], [self.s123]) diff --git a/tests/test_mac.py b/tests/test_mac.py index 310e17e3..7976a3bf 100644 --- a/tests/test_mac.py +++ b/tests/test_mac.py @@ -11,18 +11,21 @@ class TestInvalidMAC(unittest.TestCase): def test_null_str(self): val = "" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_non_str(self): for val in [None, [], {}]: - self.assertRaises(TypeError, MAC, val) + with self.assertRaises(TypeError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_colon_too_few_octets(self): val = "00:00:00:00:00" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_colon_invalid_octets(self): @@ -32,27 +35,32 @@ def test_colon_invalid_octets(self): "12:34:56g:78:90:ab" "12:34::78:90:ab" ]: - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_colon_too_many_octets(self): val = "00:00:00:00:00:00:00" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dash_too_few_octetes(self): val = "00-00-00-00-00" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dash_too_many_octets(self): val = "00-00-00-00-00-00-00" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dotquad_too_few_quads(self): val = "0000.0000" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dotquad_invalid_quads(self): @@ -62,12 +70,14 @@ def test_dotquad_invalid_quads(self): "abcd.efgh.ijkl", "1234.-5678.90Ab" ]: - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dotquad_too_many_quads(self): val = "0000.0000.0000.0000" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) class TestValidMAC(unittest.TestCase): diff --git a/tests/test_pci.py b/tests/test_pci.py index ac03c375..1895e88a 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -7,11 +7,14 @@ class TestInvalid(unittest.TestCase): def test_invalid_types(self): - - self.assertRaises(TypeError, PCI, 0) - self.assertRaises(TypeError, PCI, (0,)) - self.assertRaises(TypeError, PCI, []) - self.assertRaises(TypeError, PCI, {}) + with self.assertRaises(TypeError): + PCI(0) + with self.assertRaises(TypeError): + PCI((0,)) + with self.assertRaises(TypeError): + PCI([]) + with self.assertRaises(TypeError): + PCI({}) def test_invalid_format(self): pass From 8fee05afe1cb104afd58a31fb84f60e90a50d5ad Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 12:28:51 +0200 Subject: [PATCH 45/92] Futurize cleanups: line-wrapping fixes Signed-off-by: Yann Dirson --- xcp/pci.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xcp/pci.py b/xcp/pci.py index 85f82ec7..dc8d93d4 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -278,8 +278,7 @@ def findByClass(self, cls, subcls = None): [class1, class2, ... classN]""" if subcls: assert isinstance(cls, str) - return [x for x in self.devs.values() if x['class'] == cls and - x['subclass'] == subcls] + return [x for x in self.devs.values() if x['class'] == cls and x['subclass'] == subcls] else: assert isinstance(cls, list) return [x for x in self.devs.values() if x['class'] in cls] From ce34f68a71c5a5252c3dae7a4d45560a9b26143d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 16:51:52 +0200 Subject: [PATCH 46/92] Mark bytes literals and streams as such Signed-off-by: Yann Dirson --- tests/test_cpio.py | 8 ++--- xcp/cpiofile.py | 87 +++++++++++++++++++++++----------------------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 5af86b92..651a4204 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -8,7 +8,7 @@ from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED -def writeRandomFile(fn, size, start='', add='a'): +def writeRandomFile(fn, size, start=b'', add=b'a'): with open(fn, 'wb') as f: m = md5() m.update(start) @@ -36,7 +36,7 @@ def setUp(self): shutil.rmtree('archive', True) os.mkdir('archive') writeRandomFile('archive/data', 10491) - with open('archive/data') as fd: + with open('archive/data', 'rb') as fd: self.md5data = md5(fd.read()).hexdigest() check_call("find archive | cpio -o -H newc > archive.cpio") check_call("gzip -c < archive.cpio > archive.cpio.gz") @@ -78,7 +78,7 @@ def archiveCreate(self, fn, fmt='w'): os.unlink(fn) arc = CpioFile.open(fn, fmt) f = arc.getcpioinfo('archive/data') - with open('archive/data') as fd: + with open('archive/data', 'rb') as fd: arc.addfile(f, fd) # test recursively add "." os.chdir('archive') @@ -90,7 +90,7 @@ def archiveCreate(self, fn, fmt='w'): if fmt.endswith('xz'): with open(fn, 'rb') as f: f.seek(6) - self.assertEqual(f.read(2), '\x00\x01') + self.assertEqual(f.read(2), b'\x00\x01') self.archiveExtract(fn) def doArchive(self, fn, fmt=None): diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index a3905786..c2728940 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -72,9 +72,9 @@ # cpio constants #--------------------------------------------------------- MAGIC_NEWC = 0x070701 # magic for SVR4 portable format (no CRC) -TRAILER_NAME = "TRAILER!!!" # filename in final member +TRAILER_NAME = b"TRAILER!!!" # filename in final member WORDSIZE = 4 # pad size -NUL = "\0" # the null character +NUL = b"\0" # the null character BLOCKSIZE = 512 # length of processing blocks HEADERSIZE_SVR4 = 110 # length of fixed header @@ -255,7 +255,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.comptype = comptype self.fileobj = fileobj self.bufsize = bufsize - self.buf = "" + self.buf = b"" self.pos = 0 self.closed = False @@ -265,7 +265,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): except ImportError: raise CompressionError("zlib module is not available") self.zlib = zlib - self.crc = zlib.crc32("") + self.crc = zlib.crc32(b"") if mode == "r": self._init_read_gz() else: @@ -277,7 +277,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): except ImportError: raise CompressionError("bz2 module is not available") if mode == "r": - self.dbuf = "" + self.dbuf = b"" self.cmp = bz2.BZ2Decompressor() else: self.cmp = bz2.BZ2Compressor() @@ -288,7 +288,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): except ImportError: raise CompressionError("lzma module is not available") if mode == "r": - self.dbuf = "" + self.dbuf = b"" self.cmp = lzma.LZMADecompressor() else: self.cmp = lzma.LZMACompressor() @@ -306,7 +306,7 @@ def _init_write_gz(self): self.zlib.DEF_MEM_LEVEL, 0) timestamp = struct.pack(" Date: Mon, 18 Jul 2022 11:18:53 +0200 Subject: [PATCH 47/92] Futurize cleanups: cleanup duplicated imports Signed-off-by: Yann Dirson --- xcp/accessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index be98e545..c1318002 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -30,8 +30,8 @@ import ftplib import os import tempfile -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse +import urllib.request +import urllib.error import urllib.parse import errno From bfd5467ac779291cab1669380b4558c4d680abd8 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 15:40:45 +0200 Subject: [PATCH 48/92] test_cpio: add some docstrings Signed-off-by: Yann Dirson --- tests/test_cpio.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 651a4204..4773ef55 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -9,6 +9,7 @@ from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED def writeRandomFile(fn, size, start=b'', add=b'a'): + "Create a pseudo-random reproducible file from seeds `start` amd `add`" with open(fn, 'wb') as f: m = md5() m.update(start) From f63f4a1c63b0a1280f839a5936899019c015615f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 16:19:56 +0200 Subject: [PATCH 49/92] python3: use six.string_types not version-dependant types Use of `unicode` needed to be immediately handled, but a few checks relying on `str` could become insufficient in python2 with the larger usage of unicode strings. Signed-off-by: Yann Dirson --- xcp/cmd.py | 3 ++- xcp/logger.py | 3 ++- xcp/net/mac.py | 3 ++- xcp/pci.py | 5 +++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/xcp/cmd.py b/xcp/cmd.py index bbd94656..fbb991d0 100644 --- a/xcp/cmd.py +++ b/xcp/cmd.py @@ -24,6 +24,7 @@ """Command processing""" import subprocess +import six import xcp.logger as logger @@ -32,7 +33,7 @@ def runCmd(command, with_stdout = False, with_stderr = False, inputtext = None): stdin = (inputtext and subprocess.PIPE or None), stdout = subprocess.PIPE, stderr = subprocess.PIPE, - shell = isinstance(command, str)) + shell = isinstance(command, six.string_types)) (out, err) = cmd.communicate(inputtext) rv = cmd.returncode diff --git a/xcp/logger.py b/xcp/logger.py index 03e8fe78..ca972b5c 100644 --- a/xcp/logger.py +++ b/xcp/logger.py @@ -33,6 +33,7 @@ import logging import logging.handlers +import six LOG = logging.getLogger() LOG.setLevel(logging.NOTSET) @@ -47,7 +48,7 @@ def openLog(lfile, level=logging.INFO): try: # if lfile is a string, assume we need to open() it - if isinstance(lfile, str): + if isinstance(lfile, six.string_types): h = open(lfile, 'a') if h.isatty(): handler = logging.StreamHandler(h) diff --git a/xcp/net/mac.py b/xcp/net/mac.py index 47e586c0..1e153c9a 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -31,6 +31,7 @@ __author__ = "Andrew Cooper" import re +import six VALID_COLON_MAC = re.compile(r"^([\da-fA-F]{1,2}:){5}[\da-fA-F]{1,2}$") VALID_DASH_MAC = re.compile(r"^([\da-fA-F]{1,2}-){5}[\da-fA-F]{1,2}$") @@ -59,7 +60,7 @@ def __init__(self, addr): self.octets = [] self.integer = -1 - if isinstance(addr, (str, unicode)): + if isinstance(addr, six.string_types): res = VALID_COLON_MAC.match(addr) if res: diff --git a/xcp/pci.py b/xcp/pci.py index 1c8e081d..393be196 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -24,6 +24,7 @@ import os.path import subprocess import re +import six _SBDF = (r"(?:(?P [\da-dA-F]{4}):)?" # Segment (optional) " (?P [\da-fA-F]{2}):" # Bus @@ -66,7 +67,7 @@ def __init__(self, addr): self.function = -1 self.index = -1 - if isinstance(addr, (str, unicode)): + if isinstance(addr, six.string_types): res = VALID_SBDFI.match(addr) if res: @@ -277,7 +278,7 @@ def findByClass(self, cls, subcls = None): class, subclass [class1, class2, ... classN]""" if subcls: - assert isinstance(cls, str) + assert isinstance(cls, six.string_types) return [x for x in self.devs.values() if x['class'] == cls and x['subclass'] == subcls] else: assert isinstance(cls, list) From 1311528e5fd489ba0b5189fcd9f725be4b37c07c Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:13:54 +0200 Subject: [PATCH 50/92] Futurize pylint complements: xcp.accessor: bring consistency to "openAddress" arguments Pylint would report: xcp/accessor.py:377: W0221: (arguments-differ), HTTPAccessor.openAddress: Parameters differ from overridden 'openAddress' method Signed-off-by: Yann Dirson --- xcp/accessor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index c1318002..a63ed6f7 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -69,7 +69,7 @@ def access(self, name): return True - def openAddress(self, name): + def openAddress(self, address): """should be overloaded""" pass @@ -97,9 +97,9 @@ def __init__(self, location, ro): super(FilesystemAccessor, self).__init__(ro) self.location = location - def openAddress(self, addr): + def openAddress(self, address): try: - file = open(os.path.join(self.location, addr), 'r') + file = open(os.path.join(self.location, address), 'r') except OSError as e: if e.errno == errno.EIO: self.lastError = 5 From 1e1b9a4f377a11f89502c216f237616bc7c4c642 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 16:21:02 +0200 Subject: [PATCH 51/92] test_cpio: generate reproducible archives This helps understanding what really differs between 2 platforms. Signed-off-by: Yann Dirson --- tests/test_cpio.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 4773ef55..b6568c88 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -39,7 +39,12 @@ def setUp(self): writeRandomFile('archive/data', 10491) with open('archive/data', 'rb') as fd: self.md5data = md5(fd.read()).hexdigest() - check_call("find archive | cpio -o -H newc > archive.cpio") + # fixed timestamps for cpio reproducibility + os.utime('archive/data', (0, 0)) + os.utime('archive', (0, 0)) + + check_call( + "find archive | cpio --reproducible -o -H newc > archive.cpio") check_call("gzip -c < archive.cpio > archive.cpio.gz") check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") try: From 4424275d37b8a8b9f5f796f0459e1ed90b07fc25 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 18:07:43 +0200 Subject: [PATCH 52/92] python3: use "six.ensure_binary" and "six.ensure_text" for str/bytes conversion Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 7b2623fa..149cbcda 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -310,7 +310,7 @@ def _init_write_gz(self): self.__write(b"\037\213\010\010%s\002\377" % timestamp) if self.name.endswith(".gz"): self.name = self.name[:-3] - self.__write(self.name + NUL) + self.__write(six.ensure_binary(self.name) + NUL) def write(self, s): """Write string s to the stream. @@ -1433,7 +1433,7 @@ def extractall(self, path=".", members=None): # Set correct owner, mtime and filemode on directories. for cpioinfo in directories: - path = os.path.join(path, cpioinfo.name) + path = os.path.join(path, six.ensure_text(cpioinfo.name)) try: self.chown(cpioinfo, path) self.utime(cpioinfo, path) From c2902d719468458b34423e89c9b673a37092c8fe Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:14:49 +0200 Subject: [PATCH 53/92] Futurize pylint complements: disable warnings triggered by "import future" Signed-off-by: Yann Dirson --- xcp/accessor.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index a63ed6f7..2891a6bc 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -25,14 +25,16 @@ """accessor - provide common interface to access methods""" +# pylint: disable=wrong-import-position,wrong-import-order from future import standard_library standard_library.install_aliases() + import ftplib import os import tempfile -import urllib.request -import urllib.error -import urllib.parse +import urllib.request # pylint: disable=import-error +import urllib.error # pylint: disable=import-error +import urllib.parse # pylint: disable=import-error import errno import xcp.mount as mount From 7e06575c00ec1e981023ad0cf788e7cb8ede5fa9 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 16:22:43 +0200 Subject: [PATCH 54/92] test_cpio: avoid overwriting original archives with "archiveCreate()" When hunting for platform difference, it helps being able to tell that a difference comes from the original archives, now reproducible, or from the ones generated by the code under test. Signed-off-by: Yann Dirson --- tests/test_cpio.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index b6568c88..e41bd445 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -57,7 +57,7 @@ def setUp(self): self.doXZ = False def tearDown(self): - check_call("rm -rf archive archive.cpio* archive2") + check_call("rm -rf archive archive.cpio* archive2 archive2.cpio*") # TODO check with file (like 'r:*') # TODO use cat to check properly for pipes @@ -81,7 +81,8 @@ def archiveExtract(self, fn, fmt='r|*'): arc.close() def archiveCreate(self, fn, fmt='w'): - os.unlink(fn) + if os.path.exists(fn): + os.unlink(fn) arc = CpioFile.open(fn, fmt) f = arc.getcpioinfo('archive/data') with open('archive/data', 'rb') as fd: @@ -101,9 +102,11 @@ def archiveCreate(self, fn, fmt='w'): def doArchive(self, fn, fmt=None): self.archiveExtract(fn) - self.archiveCreate(fn, fmt is None and 'w' or 'w|%s' % fmt ) + fn2 = "archive2" + fn[len("archive"):] + print("creating %s" % fn2) + self.archiveCreate(fn2, fmt is None and 'w' or 'w|%s' % fmt) if not fmt is None: - self.archiveExtract(fn, 'r|%s' % fmt) + self.archiveExtract(fn2, 'r|%s' % fmt) def test_plain(self): self.doArchive('archive.cpio') From 05ac34f2b156c9973cdf99bd41c8da212cf0c854 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 25 May 2007 20:17:15 +0000 Subject: [PATCH 55/92] Remove direct call's to file's constructor and replace them with calls to open() as ths is considered best practice. (cherry picked from cpython commit 6cef076ba5edbfa42239924951d8acbb087b3b19) Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 149cbcda..0ddaff6c 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -951,7 +951,7 @@ def __init__(self, name=None, mode="r", fileobj=None): self.mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode] if not fileobj: - fileobj = file(name, self.mode) + fileobj = bltn_open(name, self.mode) self._extfileobj = False else: if name is None and hasattr(fileobj, "name"): @@ -1109,7 +1109,7 @@ def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9): raise CompressionError("gzip module is not available") if fileobj is None: - fileobj = file(name, mode + "b") + fileobj = bltn_open(name, mode + "b") try: t = cls.cpioopen(name, mode, gzip.GzipFile(name, mode, compresslevel, fileobj)) @@ -1354,7 +1354,7 @@ def add(self, name, arcname=None, recursive=True): # Append the cpio header and data to the archive. if cpioinfo.isreg(): - f = file(name, "rb") + f = bltn_open(name, "rb") self.addfile(cpioinfo, f) f.close() @@ -1594,7 +1594,7 @@ def makefile(self, cpioinfo, targetpath): if extractinfo: source = self.extractfile(extractinfo) - target = file(targetpath, "wb") + target = bltn_open(targetpath, "wb") copyfileobj(source, target) source.close() target.close() @@ -1926,5 +1926,5 @@ def is_cpiofile(name): except CpioError: return False -def cpioOpen(*al, **ad): - return CpioFile.open(*al, **ad) +bltn_open = open +open = CpioFile.open From bbd7cbeb3d5dde0b9454b98ef14e9f19fca1ca48 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:18:14 +0200 Subject: [PATCH 56/92] Futurize pylint complements: test_cpio: mark unused import as expected This one is only for checking feature availability. Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 4cb13825..defde234 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -46,7 +46,7 @@ def setUp(self): check_call("gzip -c < archive.cpio > archive.cpio.gz") check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") try: - import lzma + import lzma # pylint: disable=unused-variable self.doXZ = subprocess.call("xz --check=crc32 --lzma2=dict=1MiB < archive.cpio > archive.cpio.xz", shell=True) == 0 except Exception as ex: # FIXME will issue warning even if test_xz is not requested From b726df7fd00a8cec381b28de3c039df148c2afc3 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 16:46:35 +0200 Subject: [PATCH 57/92] test_cpio: improve test readability Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index e41bd445..e8e200e0 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -105,7 +105,7 @@ def doArchive(self, fn, fmt=None): fn2 = "archive2" + fn[len("archive"):] print("creating %s" % fn2) self.archiveCreate(fn2, fmt is None and 'w' or 'w|%s' % fmt) - if not fmt is None: + if fmt is not None: self.archiveExtract(fn2, 'r|%s' % fmt) def test_plain(self): From 7ad1bdd48e305a4fb80a2572a77c9e3c136d42c8 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 18:15:22 +0200 Subject: [PATCH 58/92] python3: xcp.net.mac: use six.python_2_unicode_compatible for stringification Signed-off-by: Yann Dirson --- xcp/net/mac.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xcp/net/mac.py b/xcp/net/mac.py index 1e153c9a..c0d4fba0 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -37,6 +37,7 @@ VALID_DASH_MAC = re.compile(r"^([\da-fA-F]{1,2}-){5}[\da-fA-F]{1,2}$") VALID_DOTQUAD_MAC = re.compile(r"^([\da-fA-F]{1,4}\.){2}[\da-fA-F]{1,4}$") +@six.python_2_unicode_compatible class MAC(object): """ Mac address object for manipulation and comparison @@ -123,9 +124,6 @@ def is_local(self): def __str__(self): - return unicode(self).encode('utf-8') - - def __unicode__(self): return ':'.join([ "%0.2x" % x for x in self.octets]) def __repr__(self): From fe914abddbc2e23e28d2813a348eed8607357003 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:34:17 +0200 Subject: [PATCH 59/92] Futurize pylint complements: xcp.accessor: avoid redefining "file" builtin Signed-off-by: Yann Dirson --- xcp/accessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 2891a6bc..70306933 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -101,7 +101,7 @@ def __init__(self, location, ro): def openAddress(self, address): try: - file = open(os.path.join(self.location, address), 'r') + filehandle = open(os.path.join(self.location, address), 'r') except OSError as e: if e.errno == errno.EIO: self.lastError = 5 @@ -117,7 +117,7 @@ def openAddress(self, address): except Exception as e: self.lastError = 500 return False - return file + return filehandle class MountingAccessor(FilesystemAccessor): def __init__(self, mount_types, mount_source, mount_options = None): From 7ef098cbdeeaf22228052b54f09be25846050350 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 17:13:11 +0200 Subject: [PATCH 60/92] test_cpio: expect different xz flags depending on python version References: * xz file-format spec: https://tukaani.org/xz/xz-file-format.txt * python3 defaults: https://docs.python.org/3/library/lzma.html#lzma.LZMACompressor * python2 defaults: Help on class LZMAFile in module lzma: class LZMAFile(__builtin__.object) | LZMAFile(name [, mode='r', buffering=0, memlimit=-1, | options={'format':'xz', 'check':'crc32', 'level':6, 'extreme':False, | 'dict_size':23, 'lc':3 'lp':0, 'pb':2, 'mode':2, | 'nice_len':128, 'mf':'bt4', 'depth':0]) -> file object Signed-off-by: Yann Dirson --- tests/test_cpio.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index e8e200e0..0ddd2ce8 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -1,6 +1,7 @@ from __future__ import print_function from hashlib import md5 import os +import sys import shutil import subprocess import unittest @@ -97,7 +98,12 @@ def archiveCreate(self, fn, fmt='w'): if fmt.endswith('xz'): with open(fn, 'rb') as f: f.seek(6) - self.assertEqual(f.read(2), b'\x00\x01') + # check stream flags + if sys.version_info < (3, 0): + expected_flags = b'\x00\x01' # pylzma defaults to CRC32 + else: + expected_flags = b'\x00\x04' # python3 defaults to CRC64 + self.assertEqual(f.read(2), expected_flags) self.archiveExtract(fn) def doArchive(self, fn, fmt=None): From 6bda8a2c253a1bb8e3e23512d67a186e426f3781 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 18:21:17 +0200 Subject: [PATCH 61/92] xcp.net.ifrename.logic: use "logger.warning", "logger.warn" is deprecated Signed-off-by: Yann Dirson --- xcp/net/ifrename/logic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index e2a3484b..1aa534c0 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -289,9 +289,10 @@ def rename_logic( static_rules, # Check that the function still has the same number of nics if len(lastnics) != len(newnics): - LOG.warn("multi-nic function %s had %d nics but now has %d. " - "Defering all until later for renaming" - % (fn, len(lastnics), len(newnics))) + LOG.warning( + "multi-nic function %s had %d nics but now has %d. " + "Defering all until later for renaming", + fn, len(lastnics), len(newnics)) continue # Check that all nics are still pending a rename From fe0c12156222563fc53872c99d1a2a2110b259da Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:37:42 +0200 Subject: [PATCH 62/92] Futurize pylint complements: Avoid useless ".keys()" when enumerating dicts Signed-off-by: Yann Dirson --- xcp/accessor.py | 2 +- xcp/net/ifrename/static.py | 3 +-- xcp/pci.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 70306933..4d092607 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -395,5 +395,5 @@ def __repr__(self): def createAccessor(baseAddress, *args): url_parts = urllib.parse.urlsplit(baseAddress, allow_fragments=False) - assert url_parts.scheme in SUPPORTED_ACCESSORS.keys() + assert url_parts.scheme in SUPPORTED_ACCESSORS return SUPPORTED_ACCESSORS[url_parts.scheme](baseAddress, *args) diff --git a/xcp/net/ifrename/static.py b/xcp/net/ifrename/static.py index be1e695d..c1b9b100 100644 --- a/xcp/net/ifrename/static.py +++ b/xcp/net/ifrename/static.py @@ -298,8 +298,7 @@ def write(self, header = True): if header: res += SAVE_HEADER - keys = list(set(( x for x in self.formulae.keys() - if x.startswith("eth") ))) + keys = list(set((x for x in self.formulae if x.startswith("eth")))) keys.sort(key=lambda x: int(x[3:])) for target in keys: diff --git a/xcp/pci.py b/xcp/pci.py index dc8d93d4..cff6d506 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -289,7 +289,7 @@ def slot(dev): left, _ = dev.rsplit('.', 1) return left - return [x for x in self.devs.keys() if x != dev and slot(x) == slot(dev)] + return [x for x in self.devs if x != dev and slot(x) == slot(dev)] def pci_sbdfi_to_nic(sbdfi, nics): From d24816e814a6336bf320e3f3113f67a3e72abcae Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 17:15:28 +0200 Subject: [PATCH 63/92] test_cpio: check xz magic, not just the flags Just safer than just checking for 2 single-digit bytes. Signed-off-by: Yann Dirson --- tests/test_cpio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 0ddd2ce8..101a05de 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -97,7 +97,8 @@ def archiveCreate(self, fn, fmt='w'): # special case for XZ, test check type (crc32) if fmt.endswith('xz'): with open(fn, 'rb') as f: - f.seek(6) + # check xz magic + self.assertEqual(f.read(6), b"\xfd7zXZ\0") # check stream flags if sys.version_info < (3, 0): expected_flags = b'\x00\x01' # pylzma defaults to CRC32 From 4bffe8e61aee6d82e8e873d185dfd6af10319259 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 18:22:58 +0200 Subject: [PATCH 64/92] python3: use raw strings for regexps, fixes insufficient quoting Running tests on python3 did reveal some of them. Signed-off-by: Yann Dirson --- xcp/bootloader.py | 6 +++--- xcp/dom0.py | 2 +- xcp/net/ifrename/logic.py | 6 +++--- xcp/net/ifrename/static.py | 2 +- xcp/pci.py | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index a1d19709..81a61bd4 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -336,19 +336,19 @@ def create_label(title): try: for line in fh: l = line.strip() - menu_match = re.match("menuentry ['\"]([^']*)['\"](.*){", l) + menu_match = re.match(r"menuentry ['\"]([^']*)['\"](.*){", l) # Only parse unindented default and timeout lines to prevent # changing these lines in if statements. if l.startswith('set default=') and l == line.rstrip(): default = l.split('=')[1] - match = re.match("['\"](.*)['\"]$", default) + match = re.match(r"['\"](.*)['\"]$", default) if match: default = match.group(1) elif l.startswith('set timeout=') and l == line.rstrip(): timeout = int(l.split('=')[1]) * 10 elif l.startswith('serial'): - match = re.match("serial --unit=(\d+) --speed=(\d+)", l) + match = re.match(r"serial --unit=(\d+) --speed=(\d+)", l) if match: serial = { 'port': int(match.group(1)), diff --git a/xcp/dom0.py b/xcp/dom0.py index 086a683b..b8a46c3a 100644 --- a/xcp/dom0.py +++ b/xcp/dom0.py @@ -96,7 +96,7 @@ def default_memory(host_mem_kib): return default_memory_for_version(host_mem_kib, platform_version) -_size_and_unit_re = re.compile("^(-?\d+)([bkmg]?)$", re.IGNORECASE) +_size_and_unit_re = re.compile(r"^(-?\d+)([bkmg]?)$", re.IGNORECASE) def _parse_size_and_unit(s): m = _size_and_unit_re.match(s) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index 1aa534c0..41e74c02 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -52,9 +52,9 @@ from xcp.logger import LOG from xcp.net.ifrename.macpci import MACPCI -VALID_CUR_STATE_KNAME = re.compile("^(?:eth[\d]+|side-[\d]+-eth[\d]+)$") -VALID_ETH_NAME = re.compile("^eth([\d])+$") -VALID_IBFT_NAME = re.compile("^ibft([\d])+$") +VALID_CUR_STATE_KNAME = re.compile(r"^(?:eth[\d]+|side-[\d]+-eth[\d]+)$") +VALID_ETH_NAME = re.compile(r"^eth([\d])+$") +VALID_IBFT_NAME = re.compile(r"^ibft([\d])+$") # util needs to import VALID_ETH_NAME from xcp.net.ifrename import util diff --git a/xcp/net/ifrename/static.py b/xcp/net/ifrename/static.py index c1b9b100..bf503d54 100644 --- a/xcp/net/ifrename/static.py +++ b/xcp/net/ifrename/static.py @@ -89,7 +89,7 @@ class StaticRules(object): methods = ["mac", "pci", "ppn", "label", "guess"] validators = { "mac": VALID_MAC, "pci": VALID_PCI, - "ppn": re.compile("^(?:em\d+|p(?:ci)?\d+p\d+)$") + "ppn": re.compile(r"^(?:em\d+|p(?:ci)?\d+p\d+)$") } def __init__(self, path=None, fd=None): diff --git a/xcp/pci.py b/xcp/pci.py index 393be196..d6cb4fef 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -27,9 +27,9 @@ import six _SBDF = (r"(?:(?P [\da-dA-F]{4}):)?" # Segment (optional) - " (?P [\da-fA-F]{2}):" # Bus - " (?P [\da-fA-F]{2})\." # Device - " (?P[\da-fA-F])" # Function + r" (?P [\da-fA-F]{2}):" # Bus + r" (?P [\da-fA-F]{2})\." # Device + r" (?P[\da-fA-F])" # Function ) # Don't change the meaning of VALID_SBDF as some parties may be using it @@ -37,7 +37,7 @@ VALID_SBDFI = re.compile( r"^(?P%s)" - " (?:[[](?P[\d]{1,2})[]])?$" # Index (optional) + r" (?:[[](?P[\d]{1,2})[]])?$" # Index (optional) % _SBDF , re.X) From 9c08b5d96cfbc5e1cbdebb19fc3ea3554feadab3 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:43:05 +0200 Subject: [PATCH 65/92] Futurize pylint complements: xcp.repository: honor singleton-comparison convention Signed-off-by: Yann Dirson --- xcp/repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/repository.py b/xcp/repository.py index e83e14f2..14877d5c 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -46,7 +46,7 @@ def __init__(self, repository, label, size, md5sum, optional, fname, root): self.optional, self.filename, self.destination - ) = ( repository, label, size, md5sum, (optional==True), fname, root ) + ) = (repository, label, size, md5sum, optional is True, fname, root) def __repr__(self): return "" % self.label @@ -61,7 +61,7 @@ def __init__(self, repository, label, size, md5sum, optional, fname, options): self.optional, self.filename, self.options - ) = ( repository, label, size, md5sum, (optional==True), fname, options ) + ) = (repository, label, size, md5sum, optional is True, fname, options) def __repr__(self): return "" % self.label From c16e90ce54ed46f79ff9d89c7f9e0ed281fa1a6f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 17:16:32 +0200 Subject: [PATCH 66/92] cpio: fix check for xz file magic ignoring the 6th byte Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index c2728940..f6e0dd6b 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -480,7 +480,7 @@ def getcomptype(self): return "gz" if self.buf.startswith(b"BZh91"): return "bz2" - if self.buf.startswith(b"\xfd7zXZ"): + if self.buf.startswith(b"\xfd7zXZ\0"): return "xz" return "cpio" From 686ff2d0f65e33335eb9dad73032fcfb315d5966 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 12:06:59 +0200 Subject: [PATCH 67/92] test_dom0: mock "open()" in a python3-compatible way Signed-off-by: Yann Dirson --- tests/test_dom0.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dom0.py b/tests/test_dom0.py index bf198341..440bb109 100644 --- a/tests/test_dom0.py +++ b/tests/test_dom0.py @@ -30,7 +30,7 @@ def mock_version(open_mock, version): (2*1024, 4*1024, 8*1024), # Above max ] - with patch("__builtin__.open") as open_mock: + with patch("xcp.dom0.open") as open_mock: for host_gib, dom0_mib, _ in test_values: mock_version(open_mock, '2.8.0') expected = dom0_mib * 1024; @@ -39,7 +39,7 @@ def mock_version(open_mock, version): open_mock.assert_called_with("/etc/xensource-inventory") - with patch("__builtin__.open") as open_mock: + with patch("xcp.dom0.open") as open_mock: for host_gib, _, dom0_mib in test_values: mock_version(open_mock, '2.9.0') expected = dom0_mib * 1024; From 4b92d6a5a98e6db1745d90fb8f448943980cf329 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:44:51 +0200 Subject: [PATCH 68/92] Futurize pylint complements: xcp.repository: locally disable too-few-public-methods Signed-off-by: Yann Dirson --- xcp/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/repository.py b/xcp/repository.py index 14877d5c..c084d1d8 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -33,7 +33,7 @@ import xcp.version as version import xcp.xmlunwrap as xmlunwrap -class Package(object): +class Package(object): # pylint: disable=too-few-public-methods pass class BzippedPackage(Package): From 08abd009f156a5a17ae53bf7fd22d0459d8cc639 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 16:14:32 +0200 Subject: [PATCH 69/92] test_cpio: remove unused import (pylint on python3) Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 101a05de..c3543c89 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -7,7 +7,7 @@ import unittest import warnings -from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED +from xcp.cpiofile import CpioFile, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED def writeRandomFile(fn, size, start=b'', add=b'a'): "Create a pseudo-random reproducible file from seeds `start` amd `add`" From a5853cd8a17c3cf4b85531d97f1d7913da0fee81 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 15:05:25 +0200 Subject: [PATCH 70/92] ifrename: don't rely on dict ordering in tests There is no guaranty about ordering of dict elements, and tests compare results derived from enumerating a dict element. We could have used an OrderedDict to store the formulae and get a predictible output order, but just considering the output as a set seems better. Only applying this to rules expected to hold more than one element. Signed-off-by: Yann Dirson --- tests/test_ifrename_dynamic.py | 4 ++-- tests/test_ifrename_static.py | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_ifrename_dynamic.py b/tests/test_ifrename_dynamic.py index 1cc95e39..0948d254 100644 --- a/tests/test_ifrename_dynamic.py +++ b/tests/test_ifrename_dynamic.py @@ -125,10 +125,10 @@ def test_pci_matching_invert(self): MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", kname="eth1", ppn="", label="")]) - self.assertEqual(dr.rules,[ + self.assertEqual(set(dr.rules), set([ MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1"), MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0") - ]) + ])) def test_pci_missing(self): diff --git a/tests/test_ifrename_static.py b/tests/test_ifrename_static.py index 9b11e380..674decfe 100644 --- a/tests/test_ifrename_static.py +++ b/tests/test_ifrename_static.py @@ -375,10 +375,10 @@ def test_pci_matching(self): sr.generate(self.state) - self.assertEqual(sr.rules,[ - MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth1"), - MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth0") - ]) + self.assertEqual(set(sr.rules), set([ + MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth1"), + MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth0") + ])) def test_pci_matching_invert(self): @@ -389,10 +389,10 @@ def test_pci_matching_invert(self): sr.generate(self.state) - self.assertEqual(sr.rules,[ - MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1"), - MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0") - ]) + self.assertEqual(set(sr.rules), set([ + MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1"), + MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0") + ])) def test_pci_matching_mixed(self): @@ -403,10 +403,10 @@ def test_pci_matching_mixed(self): sr.generate(self.state) - self.assertEqual(sr.rules,[ - MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0"), - MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1") - ]) + self.assertEqual(set(sr.rules), set([ + MACPCI("c8:cb:b8:d3:0c:cf", "0000:04:00.0", tname="eth0"), + MACPCI("c8:cb:b8:d3:0c:ce", "0000:04:00.0", tname="eth1") + ])) def test_pci_missing(self): From 95bc54e40fd4e61fdce8eed6388f7782a36542eb Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:02:04 +0200 Subject: [PATCH 71/92] Futurize pylint complements: "print" continuation indentations, line lengths Signed-off-by: Yann Dirson --- tests/test_cpio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index defde234..628c54ca 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -47,7 +47,8 @@ def setUp(self): check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") try: import lzma # pylint: disable=unused-variable - self.doXZ = subprocess.call("xz --check=crc32 --lzma2=dict=1MiB < archive.cpio > archive.cpio.xz", shell=True) == 0 + self.doXZ = subprocess.call("xz --check=crc32 --lzma2=dict=1MiB" + " < archive.cpio > archive.cpio.xz", shell=True) == 0 except Exception as ex: # FIXME will issue warning even if test_xz is not requested warnings.warn("will not test cpio.xz: %s" % ex) From 50028ec7b586d6741e03d69ecb690cac40f893eb Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 10 Aug 2022 12:22:39 +0200 Subject: [PATCH 72/92] cpiofile: revert unnecessary deviations from original tarfile.py Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index f6e0dd6b..d9c556e2 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#!/usr/bin/env python # -*- coding: iso-8859-1 -*- #------------------------------------------------------------------- # cpiofile.py @@ -37,7 +37,7 @@ __version__ = "0.1" __author__ = "Simon Rowe" -__credits__ = "Lars Gustäbel" +__credits__ = "Lars Gustäbel, Gustavo Niemeyer, Niels Gustäbel, Richard Townsend." #--------- # Imports @@ -666,10 +666,6 @@ def readsparsesection(self, size): # return NUL * size #class _FileInFile -SEEK_SET = 0 -SEEK_CUR = 1 -SEEK_END = 2 - class ExFileObject(object): """File-like object for reading an archive member. Is returned by CpioFile.extractfile(). @@ -763,20 +759,20 @@ def tell(self): return self.position - def seek(self, pos, whence=SEEK_SET): + def seek(self, pos, whence=os.SEEK_SET): """Seek to a position in the file. """ if self.closed: raise ValueError("I/O operation on closed file") - if whence == SEEK_SET: + if whence == os.SEEK_SET: self.position = min(max(pos, 0), self.size) - elif whence == SEEK_CUR: + elif whence == os.SEEK_CUR: if pos < 0: self.position = max(self.position + pos, 0) else: self.position = min(self.position + pos, self.size) - elif whence == SEEK_END: + elif whence == os.SEEK_END: self.position = max(min(self.size + pos, self.size), 0) else: raise ValueError("Invalid argument") @@ -1195,11 +1191,6 @@ def close(self): self.fileobj.write(buf) self.offset += len(buf) -# blocks, remainder = divmod(self.offset, BLOCKSIZE) -# if remainder > 0: -# self.fileobj.write((BLOCKSIZE - remainder) * NUL) -# self.offset += (BLOCKSIZE - remainder) - if not self._extfileobj: self.fileobj.close() self.closed = True @@ -1312,7 +1303,14 @@ def list(self, verbose=True): print("%10d" % cpioinfo.size, end=' ') print("%d-%02d-%02d %02d:%02d:%02d" % time.localtime(cpioinfo.mtime)[:6], end=' ') - print(cpioinfo.name) + print(cpioinfo.name, end="") + + if verbose: + if cpioinfo.issym(): + print("->", cpioinfo.linkname, end="") + if cpioinfo.islnk(): + print("link to", cpioinfo.linkname, end="") + print() def add(self, name, arcname=None, recursive=True): """Add the file `name' to the archive. `name' may be any type of file @@ -1457,7 +1455,6 @@ def extract(self, member, path=""): # Prepare the link cpioget for makelink(). if cpioinfo.islnk(): -# cpioinfo._link_cpioget = os.path.join(path, cpioinfo.linkname) cpioinfo._link_path = path try: @@ -1492,7 +1489,12 @@ def extractfile(self, member): else: cpioinfo = self.getmember(member) - if cpioinfo.issym(): + if cpioinfo.isreg(): + return self.fileobject(self, cpioinfo) + + elif cpioinfo.islnk(): + return self.fileobject(self, self._datamember(cpioinfo)) + elif cpioinfo.issym(): if isinstance(self.fileobj, _Stream): # A small but ugly workaround for the case that someone tries # to extract a symlink as a file-object from a non-seekable @@ -1502,10 +1504,6 @@ def extractfile(self, member): # A symlink's file object is its cpioget's file object. return self.extractfile(self._getmember(cpioinfo.linkname, cpioinfo)) - elif cpioinfo.islnk(): - return self.fileobject(self, self._datamember(cpioinfo)) - elif cpioinfo.isreg(): - return self.fileobject(self, cpioinfo) else: # If there's no data associated with the member (directory, chrdev, # blkdev, etc.), return None instead of a file object. From 889639a6078fa0eaab37811296735fa0a9d01b88 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 16:18:39 +0200 Subject: [PATCH 73/92] test_cpio: ensure paths are handled as text Caught by extended test. Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 0ddaff6c..50852c09 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1420,7 +1420,7 @@ def extractall(self, path=".", members=None): # Extract directory with a safe mode, so that # all files below can be extracted as well. try: - os.makedirs(os.path.join(path, cpioinfo.name), 0o777) + os.makedirs(os.path.join(path, six.ensure_text(cpioinfo.name)), 0o777) except EnvironmentError: pass directories.append(cpioinfo) @@ -1462,7 +1462,7 @@ def extract(self, member, path=""): cpioinfo._link_path = path try: - self._extract_member(cpioinfo, os.path.join(path, cpioinfo.name)) + self._extract_member(cpioinfo, os.path.join(path, six.ensure_text(cpioinfo.name))) except EnvironmentError as e: if self.errorlevel > 0: raise From c21cbd78766c5007728f1976b8909c4e044c9d63 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:40:49 +0200 Subject: [PATCH 74/92] Futurize pylint complements: whitespace in expressions Signed-off-by: Yann Dirson --- xcp/accessor.py | 2 +- xcp/net/ifrename/logic.py | 4 ++-- xcp/pci.py | 2 +- xcp/repository.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 4d092607..6d057927 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -120,7 +120,7 @@ def openAddress(self, address): return filehandle class MountingAccessor(FilesystemAccessor): - def __init__(self, mount_types, mount_source, mount_options = None): + def __init__(self, mount_types, mount_source, mount_options=None): ro = isinstance(mount_options, list) and 'ro' in mount_options super(MountingAccessor, self).__init__(None, ro) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index ce7fbcd1..e2a3484b 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -350,8 +350,8 @@ def rename_logic( static_rules, key=lambda x: x.order): LOG.info("Renaming brand new nic '%s'" % (nic,)) - if ( VALID_ETH_NAME.match(nic.kname) is not None and - nic.kname not in (x.tname for x in cur_state) ): + if (VALID_ETH_NAME.match(nic.kname) is not None and + nic.kname not in (x.tname for x in cur_state)): # User has been messing around with state files but not the udev # rules. If the eth name is still free, give it diff --git a/xcp/pci.py b/xcp/pci.py index cff6d506..1c8e081d 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -260,7 +260,7 @@ def __init__(self): stdout = subprocess.PIPE) for l in cmd.stdout: line = l.rstrip() - el = [x for x in line.replace('"','').split() if not x.startswith('-')] + el = [x for x in line.replace('"', '').split() if not x.startswith('-')] self.devs[el[0]] = {'id': el[0], 'class': el[1][:2], 'subclass': el[1][2:], diff --git a/xcp/repository.py b/xcp/repository.py index c084d1d8..ca284647 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -76,7 +76,7 @@ def __init__(self, repository, label, size, md5sum, fname, kernel, options): self.filename, self.kernel, self.options - ) = ( repository, label, size, md5sum, fname, kernel, options ) + ) = (repository, label, size, md5sum, fname, kernel, options) def __repr__(self): return "" % (self.label, self.kernel) @@ -90,7 +90,7 @@ def __init__(self, repository, label, size, md5sum, fname, root): self.md5sum, self.filename, self.destination - ) = ( repository, label, size, md5sum, fname, root ) + ) = (repository, label, size, md5sum, fname, root) def __repr__(self): return "" % self.label @@ -103,7 +103,7 @@ def __init__(self, repository, label, size, md5sum, fname): self.size, self.md5sum, self.filename - ) = ( repository, label, size, md5sum, fname ) + ) = (repository, label, size, md5sum, fname) def __repr__(self): return "" % self.label From e1627c3af1f54541a85dc9b83a81a3ff4882d60e Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 10 Aug 2022 12:24:20 +0200 Subject: [PATCH 75/92] cpiofile: revert overeager tar->cpio substitution on "target" Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 99 +++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index d9c556e2..aeb35272 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1453,7 +1453,7 @@ def extract(self, member, path=""): else: cpioinfo = self.getmember(member) - # Prepare the link cpioget for makelink(). + # Prepare the link target for makelink(). if cpioinfo.islnk(): cpioinfo._link_path = path @@ -1477,7 +1477,7 @@ def extractfile(self, member): """Extract a member from the archive as a file object. `member' may be a filename or a CpioInfo object. If `member' is a regular file, a file-like object is returned. If `member' is a link, a file-like - object is constructed from the link's cpioget. If `member' is none of + object is constructed from the link's target. If `member' is none of the above, None is returned. The file-like object is read-only and provides the following methods: read(), readline(), readlines(), seek() and tell() @@ -1501,7 +1501,7 @@ def extractfile(self, member): # stream of cpio blocks. raise StreamError("cannot extract symlink as file object") else: - # A symlink's file object is its cpioget's file object. + # A symlink's file object is its target's file object. return self.extractfile(self._getmember(cpioinfo.linkname, cpioinfo)) else: @@ -1509,17 +1509,17 @@ def extractfile(self, member): # blkdev, etc.), return None instead of a file object. return None - def _extract_member(self, cpioinfo, cpiogetpath): + def _extract_member(self, cpioinfo, targetpath): """Extract the CpioInfo object cpioinfo to a physical - file called cpiogetpath. + file called targetpath. """ # Fetch the CpioInfo object for the given name # and build the destination pathname, replacing # forward slashes to platform specific separators. - cpiogetpath = os.path.normpath(cpiogetpath) + targetpath = os.path.normpath(targetpath) # Create all upper directories. - upperdirs = os.path.dirname(cpiogetpath) + upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): ti = CpioInfo() ti.name = upperdirs @@ -1538,39 +1538,39 @@ def _extract_member(self, cpioinfo, cpiogetpath): self._dbg(1, cpioinfo.name) if cpioinfo.isreg(): - self.makefile(cpioinfo, cpiogetpath) + self.makefile(cpioinfo, targetpath) elif cpioinfo.isdir(): - self.makedir(cpioinfo, cpiogetpath) + self.makedir(cpioinfo, targetpath) elif cpioinfo.isfifo(): - self.makefifo(cpioinfo, cpiogetpath) + self.makefifo(cpioinfo, targetpath) elif cpioinfo.ischr() or cpioinfo.isblk(): - self.makedev(cpioinfo, cpiogetpath) + self.makedev(cpioinfo, targetpath) elif cpioinfo.issym(): - self.makesymlink(cpioinfo, cpiogetpath) + self.makesymlink(cpioinfo, targetpath) else: - self.makefile(cpioinfo, cpiogetpath) + self.makefile(cpioinfo, targetpath) - self.chown(cpioinfo, cpiogetpath) + self.chown(cpioinfo, targetpath) if not cpioinfo.issym(): - self.chmod(cpioinfo, cpiogetpath) - self.utime(cpioinfo, cpiogetpath) + self.chmod(cpioinfo, targetpath) + self.utime(cpioinfo, targetpath) #-------------------------------------------------------------------------- # Below are the different file methods. They are called via # _extract_member() when extract() is called. They can be replaced in a # subclass to implement other functionality. - def makedir(self, cpioinfo, cpiogetpath): - """Make a directory called cpiogetpath. + def makedir(self, cpioinfo, targetpath): + """Make a directory called targetpath. """ try: - os.mkdir(cpiogetpath) + os.mkdir(targetpath) except EnvironmentError as e: if e.errno != errno.EEXIST: raise - def makefile(self, cpioinfo, cpiogetpath): - """Make a file called cpiogetpath. + def makefile(self, cpioinfo, targetpath): + """Make a file called targetpath. """ extractinfo = None if cpioinfo.nlink == 1: @@ -1579,7 +1579,8 @@ def makefile(self, cpioinfo, cpiogetpath): if cpioinfo.ino in self.inodes: # actual file exists, create link # FIXME handle platforms that don't support hardlinks - os.link(os.path.join(cpioinfo._link_path, six.ensure_text(self.inodes[cpioinfo.ino][0])), cpiogetpath) + os.link(os.path.join(cpioinfo._link_path, + six.ensure_text(self.inodes[cpioinfo.ino][0])), targetpath) else: extractinfo = self._datamember(cpioinfo) @@ -1589,21 +1590,21 @@ def makefile(self, cpioinfo, cpiogetpath): if extractinfo: source = self.extractfile(extractinfo) - cpioget = file(cpiogetpath, "wb") - copyfileobj(source, cpioget) + target = file(targetpath, "wb") + copyfileobj(source, target) source.close() - cpioget.close() + target.close() - def makefifo(self, cpioinfo, cpiogetpath): - """Make a fifo called cpiogetpath. + def makefifo(self, cpioinfo, targetpath): + """Make a fifo called targetpath. """ if hasattr(os, "mkfifo"): - os.mkfifo(cpiogetpath) + os.mkfifo(targetpath) else: raise ExtractError("fifo not supported by system") - def makedev(self, cpioinfo, cpiogetpath): - """Make a character or block device called cpiogetpath. + def makedev(self, cpioinfo, targetpath): + """Make a character or block device called targetpath. """ if not hasattr(os, "mknod") or not hasattr(os, "makedev"): raise ExtractError("special devices not supported by system") @@ -1614,25 +1615,25 @@ def makedev(self, cpioinfo, cpiogetpath): else: mode |= stat.S_IFCHR - os.mknod(cpiogetpath, mode, + os.mknod(targetpath, mode, os.makedev(cpioinfo.devmajor, cpioinfo.devminor)) - def makesymlink(self, cpioinfo, cpiogetpath): + def makesymlink(self, cpioinfo, targetpath): # FIXME handle platforms that don't support symlinks - os.symlink(cpioinfo.linkname, cpiogetpath) + os.symlink(cpioinfo.linkname, targetpath) - def makelink(self, cpioinfo, cpiogetpath): - """Make a (symbolic) link called cpiogetpath. If it cannot be created + def makelink(self, cpioinfo, targetpath): + """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. """ linkpath = cpioinfo.linkname try: if cpioinfo.issym(): - os.symlink(linkpath, cpiogetpath) + os.symlink(linkpath, targetpath) else: # See extract(). - os.link(cpioinfo._link_cpioget, cpiogetpath) + os.link(cpioinfo._link_target, targetpath) except AttributeError: if cpioinfo.issym(): linkpath = os.path.join(os.path.dirname(cpioinfo.name), @@ -1640,16 +1641,16 @@ def makelink(self, cpioinfo, cpiogetpath): linkpath = normpath(linkpath) try: - self._extract_member(self.getmember(linkpath), cpiogetpath) + self._extract_member(self.getmember(linkpath), targetpath) except (EnvironmentError, KeyError): linkpath = os.path.normpath(linkpath) try: - shutil.copy2(linkpath, cpiogetpath) + shutil.copy2(linkpath, targetpath) except EnvironmentError: raise IOError("link could not be created") - def chown(self, cpioinfo, cpiogetpath): - """Set owner of cpiogetpath according to cpioinfo. + def chown(self, cpioinfo, targetpath): + """Set owner of targetpath according to cpioinfo. """ if PWD and hasattr(os, "geteuid") and os.geteuid() == 0: # We have to be root to do so. @@ -1663,24 +1664,24 @@ def chown(self, cpioinfo, cpiogetpath): u = os.getuid() try: if cpioinfo.issym() and hasattr(os, "lchown"): - os.lchown(cpiogetpath, u, g) + os.lchown(targetpath, u, g) else: if sys.platform != "os2emx": - os.chown(cpiogetpath, u, g) + os.chown(targetpath, u, g) except EnvironmentError: raise ExtractError("could not change owner") - def chmod(self, cpioinfo, cpiogetpath): - """Set file permissions of cpiogetpath according to cpioinfo. + def chmod(self, cpioinfo, targetpath): + """Set file permissions of targetpath according to cpioinfo. """ if hasattr(os, 'chmod'): try: - os.chmod(cpiogetpath, cpioinfo.mode) + os.chmod(targetpath, cpioinfo.mode) except EnvironmentError: raise ExtractError("could not change mode") - def utime(self, cpioinfo, cpiogetpath): - """Set modification time of cpiogetpath according to cpioinfo. + def utime(self, cpioinfo, targetpath): + """Set modification time of targetpath according to cpioinfo. """ if not hasattr(os, 'utime'): return @@ -1689,7 +1690,7 @@ def utime(self, cpioinfo, cpiogetpath): # to use utime() on directories. return try: - os.utime(cpiogetpath, (cpioinfo.mtime, cpioinfo.mtime)) + os.utime(targetpath, (cpioinfo.mtime, cpioinfo.mtime)) except EnvironmentError: raise ExtractError("could not change modification time") From 28dafbf8df95839210a1c17ab51929a516fbfdc3 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 26 Jul 2022 17:16:30 +0200 Subject: [PATCH 76/92] cpiofile: migrate last "list.sort()" call still using a "cmp" argument This goes away in python3. Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 50852c09..fb4d96f7 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1428,7 +1428,7 @@ def extractall(self, path=".", members=None): self.extract(cpioinfo, path) # Reverse sort directories. - directories.sort(lambda a, b: cmp(a.name, b.name)) + directories.sort(key=lambda x: x.name) directories.reverse() # Set correct owner, mtime and filemode on directories. From 753b6ddfdd1e450658dbb6ff5ed6f535e194d7e2 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 17:00:36 +0200 Subject: [PATCH 77/92] xcp.cpiofile: fix bad indentation Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index d03a2ab7..3c5dcce2 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1112,8 +1112,7 @@ def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9): fileobj = file(name, mode + "b") try: - t = cls.cpioopen(name, mode, - gzip.GzipFile(name, mode, compresslevel, fileobj)) + t = cls.cpioopen(name, mode, gzip.GzipFile(name, mode, compresslevel, fileobj)) except IOError: raise ReadError("not a gzip file") t._extfileobj = False From f1236c48504eb36eaf1fd663d0acdff4abed5a30 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 31 Aug 2022 10:53:10 +0200 Subject: [PATCH 78/92] cpiofile: catch use of non-binary streams early Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index aeb35272..9eff7ac3 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -50,6 +50,7 @@ import time import struct import copy +import io import six @@ -959,6 +960,7 @@ def __init__(self, name=None, mode="r", fileobj=None): self.name = None if name: self.name = os.path.abspath(name) + assert not isinstance(fileobj, io.TextIOBase) self.fileobj = fileobj # Init datastructures From d43e44c3a80772978065375ea1f15636d011fdf4 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 25 Jul 2022 12:32:56 +0200 Subject: [PATCH 79/92] WIP python3: fix xmlunwrap and its test to align with the use of bytes FIXME: I'm quite unsure why xcp.xmlunwrap would want to use bytes and not unicode strings, but the encode/decode calls make it quite clear it wants to work with bytes. That makes the API painful to use in python3. --- tests/test_xmlunwrap.py | 12 ++++++------ xcp/xmlunwrap.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_xmlunwrap.py b/tests/test_xmlunwrap.py index f71d778f..23313331 100644 --- a/tests/test_xmlunwrap.py +++ b/tests/test_xmlunwrap.py @@ -18,18 +18,18 @@ def test(self): self.assertEqual([getText(el) for el in getElementsByTagName(self.top_el, ["fred"])], - ["text1", "text2"]) + [b"text1", b"text2"]) - x = getMapAttribute(self.top_el, ["mode"], [('test', 42), ('stuff', 77)]) + x = getMapAttribute(self.top_el, ["mode"], [(b'test', 42), (b'stuff', 77)]) self.assertEqual(x, 42) - x = getMapAttribute(self.top_el, ["made"], [('test', 42), ('stuff', 77)], - default='stuff') + x = getMapAttribute(self.top_el, ["made"], [(b'test', 42), (b'stuff', 77)], + default=b'stuff') self.assertEqual(x, 77) x = getStrAttribute(self.top_el, ["mode"]) - self.assertEqual(x, "test") + self.assertEqual(x, b"test") x = getStrAttribute(self.top_el, ["made"]) - self.assertEqual(x, "") + self.assertEqual(x, b"") x = getStrAttribute(self.top_el, ["made"], None) self.assertEqual(x, None) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 1487afab..2832b680 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -44,11 +44,11 @@ def getElementsByTagName(el, tags, mandatory = False): raise XmlUnwrapError("Missing mandatory element %s" % tags[0]) return matching -def getStrAttribute(el, attrs, default = '', mandatory = False): +def getStrAttribute(el, attrs, default=b'', mandatory=False): matching = [] for attr in attrs: val = el.getAttribute(attr).encode() - if val != '': + if val != b'': matching.append(val) if len(matching) == 0: if mandatory: From edc16d6d69c099cfda35cb7c07a095f37849dc7e Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:40:49 +0200 Subject: [PATCH 80/92] Futurize pylint complements: whitespace in expressions Signed-off-by: Yann Dirson --- xcp/environ.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/environ.py b/xcp/environ.py index c8901f6d..8223cca8 100644 --- a/xcp/environ.py +++ b/xcp/environ.py @@ -47,8 +47,8 @@ def readInventory(root = '/'): try: fh = open(os.path.join(root, 'etc/xensource-inventory')) - for line in ( x for x in ( y.strip() for y in fh ) - if not x.startswith('#') ): + for line in (x for x in (y.strip() for y in fh) + if not x.startswith('#')): vals = line.split('=', 1) From 76459c6bb75b41ed8f2d974438e62f530a7afbcd Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 17:07:40 +0200 Subject: [PATCH 81/92] xcp: get rid of useless blackslash line continuations Signed-off-by: Yann Dirson --- xcp/bootloader.py | 3 +-- xcp/cpiofile.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 819a417a..a1d19709 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -146,8 +146,7 @@ def readExtLinux(cls, src_file): title = l[1:].lstrip() elif keywrd == 'kernel' and len(els) > 1: kernel = els[1] - elif keywrd == 'append' and len(els) > 1 and \ - kernel == 'mboot.c32': + elif keywrd == 'append' and len(els) > 1 and kernel == 'mboot.c32': if 'tboot' in els[1]: # els[2] contains tboot args, hypervisor, # hypervisor args, kernel, diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 3c5dcce2..a3905786 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1310,8 +1310,7 @@ def list(self, verbose=True): print("%10s" % ("%d,%d" % (cpioinfo.devmajor, cpioinfo.devminor)), end=' ') else: print("%10d" % cpioinfo.size, end=' ') - print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(cpioinfo.mtime)[:6], end=' ') + print("%d-%02d-%02d %02d:%02d:%02d" % time.localtime(cpioinfo.mtime)[:6], end=' ') print(cpioinfo.name) From 72b8c1bfb4b9ab2e3a8bfeaa3b8c630d929c5c1a Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 31 Aug 2022 10:54:46 +0200 Subject: [PATCH 82/92] cpiofile: hack support for text IO to archive member Really hackish in case the code gets used. Caught/tested using non-included test_cpiofile.py derived from cpython's test_tarfile.py Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 9eff7ac3..7b2623fa 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -710,6 +710,8 @@ def read(self, size=None): self.position += len(buf) return buf + # FIXME no universal-newlines, a TextIOWrapper would help but hey + # we're not using cpio archives on non-unices, right ? def readline(self, size=-1): """Read one entire line from the file. If size is present and non-negative, return a string with at most that @@ -718,16 +720,16 @@ def readline(self, size=-1): if self.closed: raise ValueError("I/O operation on closed file") - if "\n" in self.buffer: - pos = self.buffer.find("\n") + 1 + if b"\n" in self.buffer: + pos = self.buffer.find(b"\n") + 1 else: buffers = [self.buffer] while True: buf = self.fileobj.read(self.blocksize) buffers.append(buf) - if not buf or "\n" in buf: - self.buffer = "".join(buffers) - pos = self.buffer.find("\n") + 1 + if not buf or b"\n" in buf: + self.buffer = b"".join(buffers) + pos = self.buffer.find(b"\n") + 1 if pos == 0: # no newline found. pos = len(self.buffer) @@ -739,7 +741,7 @@ def readline(self, size=-1): buf = self.buffer[:pos] self.buffer = self.buffer[pos:] self.position += len(buf) - return buf + return six.ensure_text(buf) def readlines(self): """Return a list with all remaining lines. @@ -1794,8 +1796,9 @@ def _getmember(self, name, cpioinfo=None): else: end = members.index(cpioinfo) + encoded_name = six.ensure_binary(name) for i in range(end - 1, -1, -1): - if name == members[i].name: + if encoded_name == members[i].name: return members[i] def _load(self): From 7d0a2ba8bcce58106f5765b0c5bca8609e26de2f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 26 Jul 2022 16:32:02 +0200 Subject: [PATCH 83/92] xcp.repository: switch from md5 to hashlib.md5 hashlib came with python 2.5, and old md5 module disappears in 3.0 Signed-off-by: Yann Dirson --- xcp/repository.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/repository.py b/xcp/repository.py index ca284647..fdaefa5e 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -23,7 +23,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import md5 +from hashlib import md5 import os.path import xml.dom.minidom import ConfigParser @@ -246,7 +246,7 @@ def findRepositories(cls, access): def __init__(self, access, base, is_group = False): BaseRepository.__init__(self, access, base) self.is_group = is_group - self._md5 = md5.new() + self._md5 = md5() self.requires = [] self.packages = [] @@ -288,7 +288,7 @@ def _parse_repofile(self, repofile): repofile.close() # update md5sum for repo - self._md5.update(repofile_contents) + self._md5.update(repofile_contents.encode()) # build xml doc object try: From 4ed924abbc81a2680c64277e13518f15b3fb66a6 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 26 Jul 2022 16:32:18 +0200 Subject: [PATCH 84/92] WIP xcp.repository: switch from ConfigParser to configparser This is supposed to be just a module renaming to conform to PEP8, see https://docs.python.org/3/whatsnew/3.0.html#library-changes The SafeConfigParser class has been renamed to ConfigParser in Python 3.2, and backported as addon package. The `readfp` method now triggers a deprecation warning to replace it with `read_file`. FIXME: With python3 some Accessor implementations (e.g. FileAccessor) provide a text stream for repository config (and with python2 all implementations), while others (e.g. HTTPAccessor) provide a binary stream. But on python3 ConfigParser will bomb out if given a binary stream, so use a TextIOWrapper to access the config. This is a hack, which cannot be used when it is binary data which has to be read (see later commits), so I don't consider this commit to be correct in that respect. --- requirements-dev.txt | 3 +++ tests/test_accessor.py | 2 +- tests/test_repository.py | 2 +- xcp/repository.py | 19 ++++++++++++++----- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b2799686..84da09b5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,3 +8,6 @@ pytest-cov # dependencies also in setup.py until they can be used six future + +# python-2.7 only +configparser ; python_version < "3.0" diff --git a/tests/test_accessor.py b/tests/test_accessor.py index ade787e6..8c892845 100644 --- a/tests/test_accessor.py +++ b/tests/test_accessor.py @@ -4,7 +4,7 @@ class TestAccessor(unittest.TestCase): def test_http(self): - raise unittest.SkipTest("comment out if you really mean it") + #raise unittest.SkipTest("comment out if you really mean it") a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) a.start() self.assertTrue(a.access('.treeinfo')) diff --git a/tests/test_repository.py b/tests/test_repository.py index 833627d0..4768740d 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -6,7 +6,7 @@ class TestRepository(unittest.TestCase): def test_http(self): - raise unittest.SkipTest("comment out if you really mean it") + #raise unittest.SkipTest("comment out if you really mean it") a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) repo_ver = repository.BaseRepository.getRepoVer(a) self.assertEqual(repo_ver, Version([3, 2, 1])) diff --git a/xcp/repository.py b/xcp/repository.py index fdaefa5e..35a2c08d 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -24,9 +24,11 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from hashlib import md5 +import io import os.path import xml.dom.minidom -import ConfigParser +import configparser +import sys import six @@ -179,10 +181,17 @@ def _getVersion(cls, access, category): access.start() try: - treeinfofp = access.openAddress(cls.TREEINFO_FILENAME) - treeinfo = ConfigParser.SafeConfigParser() - treeinfo.readfp(treeinfofp) - treeinfofp.close() + rawtreeinfofp = access.openAddress(cls.TREEINFO_FILENAME) + if sys.version_info < (3, 0) or isinstance(rawtreeinfofp, io.TextIOBase): + # e.g. with FileAccessor + treeinfofp = rawtreeinfofp + else: + # e.g. with HTTPAccessor + treeinfofp = io.TextIOWrapper(rawtreeinfofp, encoding='utf-8') + treeinfo = configparser.ConfigParser() + treeinfo.read_file(treeinfofp) + treeinfofp = None + rawtreeinfofp.close() if treeinfo.has_section('system-v1'): ver_str = treeinfo.get('system-v1', category_map[category]) else: From ff2c1b0a37a940fc2bc32d7f8a316298641f212f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 26 Sep 2022 14:18:59 +0200 Subject: [PATCH 85/92] test: use parametrized tests Testing several accessor classes causes code duplication, which can be avoided with help from the `parametrized` package (unfortunately, `pytest` support cannot be used together with `unittest`). Not a big deal right now, but starts becoming painful when adding new tests or testing other Accessor classes. Signed-off-by: Yann Dirson --- requirements-dev.txt | 1 + tests/test_accessor.py | 16 +++++----------- tests/test_repository.py | 17 +++++------------ 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 84da09b5..4debd01d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,7 @@ diff_cover mock pytest pytest-cov +parameterized # dependencies also in setup.py until they can be used six future diff --git a/tests/test_accessor.py b/tests/test_accessor.py index 8c892845..e7e87a3e 100644 --- a/tests/test_accessor.py +++ b/tests/test_accessor.py @@ -1,19 +1,13 @@ import unittest +from parameterized import parameterized_class import xcp.accessor +@parameterized_class([{"url": "file://tests/data/repo/"}, + {"url": "https://updates.xcp-ng.org/netinstall/8.2.1"}]) class TestAccessor(unittest.TestCase): - def test_http(self): - #raise unittest.SkipTest("comment out if you really mean it") - a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) - a.start() - self.assertTrue(a.access('.treeinfo')) - self.assertFalse(a.access('no_such_file')) - self.assertEqual(a.lastError, 404) - a.finish() - - def test_file(self): - a = xcp.accessor.createAccessor("file://tests/data/repo/", True) + def test_access(self): + a = xcp.accessor.createAccessor(self.url, True) a.start() self.assertTrue(a.access('.treeinfo')) self.assertFalse(a.access('no_such_file')) diff --git a/tests/test_repository.py b/tests/test_repository.py index 4768740d..8081c33a 100644 --- a/tests/test_repository.py +++ b/tests/test_repository.py @@ -1,22 +1,15 @@ import unittest +from parameterized import parameterized_class import xcp.accessor from xcp import repository from xcp.version import Version +@parameterized_class([{"url": "file://tests/data/repo/"}, + {"url": "https://updates.xcp-ng.org/netinstall/8.2.1"}]) class TestRepository(unittest.TestCase): - def test_http(self): - #raise unittest.SkipTest("comment out if you really mean it") - a = xcp.accessor.createAccessor("https://updates.xcp-ng.org/netinstall/8.2.1", True) - repo_ver = repository.BaseRepository.getRepoVer(a) - self.assertEqual(repo_ver, Version([3, 2, 1])) - product_ver = repository.BaseRepository.getProductVersion(a) - self.assertEqual(product_ver, Version([8, 2, 1])) - repos = repository.BaseRepository.findRepositories(a) - self.assertEqual(len(repos), 1) - - def test_file(self): - a = xcp.accessor.createAccessor("file://tests/data/repo/", True) + def test_basicinfo(self): + a = xcp.accessor.createAccessor(self.url, True) repo_ver = repository.BaseRepository.getRepoVer(a) self.assertEqual(repo_ver, Version([3, 2, 1])) product_ver = repository.BaseRepository.getProductVersion(a) From 961bb79f4046b5173d48aabac41500916f3a36e7 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 26 Sep 2022 14:19:10 +0200 Subject: [PATCH 86/92] WIP test_accessor: check for I/O on binary files This test uses the same kind of I/O (file copy) that prepare_host_upgrade.py does. FIXME: the copy cannot proceed this way in python3 --- tests/data/repo/boot/isolinux/mboot.c32 | Bin 0 -> 33628 bytes tests/test_accessor.py | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/data/repo/boot/isolinux/mboot.c32 diff --git a/tests/data/repo/boot/isolinux/mboot.c32 b/tests/data/repo/boot/isolinux/mboot.c32 new file mode 100644 index 0000000000000000000000000000000000000000..3f5b2fe5acf672f54af1000ae844076807abb0d1 GIT binary patch literal 33628 zcmbTf4SZC^xj%kRc9U$v!dW0d&?u`cl{J8gP@5%Dm&hiOW+A+6z;_W6c?l4cJxSoQ zka!Zza5z>AZEb6B?XA5Rue8N0LQtwnz-*w12_Vpx+DN&2mW>iXA#g4G|2}h0Lhz;c ze}DOW&Yqb$^YYB|JoC&m&pb0}x?1?&Enn&a{vN8^tJMjD_PSTZ@8%`zVzJ-e(j1Hz zQykHY+Ut}>dus}Q#gR&z)pNDtQf9OE7SF7{nr;)^ms=un7hctZF01y|V$`ua37OfG zUuF-fic^P7yYL%z^AaS_VO{!7w+cd$_Igh&7SrCMnmt!56Ju0w^os=#>d8onZEnQx z=Xe6_zvmavRl1$}OBO+>+xu}Xsy(J{eh%L>Wh;L5X4I$US(&d}Rnm~ae7gEp1_CNl zG8!%hSMEkp^s4)_Sd8`PT_XhHHL;;tj#t_&nfsalw6?j&_@s>zgob9-e_BpfeR`v< zEm4f#&NEy#GL-QQa;DJ|GBy^7szQymc|}q2bz7tCP2QSv`fMzwv*Cge`BR)Agnc~| zCr%X9P+h#N@{tF3RgPW!5?5 zIy@4&{HNJ8o8midM}8$s2wDkxLK!e1)&C527RBVFdMCh8(SCuIweqiew_6Y%d;oB% znsPHWdI(vf6>8uF>(#%Hs`2TEna#@n*!kaG>J$$tzLVC73~B@S#ekQ!p*~h_wL5R#njdhD0EcI#q=9Wx{SN`u64m{HU*w$1m!CoC<87K#^SgV_=QGx~Edz zccE^o&TB)@Bz-hhnCC#)q6`7XbdXM}-;Tls;6WB{`NRtN%Z4FWo2D#64dB5!XxBmY zQ+Li$z;_Zjlx>>wG;&1mR6||h415sO{1p8S00iQz^BroweVMphKw14we0&gP25ZVs zQHCQkx}GIi@*OM>fON4uTR6{7b;JNw*sioWfL@PJU;C=6P*)Y3P4#!Pf)v$%k`<)t zO$n&hA>rpWG+;1wEqEbbpF;7?R0hP-@5i9bY}ONy-9-99Ld;G7h{c)!si2j7{?Mb3 zJ+fC#bM2k0Ujz~B_U44l%z8cQdT4LoNOtJQ`1dq?2VK~4)~*I@y9K8pTx=DC397mT zH8Ov9=Ed4madHy7dg(*0D0a&+&3+H_>&)My`MZn2JVkF()2c#JNDqF;WXcgSS=W@) zkS;~~2+-hxM>CtF3zu0`zpe(lH6!TO7=+iPEoM&1Af%xLd+^oVhAmA420A()VP7{CqDlq=|+z6GU}12NWd z4Yg_dzwoMB{sl!b`U85{cN!%T>8{I)+y2`d?()@Hb56?>-p;`Hi>+39 zIP0wY2m)SEE{mftDj#<%ZB8|G8d%Tv>(Yi(kZO9qvy_)Q2V;P)Qcn@QP}niNs~uC&WL9*L8y z(zAVf<#&)S?JByxRJk&yYBau0C`?M~MYe!xem0^Xy<}H7|wcpP3Hk@A9-% zS!fXFr>ey^Pe%ntoi28I+A9ZHe5c(Vs{aI==Hz^N+T`N2u@EA`G_Y_?hR81hI*asK z7@x{MduDTF9iTHg8DBZye|!sP+29~e*?`=UOk`2kIfUSCgs+*_nB1iXkeTy$_ZZIh zlzBU4wo-3>T@Vme$gZI@<-cQ*-pg#L_0H3XxywBf2%I#uX%y%;K zSFoR2?F3F6E=u+9Bfof6+BH7|IVMTJ?^tHxf((P;?Ehd8fI|RDgsNUcRdaEpBs*EI z3k5P*ZW@(ixlX+9WVz!_`AFXYa%GlUoYHVO=)Ra|i<8HyGtq31CC{-;)SpM~G>%bw z{CU)#hT2)4orGfB_~<(TFESZPfU)5p5MhyvM7~hBIzO$S@T^x|?_i-GwZ_gu5dA6O z!bH}VF84(3s@#RQMb^{h)4W(}W(*>QJRvwjcsXDw==$E0b&Ekx5Hpefpol{+FX zLK0@ZA_u3XNywX=25m4Qj0Mcm<=|4HVYENyh*(GOj;w;BFc25cop@SLMzJsot0%NcFBSW3f4! zaw}+4q#weR^5|ocf5G?b#2nNzo1TQCb$e%31H)km=C8`-T6F7Cs8p}hfRb+mT9hmI zR*kU)x|K6dlspFo@41OWtsvYj)Fug4 zw@SO>hgebkIiWTd3;UiEgk+(#?n-P-b)r$g+9UuE`W8%u%=ZGR6Ca@=E%0Jy^X@^Y zpB=yOYS`E8)JfTEDw^h>8}{viW+bpn?0m}^Ybgu7#Lg?HBm`c5EuJ!OLGwapO!Kuf zUn6Q#{f$jf-~@da^qq^o9#MX*F4Rp`$;FVf!>fw2Lt83FDJ!X(~5EX1=W1%gP zn+Rv}3*iJR9L@tG=w+i9@YK?kjNuc`BU~am1(}6X^i|Y>V!&+xO&S|X^)69fQ%V|I zuu=`Z5YA8Kb>>4gcL?EpC)EKm=u~OGtEG1g%O4-kpFnj&)c{f;lmrNUslE|`aDIkS zd&Ds9?$`LAWK>ZOlz*<;` zC$Oo)>+x->j`#Q*tL{?Ypazt6m{o7gZlFx^t=SsEB;TO=8kw&?avvh9T&wg7)rp>S zl@{i&jd2ha-ksD)0c8Ni^<+~yCdINo+gDpkWGh$V+Z z|LZH90U;kuVCyC*L%i1yT%~)l{@xcfYKK=63-4fo?H9dPt5jbD?o(=Rz`tAhXyMtH=CI)b+?XD$az2(9$YrF6p)&?!a(b8p=b`=j1 zJD6{)>f51uUEnjKJo<-8zL}?ChT3y2Z4cUr7r@QT_p%z=V+l30&`W>{;sgQ$wbK~- z9cfit<5b^PC`wnM2SHz{M48fYF_E>Q1KW0cku$Sd4ZVEfRqCWTtCZ1P^0L`eP>cEr z%=C42HN+k7s{ArQC{=3-h|+OC7fSE`xgbA>wA!W{kR(J6(l$_MB z)b&2TP+jp~sp|_&i~V)!JYv@M3##iFQgR&f^y}c-Xx6oxPpRk@!8J%~c%{Ftc&XvV zYjp)5jyy~CNK(TFqex#LgZC3mPZ^0uJ;_qTLSB^mhkEAmhyhbWJ|fW(O-=w19p02m zFFR+nXnYg&H7u|9NxPmOhLX}QMhd7BY64oG0wN98l6LK)ui0|*TD!FC9mOVnBwy#P0!G{P$YVgH?kW^V>v$$EM zYUp>Rb?*z_QfNS$+|JIi^Tt?8#fUP;(jSps^=+@S379NOSxr&j`dYw9G6zwwEm$KYBs;>jDc~Cv^DyEX72*fT) zC{>>!B!d5S1PMI|o`y~5eABG}OkaiiqUi=0bzvF?YG9xCUlicO3ZvWCr24j@)bAp} zFTbweEYRIhf$H|IS_w%czsM*0%x0200H9fztA*N0=h{9(x1uSUvZaOyVX!XLZkF zI*QN&gi2N4F$5pvi9nR{FhXcaxK{5H)XUWgn*moag&^T*;_5~LzDK`nv|NbNeKe$* zS5qiOzXwZBLqTV`)1g}j6Y7!$2+lsCUR#rxAZ=-;RX$e>&%-`Yw>N7ApjW7x+8WoL z@(j)2s|7x90{R903&WIlm)V#a=v956>&H;8ubnJvXQb-uWw}$rxpv44ryA!9DeZAA zPJQId{a8AyyqV52YQ~9LVWZJk4*=C`LW&l#Ook;i(O z5|3)QF@v%f>ttO5{VD&u>=?7VpzB`@-loB=dK_>EsV%3d3e-bEMPyL$uS%~q zcp_>?>o)OSG3;)O9$>u>@@fIY2Qb~$?Y)ue>`>>q?03gvXJge_^@oE~-X<->-R3z| z6~_)~_5^J+S7+y>c{-~`c{-{S+@0(Y>rk$asj{(htL}m5$!vb;(Y>TFJqG}&k20_S zt>u8Md5qH4qi%+6K~Sc~1y`sQR!>{SLao5*IaG1CW{*?X*uo+xEV$d)Q`Rt%21YO_ zj&`=%#%9^s44aw@te9Qj9cp20l8pTnE-ja~#oUVR^A+ zHt$8Z_YznieoEbT0W3|~gl2%NW(nZ2Q|?wxt*4Ys;MT^Dcb@fJg=OVf=VvTa^=Q>z z&-u!Dt)iJ_rVnLBR{cKkj-ePRHEBX{aOPo6`EWB9GS7h;Msuu9Z1EflI`VR*op~{7 z=bD(iH~1OZT>#BoeGZ4^g=MIx&}9b^Yz9}772qL!zmtEj!S^>gw>CpSHZ<3a+D$QT zc+!+Cu!epNAEFkeJ!bG1Er~96Q$KT17A%wEf)Z+H9r{C>J+vnMV+0ZLf>5DP-j~$fzMgsb5Co<6T}Gz zD(3SV?TIEUv@%rY`y68u3kZhx@H+zxVfXZC3fNYtN>Pha*lchFXNPHE07rBf)~Hxy zfJ+cl`4^@~r>HqD*k&FEo>(ge#I4F_4tCjd2@HM?0H?UvXBS>(x=tccMXZBpr0lcx zjX|YbWbbC~-#C4+&}el@JA0*_QZnZk!QowAJqqTD;K6yPE zm$b84tyfN>uy`WYrTBGgq#MZy*dl*70*=U^jes-KX#~VbI|5pH2Rm1LIjP#QTt1qt zO;3Psd^GYqe95_8PE)aLK8oE8$R+kOq+NRCC@PJMJWJmvNe#>K=C)*Xn)(-!kuac2 ze+Q~0b-N?-5EZ#yYRIA-`BwFNOzvP|SZ~-gq?kyHe3x>S1@B#N&sh@8%vlz^C8sbt zEGH<{52i0`r22SzKP8WY88a?IwrpUShI8Z!WPZ$lI&z-gzSEI2c*AO@T!{l!DF>s- z)eshgr!k6SIV+|5D^@{RKP6|8RDTtk&ib2jih_^kERY($kFs@sy%we3riDjjFMe_s z2bbk64VFeG;%iQ&RDY7n49-~<9Gg=foERBN$+A@c!@g2EHBvn(w~;T=%-c|{gy=f6 zR$$SoCT1SS1_BFo0mHl*V1aT)evKdG@G7v;1}Cn!;T;R6LHQU}!riSP87q2k=YZZ>B3dc%2slr`UA=l_o zD)~AXqeHH*IqW-#s-fAmdwkuM3F%~&IS4&xTrE%Yl5aY@n*i4!oD3; z1lp80I7$h139E-*dgP=M=mKkX;Ugp|h;~x-cd-t304p)d&E#fwuI2I==HJIUlUjnq zV)DM2+^pXXv_9}a%(o|sl@yq)8Eq?FPAu+%BbBQ*d8q2!gTdCU20A<)Aqxv2L$k-X zXYFV*dr(W*Rtt3TIunqW*U-#-+X0>}C_(aEbwC|Lg5ukby@zJjUhvaPq3t3pJpx5oU< zs{f$JzqK+^m3LrM@G+FVA0_EKmd<7t-$9RWYvnlRZ;Z(=#N;goJ@gqER}Vg*$}erS zCaC@$jnlB~X#8p>FNX93j~ZN#)ic%YZ;UL*z{aW?c5vmRxQ6}mAklY1guwg@$gKum zwgj5ot=WM+>-22j!OCP+ez7gVnaAW8z0t#JVx#W@#t`4)(E=BuH)+*Ybrftj(BGON zTaB6MIPHht&n{*AJJuyBp^jJ(oYCR&A6=IVdcG9rOMqKQxdX$?UlUHNGjxw zo<@VxvvKGZH7eEja&A6q!JzkbyZzmKOrAs!eJ6;+__RU0wAl8L|Bi+t2T*`ZgB%EB zQy|<13FAMFoulAn<#Jrmqb6?jcQ@LA6Pnz((2Q)MhDO0)UdpCMheIAquB4^5vnx`6WeJT!_j#3 zIP);pf8d44&wGjfTprnY--W1zIH~YzXj9wt#25yB_rPg=u5SN3P3dzU$GjA>EfZqZ z*4X#B0|WMJu)X&Xv=To5E8VHcKpO*rZc`9qckXj_7IfXDa4_j@0w*&TwY>*rKJ##N zy%PFdsKMS;j~e(~^z{tI@5Q09DE9NZkdr1Zda>T5p5Q&4HlJ6gRRC<=8?cV*KeX;p9@<_j~3pcpN(-j&hn?NlLX$&Geuvn|T0FIfQYEQ?WGUGGFutG3pa z$fn+}t+9FzRgTcs*jR-%I=J>8L8}twc=kPTDV9rT!_b|;`(Z14ivS^Y-PBWQc@yiR zPWu4v(w+%kL9o0|W+A0_%=$!iofXXtHI|5BfqY}IVgUd3#O9Nt%a=#*@bdB0U08>) z-9FY1GX?dLI}Nx8absgV6}+48y6bV2f$JyaRbWE~$PW3#?Qyz}@U<_WVIIdkE&Zq& z#JF^V8kBYpuE$`Q#tLlGt`Rd3WJPx6@}TvJVXuwDedRU^+l$aX)^M~KR=P9v8#s`U zKZ5+n*n8SMm$lC(g6$6w$sMOzCo!vw9nYTP3TD*alPnKMvU4!34Og0OL1p@%u@cDJ zm=|kA^0id5W<~$VbY))#*Zk2YaQf=s2&d;B2C`o#GeT)N&y{Q#jBcg6o<%8XSDw`{ z&!$3O%$tCCoDsKE9Gi~`M$|^pDOfAd)#vhhe~Nlb!^(ExSQIpM6W;8AVLAFPDmG0D zo3|molsVMRl(Cc*#YsCiZ>79NunBK|4xyG)2&pe1BzAI)s#_?0064w2J71vx^+W2- zShUJ~`=C>NWrL~(%M)przq{q5WUY?2p^UAmn@fLnZD(rtwSB1sX;*v8N7lNC1KQ&< z@lAR`mrz6dV#ndbA~k#g7`CVAKZat+!kFdAMDQWf!pcef@{KCxC_;R%O6fp|{=&)$ zgb3N(g0AdAjP-J3vqGv**teas@lC63s(+idnV8WqV8yWTF^phGk;UA*k{ZH@(EgRQ z;SUIrsobF!Sk-bHc2||lg7nOfC^t3|C82sMw~b?`$WY+ifu_;^J+O>cjI55&mYb?1 z@z~J!>)Oanq8yJ!t>l9M#~6o@x!<&^qa*+l`b$M5ts6~}w^V^}+I7Fyu zB%av0KD=S8QfeR|(Mzg?+u#T@lrC`%VTay8OQ_IxA()6g`R9-eJVY2Zw!;RXqX%7r zaHJiv91vD7g|M9r>VjCtE>g1!b@Ui*RTpCVq|d)?`3O7M_$-KhFhy@S>lYf4aTA$G zg~dqWnzMne5tY_>uhMIi&$Ri@M%2I_HP5N&6Wv%3^k}}OWfQ^OdJ`6W;0n1GdzRaa zmWj}I)c};yCVd;&;K2v_a1AtF{WBq8`T~pw2xdWF3Of*Y1-jRWKY%!DN`kEZBLr@A zd>Cs`^u*Y1CFd=Q3SqAmj243&rWorXdk7i$_Dl7bkzhz-yZ#(+?bv)|Y16Djf$jn_ zBPs94)Jdve@EoYZ#B8Mvr3Mqdq!3d(&QN2NQZikRod90XxH|u*oUE|I%nef#Y(pwowl+Js?F2`~+*Ho$6?2g6q7e&2JviZ*Dkz<6~G_Q+vsf*N!II>4I79Om!8AYSi@_JBJu z(r4=-o?|a^ur}yOprXJ&K@Yxbkl;@d&VP1{L46x=gJy2Vn0LTg08?2Xza?rlHnqwY z4I>-3kQo-NVtov+y@YB%PF{GMccKpcBIw1L{HA5tH-N(myQ1&J$iR%#qyGaCEO?Oh zeoZPV-C6qk$EiOfs6OZmCUu(Z;Ch|39`-KHSH!x|GVBAijMDIQS4I+q?sS*yDS_#H znwsu%XtEufGMLE$I5u?R5J?tn__YeQzU^QHjBV2dY?g8jq7Zw~7pJ?Xh{1T}P%N4S zh9-}*cxhKp6dovkt6@6)h0f*WNz-)5ZL5h82_!phEH3jR*nR~;-NXB;?91XjJ6Dgu z+p%u%9rNG>q|X7p9^$&*5uQ34scMlO{&>n%QBGCYShXorJ#F$3%Q{$YtuVdA3?Frd z(I;SGa~XbB*n`U*48~X3-C=HpQNESTzmwEl6pr&8l3fYCm5AD~_ja6}yYQa|{Mi5Dr90rk>45oS7H{=RDj4AgBa)PqXDgF97HBoEp>$U)OK?S2yX z7;-_k@up{@X;|E2e7&+U4qljd;9 zVrto|UqYm=<(+T%Y{08@FVF)=2wOi#9vJ*cn7yp}?9BJlH3brFU~ew$+eQJn%qmx6 z!J%Ys-_k1WJS+ybk$C@Qs#e*fhcth)=G$lbJNx$N>F{== z5*jRn`YA*J$p6rb0D=DqPynMwdHw8A({yrhMS zZA{+o_HS28wttCjcTcPI^q)|C8SIcR>UBUWpphD&cB5-eTnE~!e>TC;i?@uRVIvB5%_+3~b-y+d&499g(#hIHM1yj-R)@&93Lbl)~qcCs7qUkE*g%8%u^% zVl&8N8NyI$wNvFVWLqID6ImRrM5ryM=G#;uJrPoga;&$h`CQF80c##MilJBJTEn?q znbLAW<{dDw*fTGh_GGwFqd_z`2-=~3`gU{$)Fnh3DNr+TB7tLe}H0~Wc}5mJL2VdyO>#9}BacFq0CKm8??P6dz3`vW;V71f zejNjip|sdI&*H7@OXEs{;b_o&h=`X^2;_XDqXDgUNPGYP?Px&r!g5~eYv2F+ypW*Y z^UqCE`fnlH>RM!!Em+!2v!41qO+)H*M;hk%p}gk#RC7DGcByz0PgHPcaKqRf?R@= zG;e01AX#Yy{nub406LxJjaQ3Z(EaH+kabxWrV?zky$KWu`X-11i2G~m5bwiKP0{g{ zCBP4uIGFaEdi3=m0+yVtw_m=|V#@}^bAge810q)~)fu$9s=OUq&qn)H0W)B7GFCpID^&=}9aER2DIrLdu}WyXAm^!m z7w^=OF6M*lCHMf$5q3%PZl@Xg)EIxoEJ?g*w)8`XAq zn;M$JTHwy$%TRsenQsD?ST5C1d?N%ceSdl%LPT+xWciw!hFUSn)!;*FO&avJ zdF)uj;nfM3KFr)7b+FEs%hqYil{?nQD<4bP$j$3jdK0TN)cI3zNFvo!?ZTNythjyS zV*ytE-1|q;}cMTU5ORD!g-u5FcFMyf}D))<-hO&C8Gmu4={H1 zThPdBL;~3QWF$Vg-#{Oi7f_bc(BQqmo+Iov$l>MWh~B1LajgH2T9czqn5z84S$!kl zBWmnC^6|{EQ(Wxwk&nf4*zR(`VTPrHqpDBGovuICPUzX?15JG=U~4AUY5)&`Vl326 z$D*ncQ)=vjRKJ2kI7^kR!1r2ec$Biyq0m8cf+4uDAI1;Dbf{Q+LgC3`p_5oj!BWT1 zzhuEbLS3K3LOO$+sISZ1-PaWH3t-N-omY0V^h`WeEWqC+73t(}f)8qu4Z2mjDIhLEg;6m(GEY;h*q*2+SxP1x~}F04rh;tEeZCg#B5T zmx@XvM~1?NzT&i%c1Ayvlor_Ey5Tqu@k1e2;EFnMM9i+N&~f4_M78ZT;|yk3Zv)8oi})!Zb8)|12_Qb;mG_MN6zL<0{Rcx(U!{|g?mNsLo|Y@LJk=Gf?D z81*4?FZlHms_VkBL!>=VY10OAwor?l=s$%2=@!tUOC%K=2pbv~{Emv9=$=L#JcRST zt-fLSnt05jIN07N|U&j125@=6Jykp$2= zX=f){O!A%N!$;7I0Ie(21(v4;ZJqe` zVY0*_lYL5PpAdYQD&pL88_ZZ(<2d0*>TXx(q{3UCy)TBj;k6T*a}U-B@<}WYOi+hI z_486LIQZy4!6e3!w9oB>On?(zw>uxrbt^SqNU!IIa5fElz37Ax7RbhK)ME>-F>tmL zSu!u?aoI9GBi2ZZ^?m)o-K?&mNo|zs^=9WfD+(a{Iv0hIZtFuM9- z9RGiXG4KsAKKK>h?!29z(zpW_9l@;nTBcBX(~6S5d1d?K`s?dIT%iaG!i2J2CIib)1N`@`BqQ> zGl>mIhbt|oQn@q`Te*-JC%{TUVIF|67ds%bh`mB5Sbj!00q}l(7}B%}t3GB$cp{eL zK(*ON^q&L$XU74jQtS=Hp;C zcS%Y-vf{K-qW6Y{8>r*2U@d8M{8z~O&pIB5j?+X2#PXh5;Q@k(g&g+^>0tJp0<2RK z0GtqgSW{A9Y~Y|3hV$ov}~5>XT?4K>4?Jh$Px+X;%m2`giH5m8X?h ztCLvkSCE+CQ&!{+3mD)W#OvjhR3Aj$oT*74<4oO(Q zyi3p@2s_O8yj$g5)_Rw~K0h0+#`WPg2#T7x4MDM*&lRgvBFF;8D*6G5I;cwf@8vR?$ zaSE^i>2!{5t zUN6la)0UL7;Og?oOemPB%^^R5?T(r_PpH4jKLI^v`vFur=X%l{W9N)2X-bjmkC}J~ z;gs3*nw6YP7Nyk!BE^AM;v%mi1KJtN+Jf;~O=7fCtFa7dtn6B2el#}wD~*xxI(OtO zoY}8{{8VoQv-oL9Gc0*~6x|ZN-#{9ivc4T?v{!NBx~kSIt%F)3i36*G&#;cujsAUN zCl0e!jnb5K5WZiD4htkYqBqTe1zxIm5KBTunYBcFN@Uf*>GWCU(Z7S&lzVQH>bGD> zlqzqRllnr3|9!+lK@tmK>Ry8x^UGL%pJ-#fbz$N~t-MSvcw8$gpRY8}D=Ow}qkoB! zJdhukQDtKYHu&)IWpX%o0aq?&ELL+L*XHGx&j;rz%?nTiL7=}mum+e zCGkj;Yo9EY>i+~8!jrH_B4H9=wYDyFlte^TCCxPc4(z|d1(+2%NFZh4!We9$qw82+ zu$ftEKxf4{DvlL|^~sQKbb$e`F6dATa23T-)xqEzO)Iu8H7^TBYBethh*$2xVaZ3~ zph^wb_O0_Hg#%Lnji8nnGI`&%5mkVl20otlVCz7$Z`deqoGnVbR%1}X8(GKcC38?& z(W!44RPSJqsA_AET%p-BOiPWHOIsMyE^<~fss|V1zKI7H(9ydmWMHm-_$PAqN+vnIJNc!R?=rKx}H zu=dCr3@>z?)l3Kb!N|nQ3&H}9gncv0Ymxh(2-dHoeEM}Lr%_7QHYj;?smYzZ?RIw$ z(V}>|r?qM({KAVTRHZ3zlgUgVXb)Y3wkLAXoRSJMJO@I>(zGI0{_^UjKok;05=5uO-kC^L2PP7C=CDG|r5{x8y7k zq40&zk73dZbH}}iil7oYiitv@#yQtOk!ehf`HvcRJZSki5Y!QQ0vUM)HstL9^aB;K ze3bq6Vf6uAv(w4%{Dg=DfjF05kOn3GJ2)}nJ^2>C2Nbh+4uNYN)8cd~Y z!g9%s0jTJr8rcO4cPq4VtkD0dPM)2Y8oWtUy8BTZPN1oQA9`qy{1pCdM(}!-4}gUQ zB8%C};DZEPa31gHEi{P*7kzsl|AbNyuh|7@!4v?l%G+5*sxc)+ZbOUT&=#%jqDioT z!GSbJ*GCZf(K;r=4pw7V{YU9=9IiNn6u*9{fD3pEa+DfawjD~==<117CeWT;pbFtFK)kGN}n6lc6!w8qKUhrC`<)}$;hPXUP`cdnQ zx>L1iRqRkM4}lvbQisd&Z<4)Oq!TKvXhPCe_>VxH3pHEi5l!$51w$UhnG~d%2Ft!3 z1GscSNKP_Lp|`0sU0@@pT13|3c$iybooaC!z59)>{U*;x`KC{?-taovI{%n~SuYIJ zGO6xABAhHqYxvYSN5zgQAz&dv2?@#aTKa{}em-no`Y%4=jy-q4bEqGq(Z|en+IJvS z$TodmPeUl%ml{0K#OZnu?H0g@9hg!zocoN4S8@N#%W7aejQP=F>2?>)`o~ceX(;cX z5cb)L5BhAieBTefzQO4Aa2Oopf`eI+)3nVGG_}+22h7q^{Wj`Tc^XNfUm-|~LzryW zOSC*t+LTh%KVgY0?m@3(Qv~SZs){D#|d%$`y7~BX^$f~6h~vG4wmV})S4Ut{3)MgoPwY=AQUhg1HaoIC&$G=g0~-t<01|ef z!nCb~-DP;IW8c}z%F}?(>$VoI;Spvk+Cp%IYiahO&TvGVXvCLH1JSn^s~c%Tg@3Vn zCzdmEyqfEc!kbaKnkpx$Pm@ZKp6eCcAKlvogv9vJs_Wi_3iWZ@X?!i82qMhGem%B4 zGY?~-H?L3-asjI2GAvTC>RVd|G;Bq$N?FRx-01tD$=!5L;|A0{fheZE{wB7IuL?ixA$9^SBLD*Y5Fl>&V)ZuiVI1ny9$w^fa&ao*B#C?u~$y$$` zLWdmK3l1ZjcX{iVcuiNaZHy&^*b6MALal=HiZ=aekzGzNbfs>b=eoPGD27)~qoWk# z{7zh8BC-SN$O{*d>6nMCtTTE-^BcEZ9_ea;_qw%Hk6r=|2jf!8u_-QW?OI)!=M8t; z(pX_xp*)6jT@os`Mn(hGMmN22Xdc63LM}dAtCJOUJ}z>Wmb6yaSl|$ycLpOE4&-QK zMXys(z^mDx$B0Xzp5pD`zH~=E(d>6w3V}*=1DPC}{myI*pvn}K;t3dqv)L8*fri8D z$8pBz$T=m#TeY9gBMdqZi3|q}F;Gm)85Vu5pT>hYjd6?wf?LaJ0Cfd*3m=l|FsiN5 z{`A1OwXtxj2v}0oO*~WjP?Cp9upN2gKniEXjz)mDWm-XOEat~{;7!5#{on%_GOmFr z7}YNYP%fnr#6i(q7v|KYw3zJ1&Nr5dbv2l z);-IDNTp*Ab^A!f_50x{-9`FcuadZ1J09%Z1_)B2N2pn|uzJ>}C;kr-;4c9GD{W>B zH*r7*zyiA+X9ikX4|oA&)ar-CXsk|mjVC7naJ1^1fL9J20q}CDzA1QlRXGcz5w_zT zyr8xf;+3JwMR-k7{qs1{*s&uYYgHrwrmOx1M^0%K6fU3>*VQ8WX~k;l zV;J<}v;y0aQ{s6L+71S@I+QT6oqg;)Q~|$fZOU^SM{b;nC1Dkoge^xP9^J@Re^?%Z z{R^uL==4TTVu)XCbx8HMLL5lD2njY^T-Vrvg)@jX#RGsVQ_z|S3Ruyc*_bVVz+wt% z#BIC<CuD$Nd5V+FGKRDq8vw9$@|aM*XE_7vU^Q~(7beV!}Jb4=RsD26s9 zStmwbPF0HX2V?m+NZU-yzsNBf`13fOu*%6qaDw}Q1qsM*lazPCb#8=!bFn>Ub>(hM zR;Rcchrs>&T;$*I(SspY83b@Nt44n!AOHS0u9Y5yh;*0tFLCHBd+1#v!P# z(S{M?^t8*LqOi?_3MwC>YTSsy(iU`iaVnUyG|m%6FNBUmK11L6h-9tN+X1TM8mN^w ze;b(WJTQ?0!rJ1h#%`P;QriXM#u;&q1=yFiFpTe3%f@6J11@vfEYV~5PMHO%-U8g9 z0CMxvK^uUNe1`rgR#)7{IdP3eago0v3S_g+bP!|XG_f&P+&C?+F*i=Du&^mEZJLEa zI&7R~Y0S0o1`I%`^C|3n^gXjF(u_Awvo_`u2SUjnq~UTKv{s9BEjJ#)jrYel&WMjz zTx|8qH}lB^R){XvIs;?IE*=AFz=qo+K@x+TxgCPs2a8g}KEpOy`0WKT+_pzHVM?z( zL{w<9B}ty?i@TE#DGYts z#=xN{V!s7%Z9U1_R5lez)i7zTv19&nDdBirjIzdx@OW((Y0a?LRya@ymcK0K&r$Xj zaZv~VnIas@s@wbHvD_aF$4Tt^T^*2G#&HsO*l1-qW>a9r&6-$$;;7y|9!K?vS)Q{M z_RPcg!w>q>pKoc6CVP%U!{~_~!+jJT=6M+S|Nn-n&PLS`p#4YA9^A!`pE!7ju+fC! ztNUO-hOZZn11+)PuqkNLFN%w)q?!pgX-`L0I(ygs0X$;e5p|?^hJh_w(Q*A*OewhE zk`6Si#4^UNt;DsT$mY3RmFB*TqT&@bLG0B_ujN}u`Iz44Ei+dy;J`k6E0kWi-2M1~ z;cI&xl)r?Tmmgd1bFZn62MbXYlOCMI$+{v$im-0L zK3|w3f*yts6O6IQ@Hbo#Pf55IUI{&ilLl(&IUEniAq6#2{|}l@5;SZy!gq2CHpiSj$HXFXDtZT-~v)g zR`(T)=4BG2|5ghQmUS_pA<)6*+trb<8pDC9g9W-&bjWLuxIkOhrnJGy z0$z4Vh1)?#Gf^G>`e#8T;w|*M=))|O#{GhlDo4AE?(ye94u~qn-)Yn zvCW}{PGh1Q0aoH}`dH(3P>w45g6Jn2_U~YlmUUW)KY|%jP4MJdWr<$^t)sHf;K>>N zn@0CRH^MW3Zuw7DZJq);&B7rg?k8y&X z8?d(NzkwYH@C4xWr4ED7egGa4I7e%PJ$KL_k?inX(H7}C=Qo3@F^b zirz=V>6MBr4mZ;P5iH|G)hF2b`9qpIQG`B2z~|7ZXu)Hn<>c7}W-??t*J`$tr^ z!tUv;wvzJBI>}oFyQzrWeBG7EsInaB$3c*wBa%lT-o@l@+%-}VJx=CO!Ufs-(&45A zIG}LO5Sjmkrv(?ak^!+jKRtAU?y}f|1B@2xO;38L-4n*XH8`Lhc?dlO`wsoRcX37p zIl&f{!{`ze+|7DIe;GYeA+q)nKb!!s7E%JPrxBHAV6T}lP;NDlnjJb?Uuu*APY{fBIWcdZML>}Y=@t~SP+%*!K z8Sa@U)~Y8l?XQ=f-U#GE!b_|LSKA%Po@tYwCQSgG7x|duy_xjg}x)qI!0yzH+$O?(W4UG&so^Ie5g`JB!kYAf3;@UbKG(>mL8yHckuzU~)N&(AA{ayHIE{OaW1_kc_eKz`Y*3)Mb zovXk0uQZ2Vf>j0=r{jPKj4wE!@f>S?0H+FWU{6?Qvt_uZ!jAjg*o>6DKU8l}pRlUS zY-)vFUE_#7kwQMQWxSIALM5zC|MTx5DCq{aubiTzjQ8=33>is%qU-d&M0@%h0FZ2t zH8TdfoomK1y#kk3Osa(=HXGjQt4BacJ_U649gBm65u(FPKiP1e6H#nR@-KFdBE=+&pe&mQpKS#IZ5YM`g(^v7h zwu|!qp$9)$Mqp2KX{_!a6st%4JDOpXgG1qI_6aBhYz2;0oC z6=Ay>jz`#Gh7%A@F~f-nr@i@52YDsZnQE0#x^H{S{7JJEdG%y*LcPB!0z z%(vZq4>sRJ%(p~urOmd-ib(7&CDn~a;0loo6BGa5#a8^2C;m%2lV_ZOWOtZZe@_QUnFr`#{^Xv||vicQxYWRGL2JZSuvWwv8vUE%o$G zJnU6^o$?vnI%>n=0)8!}O2ZwC_ydp_>#4N~F6`Fu?G#;KiG)jGGzN1yJC7WBl%uT^ zt1@q7*oyzz)2V0TC}O96zkH6>Z=QhR zm|J&+OxyCGOEH6{=Wvly$yI~??gjznbiRaj+8z#Wt zD|0_g7FO{?-24fr?#b4-wVX_haaKP{fT17q=D$I4_tGlKTU>sqK(-DNtDC2vz&t$-5=xau2S3@a&snf`FXl7}I@FDm9yt-1P5KOwX& zqEig!SEw9YF?uwGy481lGDZLV$CPm@GV&{fNw)oxP!Lu(Qwy}Ssh88J>&4b8*+89_ z+OUuif~k3pt#=mVy4u6kR|AzXyDR!gVK~y2$SC$F?k-u66|023it$4lwgmqN6+<L0~`3SN9+vYAKlvWJV!>#E0c@AoV;Z>N8c8sOXPrx&ckI{PnC9JAl6N;=cto* z9-CaxF)+D)H{mQY6NH9ok6lte>9Sx(Qa&C7YSN@)#jbG>_B~ko(xIa$mn~I(GTL(c z|Kz+cSzAAa>?VD2vLcmC%^;U*eSUKNs}rzC$<9kVFHWw1eF7ZLFj(vqgxw+f@1(9M zr_wOfq_~t*mf&ddN`7O@owLAT`#FPgrtut?BbUr;cT>0~6&VekG?IcJ{Xd$)$}Owl zSgO_05y8Dp0VnQg(|^Fz=Odk;h{VN|_0Ok*8F4i9;%4Ih9DHIOP&@R6u$9oA%5+0B z?tkX#`u;yqqA+pgDq&*fs*NWvXaHO2&Ru986gobOP^S^ zD(GCgvUHWRGFTxm3d$8F*JWH-R<>%TJ^sQX@1j^1e zFzTFMa9j`uJVPIl|UwyA!Ix0FHl+H3@#~g-nwuSB`q(L7e85Yt8;0ovxL{S zc!IN}s$?awT;cQ;=ArDhzj;fRRsv7KMN2>is^l8dE6Yn3O>jPe8W%4uEiI`4nJQMT zaOPLYPb3e-@VGUy^NH2LlFEVf>nF?7l}m$57nYgqKqLYA0xK)!^72&`!IH%T8Z0ea z_++KCvLrawIc{uCvalY&ilQj^IVa%*r;l@C(3w#)E+ebPIlc_NGQw_9%qg#2SOpFu z3?>V+s!A%#7M7P2>zPGOMgu|#56&ee3m2DE41}5(rl?^09OvSN!G+)WVNFTJDrZH> zlc3yJzgMmXHjq1J4yaQ-2zZZNfYMm&Ws>0bxp0z7k(!#!n1L|5t=FGVh!`<-2!%FUzy=iA9S` zN@>ErmNqT-zUjU^A#>85lfEk~UAbsgMFp_d2Q&C|(Q-jvxqRiS>Xpt#t5%c~#%RQ@ zC`^s08l$R{L_JtQA0Ro9ctX#5k@x)c76FCh1#mklkODYzY-SJIT_W{F4*0+=Z zA{SrlW?zYKNu>^#F061qu@IEFMj&ELr$L@;i6%~rHe4XHhWxx?1hw^nOKbp#?4vkNw@(Cm2 z7gG7h9r0EwZ%K*Ir1BZV;s;ZC`^fkfUjA`xlnAyWJPsknRw)cigmP{x$CDulNzH;I z5p%!oaXe1MU&5R6*W&Ra-i7Z_2y6@Rq$2*DFg!6E;Tk+yh<6A>6EQ8@mf#tWI1INo zRA6hwQ-F96%0rd0&BK!-jIjM3<Cp7vZ&dau7d({)|Gn49^6_oA9>d*@UMM@m`dNer%hAXBgW5HQLASlC1&{ zTtSldp?!Fr*`C02E8;KX4Sn18G@e|Tx*2N7+>ct#3dbY*zrWQ42m+=cH4(LQ{eY!F4Z?;ySnZ^HKm zJb8$BqkIy=xZBkRFTL+*?;<*L!f5RKPn5_=aG{jHgJHhh+9@vhOerVEvHR2w`51aH~ ziZ}#!(tqJi@G?C2BYqy`iJk#GBLs&nPBiE*BmP}{KVZ_o1o3f*|0mvruLe9m#Lu8S z(f<)V@Hb8RnMwZ;;&&tdXOsRb5Qm{N=}o+epENvv#IK+{(Vuu`IQqZWq<BlDh{{``E#E+TuUyeBVI%y}~#7`=o?<0N@ z<*7gO@r)Wk|7yg)hwq0>`ag;IZHWI4Z))!uJo$)!hVq2Z$M9g>CjF~P|MiI9i};5o z{Z}D=2jXwzP5kr&JhKqLYSRDP#(xXyC;C5*$BF#E!dt*oi^q%jNBBKiw|2>obrO5Lg zY~jrd{(ouGzi<5i z&7}V-`;LqNmW)XbUyWHN?Dy*iVUB#&lLq&8V$`zR8$@F*!42@@CjL zrf6JKGoERfmbr)fL;u-0(pbu@z{Zotl*VOab9GbS8K3wb#=FBhM_6Ntb&WEnN~C8v z(d@h~>567`M{|0rRc$ivx9tBJSn(+J`?j2q^M7x4VR1R|=N~M2&FSgOlNTG!hIgwS zJoIKdeTo?dm|VVa#2GnSbSu^I$6eC%jvp6U?O<7K8Fbr^Y|gB>B<}{nLU3q}-^6Ka zV)v;^dp&>7XBXh0Ue$dpI7&O%f`dZbmgtn1w(cN=enr}-OMl=OkF%_7z32EXSb}ZX z*_1w5+dFBO*B6C*!G3AG?g$a#^d)hHhV3Ma&h0Eq)=}mYuoFwvpk>JtPe|hAWc9RNRzZva2;+# z2cCev_MEs^;T5c5TZfmx)}4wg+@o*;YT!W~8ZZSn;4Un}8mz->*nm&)8NR|6Y{L%h L!VlPkeX#Nmn``P+ literal 0 HcmV?d00001 diff --git a/tests/test_accessor.py b/tests/test_accessor.py index e7e87a3e..dfefa00d 100644 --- a/tests/test_accessor.py +++ b/tests/test_accessor.py @@ -1,4 +1,7 @@ import unittest +import hashlib +from tempfile import NamedTemporaryFile + from parameterized import parameterized_class import xcp.accessor @@ -13,3 +16,23 @@ def test_access(self): self.assertFalse(a.access('no_such_file')) self.assertEqual(a.lastError, 404) a.finish() + + def test_file_binfile(self): + BINFILE = "boot/isolinux/mboot.c32" + a = xcp.accessor.createAccessor(self.url, True) + a.start() + self.assertTrue(a.access(BINFILE)) + inf = a.openAddress(BINFILE) + with NamedTemporaryFile("w") as outf: + outf.writelines(inf) + outf.flush() + hasher = hashlib.md5() + with open(outf.name, "rb") as bincontents: + while True: + data = bincontents.read() + if not data: # EOF + break + hasher.update(data) + csum = hasher.hexdigest() + self.assertEqual(csum, "eab52cebc3723863432dc672360f6dac") + a.finish() From 72ac6b43c176f28f856118a16d021f617d52e043 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 26 Sep 2022 14:19:21 +0200 Subject: [PATCH 87/92] WIP test_accessor: write into copy file as binary This works properly for the http case, but FileAccessor provides us with a text fileobj handle, and `read()` gets a UTF-8 decoding error. FIXME: Accessor ctor requires a `mode` argument --- tests/test_accessor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_accessor.py b/tests/test_accessor.py index dfefa00d..d001edb8 100644 --- a/tests/test_accessor.py +++ b/tests/test_accessor.py @@ -23,8 +23,12 @@ def test_file_binfile(self): a.start() self.assertTrue(a.access(BINFILE)) inf = a.openAddress(BINFILE) - with NamedTemporaryFile("w") as outf: - outf.writelines(inf) + with NamedTemporaryFile("wb") as outf: + while True: + data = inf.read() + if not data: # EOF + break + outf.write(data) outf.flush() hasher = hashlib.md5() with open(outf.name, "rb") as bincontents: From d334f677b9014a5a3f8a12901b7c1cbbad0fc367 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 12:06:18 +0200 Subject: [PATCH 88/92] Pylint complements: honor len-as-condition convention Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index c3543c89..c1c4cd37 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -14,7 +14,7 @@ def writeRandomFile(fn, size, start=b'', add=b'a'): with open(fn, 'wb') as f: m = md5() m.update(start) - assert(len(add) != 0) + assert add while size > 0: d = m.digest() if size < len(d): From a498dd74608688e2a980f6059498893f29ba0113 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:40:49 +0200 Subject: [PATCH 89/92] Pylint complements: whitespace in expressions Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- xcp/cmd.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index c1c4cd37..fdb34f40 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -18,7 +18,7 @@ def writeRandomFile(fn, size, start=b'', add=b'a'): while size > 0: d = m.digest() if size < len(d): - d=d[:size] + d = d[:size] f.write(d) size -= len(d) m.update(add) diff --git a/xcp/cmd.py b/xcp/cmd.py index fbb991d0..16cfb64f 100644 --- a/xcp/cmd.py +++ b/xcp/cmd.py @@ -29,11 +29,11 @@ import xcp.logger as logger def runCmd(command, with_stdout = False, with_stderr = False, inputtext = None): - cmd = subprocess.Popen(command, bufsize = 1, - stdin = (inputtext and subprocess.PIPE or None), - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - shell = isinstance(command, six.string_types)) + cmd = subprocess.Popen(command, bufsize=1, + stdin=(inputtext and subprocess.PIPE or None), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=isinstance(command, six.string_types)) (out, err) = cmd.communicate(inputtext) rv = cmd.returncode From 23ae533d05c5b06f3fe28cfa017550054ca68dc7 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 16:24:30 +0200 Subject: [PATCH 90/92] Pylint complements: test_ifrename_logic: disable "no-member" warning Reported under python3 for members created on-the-fly in `setUp()` Signed-off-by: Yann Dirson --- tests/test_ifrename_logic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index 3bb7a911..a2715334 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -518,6 +518,7 @@ def test_ibft_nic_to_ibft(self): class TestInputSanitisation(unittest.TestCase): + # pylint: disable=no-member def setUp(self): """ From f1807548e3b873db5cebd11ca28bbd9b7fae1904 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 16:33:12 +0200 Subject: [PATCH 91/92] Pylint complements: avoid no-else-raise "refactor" issues With python3, pylint complains about `else: raise()` constructs. This rework avoids them and reduces cyclomatic complexity by using the error-out-first idiom. Signed-off-by: Yann Dirson --- xcp/net/mac.py | 32 ++++++++++++------------ xcp/pci.py | 66 ++++++++++++++++++++++++-------------------------- 2 files changed, 47 insertions(+), 51 deletions(-) diff --git a/xcp/net/mac.py b/xcp/net/mac.py index c0d4fba0..56ba4b7b 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -61,27 +61,25 @@ def __init__(self, addr): self.octets = [] self.integer = -1 - if isinstance(addr, six.string_types): - - res = VALID_COLON_MAC.match(addr) - if res: - self._set_from_str_octets(addr.split(":")) - return + if not isinstance(addr, six.string_types): + raise TypeError("String expected") - res = VALID_DASH_MAC.match(addr) - if res: - self._set_from_str_octets(addr.split("-")) - return + res = VALID_COLON_MAC.match(addr) + if res: + self._set_from_str_octets(addr.split(":")) + return - res = VALID_DOTQUAD_MAC.match(addr) - if res: - self._set_from_str_quads(addr.split(".")) - return + res = VALID_DASH_MAC.match(addr) + if res: + self._set_from_str_octets(addr.split("-")) + return - raise ValueError("Unrecognised MAC address '%s'" % addr) + res = VALID_DOTQUAD_MAC.match(addr) + if res: + self._set_from_str_quads(addr.split(".")) + return - else: - raise TypeError("String expected") + raise ValueError("Unrecognised MAC address '%s'" % addr) def _set_from_str_octets(self, octets): diff --git a/xcp/pci.py b/xcp/pci.py index d6cb4fef..1f911a6b 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -67,48 +67,46 @@ def __init__(self, addr): self.function = -1 self.index = -1 - if isinstance(addr, six.string_types): - - res = VALID_SBDFI.match(addr) - if res: - groups = res.groupdict() + if not isinstance(addr, six.string_types): + raise TypeError("String expected") - if "segment" in groups and groups["segment"] is not None: - self.segment = int(groups["segment"], 16) - else: - self.segment = 0 + res = VALID_SBDFI.match(addr) + if res: + groups = res.groupdict() - self.bus = int(groups["bus"], 16) - if not ( 0 <= self.bus < 2**8 ): - raise ValueError("Bus '%d' out of range 0 <= bus < 256" - % (self.bus,)) + if "segment" in groups and groups["segment"] is not None: + self.segment = int(groups["segment"], 16) + else: + self.segment = 0 - self.device = int(groups["device"], 16) - if not ( 0 <= self.device < 2**5): - raise ValueError("Device '%d' out of range 0 <= device < 32" - % (self.device,)) + self.bus = int(groups["bus"], 16) + if not ( 0 <= self.bus < 2**8 ): + raise ValueError("Bus '%d' out of range 0 <= bus < 256" + % (self.bus,)) - self.function = int(groups["function"], 16) - if not ( 0 <= self.function < 2**3): - raise ValueError("Function '%d' out of range 0 <= device " - "< 8" % (self.function,)) + self.device = int(groups["device"], 16) + if not ( 0 <= self.device < 2**5): + raise ValueError("Device '%d' out of range 0 <= device < 32" + % (self.device,)) - if "index" in groups and groups["index"] is not None: - self.index = int(groups["index"]) - else: - self.index = 0 + self.function = int(groups["function"], 16) + if not ( 0 <= self.function < 2**3): + raise ValueError("Function '%d' out of range 0 <= device " + "< 8" % (self.function,)) - self.integer = (int(self.segment << 16 | - self.bus << 8 | - self.device << 3 | - self.function) << 8 | - self.index) - return + if "index" in groups and groups["index"] is not None: + self.index = int(groups["index"]) + else: + self.index = 0 - raise ValueError("Unrecognised PCI address '%s'" % addr) + self.integer = (int(self.segment << 16 | + self.bus << 8 | + self.device << 3 | + self.function) << 8 | + self.index) + return - else: - raise TypeError("String expected") + raise ValueError("Unrecognised PCI address '%s'" % addr) def __str__(self): From c41e019934a2954a0c16a0c2cdba910a9e71c2e6 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 26 Jul 2022 16:50:54 +0200 Subject: [PATCH 92/92] CI: also run tests with python3 diff-cover defaults to origin/main in new version, it seems. Signed-off-by: Yann Dirson --- .github/workflows/main.yml | 32 +++++++++++++++++++------------- requirements-dev.txt | 1 + 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53462463..adf75fd7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,17 +3,24 @@ name: Unit tests on: [push, pull_request] jobs: - test_py2: - runs-on: ubuntu-20.04 + test: + strategy: + matrix: + include: + - pyversion: '2.7' + os: ubuntu-20.04 + - pyversion: '3' + os: ubuntu-latest + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up Python 2.7 + - name: Set up Python ${{ matrix.pyversion }} uses: actions/setup-python@v2 with: - python-version: '2.7' + python-version: ${{ matrix.pyversion }} - name: Install dependencies run: | @@ -21,7 +28,6 @@ jobs: pip install -r requirements-dev.txt # FIXME: branding.py still has no permanent home curl https://gist.github.com/ydirson/3c36a7e19d762cc529a6c82340894ccc/raw/5ca39f621b1feab813e171f535c1aad1bd483f1d/branding.py -O -L - pip install pyliblzma pip install -e . command -v xz @@ -29,22 +35,22 @@ jobs: run: | pytest --cov -rP coverage xml - coverage html - coverage html -d htmlcov-tests --include="tests/*" - diff-cover --html-report coverage-diff.html coverage.xml + coverage html -d htmlcov-${{ matrix.pyversion }} + coverage html -d htmlcov-tests-${{ matrix.pyversion }} --include="tests/*" + diff-cover --compare-branch=origin/master --html-report coverage-diff-${{ matrix.pyversion }}.html coverage.xml - name: Pylint run: | pylint --version pylint --exit-zero xcp/ tests/ setup.py pylint --exit-zero --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" xcp/ tests/ setup.py > pylint.txt - diff-quality --violations=pylint --html-report pylint-diff.html pylint.txt + diff-quality --compare-branch=origin/master --violations=pylint --html-report pylint-diff-${{ matrix.pyversion }}.html pylint.txt - uses: actions/upload-artifact@v3 with: name: Coverage and pylint reports path: | - coverage-diff.html - pylint-diff.html - htmlcov - htmlcov-tests + coverage-diff-${{ matrix.pyversion }}.html + pylint-diff-${{ matrix.pyversion }}.html + htmlcov-${{ matrix.pyversion }} + htmlcov-tests-${{ matrix.pyversion }} diff --git a/requirements-dev.txt b/requirements-dev.txt index 4debd01d..ce760b45 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,3 +12,4 @@ future # python-2.7 only configparser ; python_version < "3.0" +pyliblzma ; python_version < "3.0"