From 353235690df5545b371685eaeb4d4fb80182e1ed Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 28 Jul 2022 17:22:01 +0200 Subject: [PATCH 01/56] Prepare for futurize pass: fix "except" clause This is a common error documented in https://docs.python.org/2.7/whatsnew/2.6.html#pep-3110-exception-handling-changes Signed-off-by: Yann Dirson --- xcp/bootloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 51e362e8..4653caa5 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -431,7 +431,7 @@ def create_label(title): # If this fails, it is probably a string, so leave it unchanged. try: default = menu_order[int(default)] - except ValueError, KeyError: + except (ValueError, KeyError): pass finally: fh.close() From 3ee561510c28cf220512b5a11a3e4250a9c96322 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 11 Aug 2022 16:21:13 +0200 Subject: [PATCH 02/56] Futurize: first pass, using -wn1 Note there are indentation issues introduced in xcp.repository, as well as uses of the `future` package (which are not supposed to happen in pass 1), and several smaller issues. All of this is handled separately in further patches, so we can easily reconstruct this patch by running `futurize -wn1`. Signed-off-by: Yann Dirson --- requirements-dev.txt | 2 + setup.py | 1 + tests/test_cpio.py | 3 +- tests/test_ifrename_logic.py | 29 +++++----- tests/test_pci.py | 2 +- xcp/accessor.py | 4 +- xcp/bootloader.py | 109 ++++++++++++++++++----------------- xcp/cpiofile.py | 101 ++++++++++++++++---------------- xcp/dom0.py | 3 +- xcp/environ.py | 4 +- xcp/mount.py | 5 +- xcp/net/ifrename/dynamic.py | 18 +++--- xcp/net/ifrename/static.py | 12 ++-- xcp/net/mac.py | 2 +- xcp/pci.py | 14 ++--- xcp/repository.py | 23 ++++---- xcp/xmlunwrap.py | 9 +-- 17 files changed, 176 insertions(+), 165 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 37123724..ecda27a2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,3 +5,5 @@ diff_cover mock pytest pytest-cov +# dependencies also in setup.py until they can be used +future diff --git a/setup.py b/setup.py index 9fb2bd83..0fdb3f1b 100644 --- a/setup.py +++ b/setup.py @@ -35,5 +35,6 @@ ], requires=[ 'branding', + 'future', ], ) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 84763b26..4cb13825 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -1,3 +1,4 @@ +from __future__ import print_function import os import shutil import subprocess @@ -113,7 +114,7 @@ def test_bz2(self): def test_xz(self): if not self.doXZ: raise unittest.SkipTest("lzma package or xz tool not available") - print 'Running test for XZ' + print('Running test for XZ') self.doArchive('archive.cpio.xz', 'xz') # CpioFileCompat testing diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index 9fa9255c..c74522eb 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -1,3 +1,4 @@ +from __future__ import print_function import logging import sys import unittest @@ -43,15 +44,15 @@ def tearDown(self): self.siobuff.close() def debug_state(self, ts): - print >>sys.stderr, "" - print >>sys.stderr, self.siobuff.getvalue() - print >>sys.stderr, "" + print("", file=sys.stderr) + print(self.siobuff.getvalue(), file=sys.stderr) + print("", file=sys.stderr) if len(ts): for (s,d) in ts: - print >>sys.stderr, "'%s' -> '%s'" % (s, d) + print("'%s' -> '%s'" % (s, d), file=sys.stderr) else: - print >>sys.stderr, "No transactions" - print >>sys.stderr, "" + print("No transactions", file=sys.stderr) + print("", file=sys.stderr) def test_newhw_norules_1eth(self): @@ -268,16 +269,16 @@ def tearDown(self): self.siobuff.close() def debug_state(self, ts): - print >>sys.stderr, "" - print >>sys.stderr, self.siobuff.getvalue() - print >>sys.stderr, "" + print("", file=sys.stderr) + print(self.siobuff.getvalue(), file=sys.stderr) + print("", file=sys.stderr) if len(ts): - print >>sys.stderr, "Transactions:" + print("Transactions:", file=sys.stderr) for (s,d) in ts: - print >>sys.stderr, "'%s' -> '%s'" % (s, d) + print("'%s' -> '%s'" % (s, d), file=sys.stderr) else: - print >>sys.stderr, "No transactions" - print >>sys.stderr, "" + print("No transactions", file=sys.stderr) + print("", file=sys.stderr) def test_usecase1(self): """ @@ -559,7 +560,7 @@ def assertNotRaises(self, excp, fn, *argl, **kwargs): """Because unittest.TestCase seems to be missing this functionality""" try: fn(*argl, **kwargs) - except excp, e: + except excp as e: self.fail("function raised %s unexpectedly: %s" % (excp, e)) diff --git a/tests/test_pci.py b/tests/test_pci.py index 736b55de..17991428 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -9,7 +9,7 @@ class TestInvalid(unittest.TestCase): def test_invalid_types(self): self.assertRaises(TypeError, PCI, 0) - self.assertRaises(TypeError, PCI, 0L) + self.assertRaises(TypeError, PCI, 0) self.assertRaises(TypeError, PCI, (0,)) self.assertRaises(TypeError, PCI, []) self.assertRaises(TypeError, PCI, {}) diff --git a/xcp/accessor.py b/xcp/accessor.py index 22f9e24f..ecabb6c5 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -118,7 +118,7 @@ def openAddress(self, addr): class MountingAccessor(FilesystemAccessor): def __init__(self, mount_types, mount_source, mount_options = None): - ro = isinstance(mount_options, types.ListType) and 'ro' in mount_options + ro = isinstance(mount_options, list) and 'ro' in mount_options super(MountingAccessor, self).__init__(None, ro) self.mount_types = mount_types @@ -135,7 +135,7 @@ def start(self): try: opts = self.mount_options if fs == 'iso9660': - if isinstance(opts, types.ListType): + if isinstance(opts, list): if 'ro' not in opts: opts.append('ro') else: diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 4653caa5..441d3804 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -23,6 +23,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function import os import os.path import re @@ -455,47 +456,47 @@ def loadExisting(cls, root = '/'): elif os.path.exists(os.path.join(root, "boot/grub/menu.lst")): return cls.readGrub(os.path.join(root, "boot/grub/menu.lst")) else: - raise RuntimeError, "No existing bootloader configuration found" + raise RuntimeError("No existing bootloader configuration found") def writeExtLinux(self, dst_file = None): if hasattr(dst_file, 'name'): fh = dst_file else: fh = open(dst_file, 'w') - print >> fh, "# location " + self.location + print("# location " + self.location, file=fh) if self.serial: if self.serial.get('flow', None) is None: - print >> fh, "serial %s %s" % (self.serial['port'], - self.serial['baud']) + print("serial %s %s" % (self.serial['port'], + self.serial['baud']), file=fh) else: - print >> fh, "serial %s %s %s" % (self.serial['port'], + print("serial %s %s %s" % (self.serial['port'], self.serial['baud'], - self.serial['flow']) + self.serial['flow']), file=fh) if self.default: - print >> fh, "default " + self.default - print >> fh, "prompt 1" + print("default " + self.default, file=fh) + print("prompt 1", file=fh) if self.timeout: - print >> fh, "timeout %d" % self.timeout + print("timeout %d" % self.timeout, file=fh) for label in self.menu_order: - print >> fh, "\nlabel " + label + print("\nlabel " + label, file=fh) m = self.menu[label] if m.title: - print >> fh, " # " + m.title + print(" # " + m.title, file=fh) if m.tboot: - print >> fh, " kernel mboot.c32" - print >> fh, " append %s %s --- %s %s --- %s %s --- %s" % \ + print(" kernel mboot.c32", file=fh) + print(" append %s %s --- %s %s --- %s %s --- %s" % \ (m.tboot, m.tboot_args, m.hypervisor, m.hypervisor_args, - m.kernel, m.kernel_args, m.initrd) + m.kernel, m.kernel_args, m.initrd), file=fh) elif m.hypervisor: - print >> fh, " kernel mboot.c32" - print >> fh, " append %s %s --- %s %s --- %s" % \ - (m.hypervisor, m.hypervisor_args, m.kernel, m.kernel_args, m.initrd) + print(" kernel mboot.c32", file=fh) + print(" append %s %s --- %s %s --- %s" % \ + (m.hypervisor, m.hypervisor_args, m.kernel, m.kernel_args, m.initrd), file=fh) else: - print >> fh, " kernel " + m.kernel - print >> fh, " append " + m.kernel_args - print >> fh, " initrd " + m.initrd + print(" kernel " + m.kernel, file=fh) + print(" append " + m.kernel_args, file=fh) + print(" initrd " + m.initrd, file=fh) if not hasattr(dst_file, 'name'): fh.close() @@ -504,32 +505,32 @@ def writeGrub(self, dst_file = None): fh = dst_file else: fh = open(dst_file, 'w') - print >> fh, "# location " + self.location + print("# location " + self.location, file=fh) if self.serial: - print >> fh, "serial --unit=%s --speed=%s" % (self.serial['port'], - self.serial['baud']) - print >> fh, "terminal --timeout=10 console serial" + print("serial --unit=%s --speed=%s" % (self.serial['port'], + self.serial['baud']), file=fh) + print("terminal --timeout=10 console serial", file=fh) else: - print >> fh, "terminal console" + print("terminal console", file=fh) if self.default: for i in range(len(self.menu_order)): if self.menu_order[i] == self.default: - print >> fh, "default %d" % i + print("default %d" % i, file=fh) break if self.timeout: - print >> fh, "timeout %d" % (self.timeout / 10) + print("timeout %d" % (self.timeout / 10), file=fh) for label in self.menu_order: m = self.menu[label] - print >> fh, "\ntitle " + m.title + print("\ntitle " + m.title, file=fh) if m.hypervisor: - print >> fh, " kernel " + m.hypervisor + " " + m.hypervisor_args - print >> fh, " module " + m.kernel + " " + m.kernel_args - print >> fh, " module " + m.initrd + print(" kernel " + m.hypervisor + " " + m.hypervisor_args, file=fh) + print(" module " + m.kernel + " " + m.kernel_args, file=fh) + print(" module " + m.initrd, file=fh) else: - print >> fh, " kernel " + m.kernel + " " + m.kernel_args - print >> fh, " initrd " + m.initrd + print(" kernel " + m.kernel + " " + m.kernel_args, file=fh) + print(" initrd " + m.initrd, file=fh) if not hasattr(dst_file, 'name'): fh.close() @@ -540,19 +541,19 @@ def writeGrub2(self, dst_file = None): fh = open(dst_file, 'w') if self.serial: - print >> fh, "serial --unit=%s --speed=%s" % (self.serial['port'], - self.serial['baud']) - print >> fh, "terminal_input serial console" - print >> fh, "terminal_output serial console" + print("serial --unit=%s --speed=%s" % (self.serial['port'], + self.serial['baud']), file=fh) + print("terminal_input serial console", file=fh) + print("terminal_output serial console", file=fh) if self.default: for i in range(len(self.menu_order)): if self.menu_order[i] == self.default: - print >> fh, "set default=%d" % i + print("set default=%d" % i, file=fh) break else: - print >> fh, "set default='%s'" % str(self.default) + print("set default='%s'" % str(self.default), file=fh) if self.timeout: - print >> fh, "set timeout=%d" % (self.timeout / 10) + print("set timeout=%d" % (self.timeout / 10), file=fh) boilerplate = getattr(self, 'boilerplate', [])[:] boilerplate.reverse() @@ -563,41 +564,41 @@ def writeGrub2(self, dst_file = None): if boilerplate: text = boilerplate.pop() if text: - print >> fh, "\n".join(text) + print("\n".join(text), file=fh) extra = ' ' try: extra = m.extra except AttributeError: pass - print >> fh, "menuentry '%s'%s{" % (m.title, extra) + print("menuentry '%s'%s{" % (m.title, extra), file=fh) try: contents = "\n".join(m.contents) if contents: - print >> fh, contents + print(contents, file=fh) except AttributeError: pass if m.root: - print >> fh, "\tsearch --label --set root %s" % m.root + print("\tsearch --label --set root %s" % m.root, file=fh) if m.hypervisor: if m.tboot: - print >> fh, "\tmultiboot2 %s %s" % (m.tboot, m.tboot_args) - print >> fh, "\tmodule2 %s %s" % (m.hypervisor, m.hypervisor_args) + print("\tmultiboot2 %s %s" % (m.tboot, m.tboot_args), file=fh) + print("\tmodule2 %s %s" % (m.hypervisor, m.hypervisor_args), file=fh) else: - print >> fh, "\tmultiboot2 %s %s" % (m.hypervisor, m.hypervisor_args) + print("\tmultiboot2 %s %s" % (m.hypervisor, m.hypervisor_args), file=fh) if m.kernel: - print >> fh, "\tmodule2 %s %s" % (m.kernel, m.kernel_args) + print("\tmodule2 %s %s" % (m.kernel, m.kernel_args), file=fh) if m.initrd: - print >> fh, "\tmodule2 %s" % m.initrd + print("\tmodule2 %s" % m.initrd, file=fh) else: if m.kernel: - print >> fh, "\tlinux %s %s" % (m.kernel, m.kernel_args) + print("\tlinux %s %s" % (m.kernel, m.kernel_args), file=fh) if m.initrd: - print >> fh, "\tinitrd %s" % m.initrd - print >> fh, "}" + print("\tinitrd %s" % m.initrd, file=fh) + print("}", file=fh) if not hasattr(dst_file, 'name'): fh.close() @@ -642,9 +643,9 @@ def newDefault(cls, kernel_link_name, initrd_link_name, root = '/'): if b.menu[b.default].kernel != kernel_link_name: backup = [] if not os.path.exists(os.path.join(root, kernel_link_name[1:])): - raise RuntimeError, "kernel symlink not found" + raise RuntimeError("kernel symlink not found") if not os.path.exists(os.path.join(root, initrd_link_name[1:])): - raise RuntimeError, "initrd symlink not found" + raise RuntimeError("initrd symlink not found") old_kernel_link = b.menu[b.default].kernel old_ver = 'old' m = re.search(r'(-\d+\.\d+)-', old_kernel_link) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index a490aeff..a259efc3 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -33,6 +33,7 @@ Derived from Lars Gustäbel's tarfile.py """ +from __future__ import print_function __version__ = "0.1" __author__ = "Simon Rowe" @@ -55,7 +56,7 @@ # handling. In many places it is assumed a simple substitution of / by the # local os.path.sep is good enough to convert pathnames, but this does not # work with the mac rooted:path:name versus :nonrooted:path:name syntax - raise ImportError, "cpiofile does not work for platform==mac" + raise ImportError("cpiofile does not work for platform==mac") try: import grp as GRP, pwd as PWD @@ -78,26 +79,26 @@ #--------------------------------------------------------- # Bits used in the mode field, values in octal. #--------------------------------------------------------- -S_IFLNK = 0120000 # symbolic link -S_IFREG = 0100000 # regular file -S_IFBLK = 0060000 # block device -S_IFDIR = 0040000 # directory -S_IFCHR = 0020000 # character device -S_IFIFO = 0010000 # fifo - -TSUID = 04000 # set UID on execution -TSGID = 02000 # set GID on execution -TSVTX = 01000 # reserved - -TUREAD = 0400 # read by owner -TUWRITE = 0200 # write by owner -TUEXEC = 0100 # execute/search by owner -TGREAD = 0040 # read by group -TGWRITE = 0020 # write by group -TGEXEC = 0010 # execute/search by group -TOREAD = 0004 # read by other -TOWRITE = 0002 # write by other -TOEXEC = 0001 # execute/search by other +S_IFLNK = 0o120000 # symbolic link +S_IFREG = 0o100000 # regular file +S_IFBLK = 0o060000 # block device +S_IFDIR = 0o040000 # directory +S_IFCHR = 0o020000 # character device +S_IFIFO = 0o010000 # fifo + +TSUID = 0o4000 # set UID on execution +TSGID = 0o2000 # set GID on execution +TSVTX = 0o1000 # reserved + +TUREAD = 0o400 # read by owner +TUWRITE = 0o200 # write by owner +TUEXEC = 0o100 # execute/search by owner +TGREAD = 0o040 # read by group +TGWRITE = 0o020 # write by group +TGEXEC = 0o010 # execute/search by group +TOREAD = 0o004 # read by other +TOWRITE = 0o002 # write by other +TOEXEC = 0o001 # execute/search by other #--------------------------------------------------------- # Some useful functions @@ -253,7 +254,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.fileobj = fileobj self.bufsize = bufsize self.buf = "" - self.pos = 0L + self.pos = 0 self.closed = False if comptype == "gz": @@ -347,8 +348,8 @@ def close(self): # while the same crc on a 64-bit box may "look positive". # To avoid irksome warnings from the `struct` module, force # it to look positive on all boxes. - self.fileobj.write(struct.pack(" 1: - if self.hardlinks and self.inodes.has_key(cpioinfo.ino): + if self.hardlinks and cpioinfo.ino in self.inodes: # this inode has already been added cpioinfo.size = 0 self.inodes[cpioinfo.ino].append(cpioinfo.name) @@ -1418,7 +1419,7 @@ def extractall(self, path=".", members=None): # Extract directory with a safe mode, so that # all files below can be extracted as well. try: - os.makedirs(os.path.join(path, cpioinfo.name), 0777) + os.makedirs(os.path.join(path, cpioinfo.name), 0o777) except EnvironmentError: pass directories.append(cpioinfo) @@ -1436,7 +1437,7 @@ def extractall(self, path=".", members=None): self.chown(cpioinfo, path) self.utime(cpioinfo, path) self.chmod(cpioinfo, path) - except ExtractError, e: + except ExtractError as e: if self.errorlevel > 1: raise else: @@ -1462,7 +1463,7 @@ def extract(self, member, path=""): try: self._extract_member(cpioinfo, os.path.join(path, cpioinfo.name)) - except EnvironmentError, e: + except EnvironmentError as e: if self.errorlevel > 0: raise else: @@ -1470,7 +1471,7 @@ def extract(self, member, path=""): self._dbg(1, "cpiofile: %s" % e.strerror) else: self._dbg(1, "cpiofile: %s %r" % (e.strerror, e.filename)) - except ExtractError, e: + except ExtractError as e: if self.errorlevel > 1: raise else: @@ -1525,7 +1526,7 @@ def _extract_member(self, cpioinfo, cpiogetpath): if upperdirs and not os.path.exists(upperdirs): ti = CpioInfo() ti.name = upperdirs - ti.mode = S_IFDIR | 0777 + ti.mode = S_IFDIR | 0o777 ti.mtime = cpioinfo.mtime ti.uid = cpioinfo.uid ti.gid = cpioinfo.gid @@ -1567,7 +1568,7 @@ def makedir(self, cpioinfo, cpiogetpath): """ try: os.mkdir(cpiogetpath) - except EnvironmentError, e: + except EnvironmentError as e: if e.errno != errno.EEXIST: raise @@ -1578,7 +1579,7 @@ def makefile(self, cpioinfo, cpiogetpath): if cpioinfo.nlink == 1: extractinfo = cpioinfo else: - if self.inodes.has_key(cpioinfo.ino): + if cpioinfo.ino in self.inodes: # actual file exists, create link # FIXME handle platforms that don't support hardlinks os.link(os.path.join(cpioinfo._link_path, @@ -1738,7 +1739,7 @@ def next(self): cpioinfo = self.proc_member(cpioinfo) - except ValueError, e: + except ValueError as e: if self.offset == 0: raise ReadError("empty, unreadable or compressed " "file: %s" % e) @@ -1803,7 +1804,7 @@ def _load(self): members. """ while True: - cpioinfo = self.next() + cpioinfo = next(self) if cpioinfo is None: break self._loaded = True @@ -1829,7 +1830,7 @@ def _dbg(self, level, msg): """Write debugging output to sys.stderr. """ if level <= self.debug: - print >> sys.stderr, msg + print(msg, file=sys.stderr) # class CpioFile class CpioIter(object): @@ -1856,7 +1857,7 @@ def next(self): # happen that getmembers() is called during iteration, # which will cause CpioIter to stop prematurely. if not self.cpiofile._loaded: - cpioinfo = self.cpiofile.next() + cpioinfo = next(self.cpiofile) if not cpioinfo: self.cpiofile._loaded = True raise StopIteration diff --git a/xcp/dom0.py b/xcp/dom0.py index f2ca7097..66a23a9e 100644 --- a/xcp/dom0.py +++ b/xcp/dom0.py @@ -23,9 +23,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import re -import version +from . import version import sys def default_memory_v2(host_mem_kib): diff --git a/xcp/environ.py b/xcp/environ.py index 27e57323..c8901f6d 100644 --- a/xcp/environ.py +++ b/xcp/environ.py @@ -47,7 +47,7 @@ def readInventory(root = '/'): try: fh = open(os.path.join(root, 'etc/xensource-inventory')) - for line in ( x for x in ( y.strip() for y in fh.xreadlines() ) + for line in ( x for x in ( y.strip() for y in fh ) if not x.startswith('#') ): vals = line.split('=', 1) @@ -59,7 +59,7 @@ def readInventory(root = '/'): d[vals[0]] = vals[1].strip('"\'') - except IOError, e: + except IOError as e: raise InventoryError("Error reading from file '%s'" % (e,)) finally: diff --git a/xcp/mount.py b/xcp/mount.py index 60d191bc..647817fa 100644 --- a/xcp/mount.py +++ b/xcp/mount.py @@ -23,6 +23,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from future.utils import raise_ import os import os.path import tempfile @@ -52,13 +53,13 @@ def mount(dev, mountpoint, options = None, fstype = None, label = None): rc, out, err = xcp.cmd.runCmd(cmd, with_stdout=True, with_stderr=True) if rc != 0: - raise MountException, "out: '%s' err: '%s'" % (out, err) + raise_(MountException, "out: '%s' err: '%s'" % (out, err)) def bindMount(source, mountpoint): cmd = [ '/bin/mount', '--bind', source, mountpoint] rc, out, err = xcp.cmd.runCmd(cmd, with_stdout=True, with_stderr=True) if rc != 0: - raise MountException, "out: '%s' err: '%s'" % (out, err) + raise_(MountException, "out: '%s' err: '%s'" % (out, err)) def umount(mountpoint, force = False): # -d option also removes the loop device (if present) diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index 62844848..befab65a 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -104,7 +104,7 @@ def load_and_parse(self): LOG.error("No source of data to parse") return False - except IOError, e: + except IOError as e: LOG.error("IOError while reading file: %s" % (e,)) return False finally: @@ -137,7 +137,7 @@ def load_and_parse(self): if len(entry) != 3: raise ValueError("Expected 3 entries") macpci = MACPCI(entry[0], entry[1], tname=entry[2]) - except (TypeError, ValueError), e: + except (TypeError, ValueError) as e: LOG.warning("Invalid lastboot data entry: %s" % (e,)) continue @@ -149,7 +149,7 @@ def load_and_parse(self): if len(entry) != 3: raise ValueError("Expected 3 entries") macpci = MACPCI(entry[0], entry[1], tname=entry[2]) - except (TypeError, ValueError), e: + except (TypeError, ValueError) as e: LOG.warning("Invalid old data entry: %s" % (e,)) continue self.old.append(macpci) @@ -178,7 +178,7 @@ def generate(self, state): if nic.mac == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -199,7 +199,7 @@ def generate(self, state): if nic.ppn == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -213,7 +213,7 @@ def generate(self, state): try: nic = pci_sbdfi_to_nic(value, state) rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -226,7 +226,7 @@ def generate(self, state): if nic.label == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -257,7 +257,7 @@ def validate(entry): return False MACPCI(entry[0], entry[1], tname=entry[2]) return True - except Exception, e: + except Exception as e: LOG.warning("Failed to validate '%s' because '%s'" % (entry, e)) return False @@ -301,7 +301,7 @@ def save(self, header = True): LOG.error("No source of data to parse") return False - except IOError, e: + except IOError as e: LOG.error("IOError while reading file: %s" % (e,)) return False finally: diff --git a/xcp/net/ifrename/static.py b/xcp/net/ifrename/static.py index 89a77fda..615c2073 100644 --- a/xcp/net/ifrename/static.py +++ b/xcp/net/ifrename/static.py @@ -127,7 +127,7 @@ def load_and_parse(self): LOG.error("No source of data to parse") return False - except IOError, e: + except IOError as e: LOG.error("IOError while reading file: %s" % (e,)) return False finally: @@ -230,7 +230,7 @@ def generate(self, state): if nic.mac == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -251,7 +251,7 @@ def generate(self, state): if nic.ppn == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -265,7 +265,7 @@ def generate(self, state): try: nic = pci_sbdfi_to_nic(value, state) rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -278,7 +278,7 @@ def generate(self, state): if nic.label == value: try: rule = MACPCI(nic.mac, nic.pci, tname=target) - except Exception, e: + except Exception as e: LOG.warning("Error creating rule: %s" % (e,)) continue self.rules.append(rule) @@ -340,7 +340,7 @@ def save(self, header = True): LOG.error("No source of data to parse") return False - except IOError, e: + except IOError as e: LOG.error("IOError while reading file: %s" % (e,)) return False finally: diff --git a/xcp/net/mac.py b/xcp/net/mac.py index a675a179..9b042c8f 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -57,7 +57,7 @@ def __init__(self, addr): """Constructor""" self.octets = [] - self.integer = -1L + self.integer = -1 if isinstance(addr, (str, unicode)): diff --git a/xcp/pci.py b/xcp/pci.py index 8b371152..367e7dc5 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -124,7 +124,7 @@ def __eq__(self, rhs): else: try: return self.integer == PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __ne__(self, rhs): @@ -133,7 +133,7 @@ def __ne__(self, rhs): else: try: return self.integer != PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __hash__(self): @@ -145,7 +145,7 @@ def __lt__(self, rhs): else: try: return self.integer < PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __le__(self, rhs): @@ -154,7 +154,7 @@ def __le__(self, rhs): else: try: return self.integer <= PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __gt__(self, rhs): @@ -163,7 +163,7 @@ def __gt__(self, rhs): else: try: return self.integer > PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented def __ge__(self, rhs): @@ -172,7 +172,7 @@ def __ge__(self, rhs): else: try: return self.integer >= PCI(rhs).integer - except StandardError: + except Exception: return NotImplemented @@ -232,7 +232,7 @@ def read(cls): for f in ['/usr/share/hwdata/pci.ids']: if os.path.exists(f): return cls(f) - raise Exception, 'Failed to open PCI database' + raise Exception('Failed to open PCI database') def findVendor(self, vendor): return vendor in self.vendor_dict and self.vendor_dict[vendor] or None diff --git a/xcp/repository.py b/xcp/repository.py index 3da2f17b..5b23a6ba 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -23,6 +23,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from future.utils import raise_ import md5 import os.path import xml.dom.minidom @@ -187,8 +188,8 @@ def _getVersion(cls, access, category): ver_str = treeinfo.get(category, 'version') repo_ver = version.Version.from_string(ver_str) - except Exception, e: - raise RepoFormatError, "Failed to open %s: %s" % (cls.TREEINFO_FILENAME, str(e)) + except Exception as e: + raise_(RepoFormatError, "Failed to open %s: %s" % (cls.TREEINFO_FILENAME, str(e))) access.finish() return repo_ver @@ -230,8 +231,8 @@ def findRepositories(cls, access): for line in extra: package_list.append(line.strip()) extra.close() - except Exception, e: - raise RepoFormatError, "Failed to open %s: %s" % (cls.REPOLIST_FILENAME, str(e)) + except Exception as e: + raise_(RepoFormatError, "Failed to open %s: %s" % (cls.REPOLIST_FILENAME, str(e))) for loc in package_list: if cls.isRepo(access, loc): @@ -250,17 +251,17 @@ def __init__(self, access, base, is_group = False): try: repofile = access.openAddress(os.path.join(base, self.REPOSITORY_FILENAME)) - except Exception, e: + except Exception as e: access.finish() - raise NoRepository, e +raise_(NoRepository, e) self._parse_repofile(repofile) repofile.close() try: pkgfile = access.openAddress(os.path.join(base, self.PKGDATA_FILENAME)) - except Exception, e: + except Exception as e: access.finish() - raise NoRepository, e +raise_(NoRepository, e) self._parse_packages(pkgfile) pkgfile.close() @@ -290,7 +291,7 @@ def _parse_repofile(self, repofile): try: xmldoc = xml.dom.minidom.parseString(repofile_contents) except: - raise RepoFormatError, "%s not in XML" % self.REPOSITORY_FILENAME + raise_(RepoFormatError, "%s not in XML" % self.REPOSITORY_FILENAME) try: repo_node = xmlunwrap.getElementsByTagName(xmldoc, ['repository'], mandatory = True) @@ -314,7 +315,7 @@ def _parse_repofile(self, repofile): assert req['test'] in self.OPER_MAP self.requires.append(req) except: - raise RepoFormatError, "%s format error" % self.REPOSITORY_FILENAME + raise_(RepoFormatError, "%s format error" % self.REPOSITORY_FILENAME) self.identifier = "%s:%s" % (self.originator, self.name) ver_str = self.version @@ -333,7 +334,7 @@ def _parse_packages(self, pkgfile): try: xmldoc = xml.dom.minidom.parseString(pkgfile_contents) except: - raise RepoFormatError, "%s not in XML" % self.PKGDATA_FILENAME + raise_(RepoFormatError, "%s not in XML" % self.PKGDATA_FILENAME) for pkg_node in xmlunwrap.getElementsByTagName(xmldoc, ['package']): pkg = self._create_package(pkg_node) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 654b84bf..6f50988c 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -23,6 +23,7 @@ """xmlunwrap - general methods to unwrap XML elements & attributes""" +from future.utils import raise_ class XmlUnwrapError(Exception): pass @@ -39,7 +40,7 @@ def getElementsByTagName(el, tags, mandatory = False): for tag in tags: matching.extend(el.getElementsByTagName(tag)) if mandatory and len(matching) == 0: - raise XmlUnwrapError, "Missing mandatory element %s" % tags[0] + raise_(XmlUnwrapError, "Missing mandatory element %s" % tags[0]) return matching def getStrAttribute(el, attrs, default = '', mandatory = False): @@ -50,7 +51,7 @@ def getStrAttribute(el, attrs, default = '', mandatory = False): matching.append(val) if len(matching) == 0: if mandatory: - raise XmlUnwrapError, "Missing mandatory attribute %s" % attrs[0] + raise_(XmlUnwrapError, "Missing mandatory attribute %s" % attrs[0]) return default return matching[0] @@ -69,7 +70,7 @@ def getIntAttribute(el, attrs, default = None): try: int_val = int(val, 0) except: - raise XmlUnwrapError, "Invalid integer value for %s" % attrs[0] + raise_(XmlUnwrapError, "Invalid integer value for %s" % attrs[0]) return int_val def getMapAttribute(el, attrs, mapping, default = None): @@ -78,7 +79,7 @@ def getMapAttribute(el, attrs, mapping, default = None): key = getStrAttribute(el, attrs, default, mandatory) if key not in k: - raise XmlUnwrapError, "Unexpected key %s for attribute" % key + raise_(XmlUnwrapError, "Unexpected key %s for attribute" % key) k_list = list(k) return v[k_list.index(key)] From 8ef728818a2f494c0128d9fcd9ca193771732d39 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 28 Jul 2022 17:28:13 +0200 Subject: [PATCH 03/56] Futurize fixes: drop now-duplicated test Signed-off-by: Yann Dirson --- tests/test_pci.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pci.py b/tests/test_pci.py index 17991428..ac03c375 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -8,7 +8,6 @@ class TestInvalid(unittest.TestCase): def test_invalid_types(self): - self.assertRaises(TypeError, PCI, 0) self.assertRaises(TypeError, PCI, 0) self.assertRaises(TypeError, PCI, (0,)) self.assertRaises(TypeError, PCI, []) From ddf7ba634aa03403f62772cd2ce890a31213256b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 12 Jul 2022 11:50:45 +0200 Subject: [PATCH 04/56] Futurize fixes: fix broken indentations Signed-off-by: Yann Dirson --- xcp/repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/repository.py b/xcp/repository.py index 5b23a6ba..63b1277a 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -253,7 +253,7 @@ def __init__(self, access, base, is_group = False): repofile = access.openAddress(os.path.join(base, self.REPOSITORY_FILENAME)) except Exception as e: access.finish() -raise_(NoRepository, e) + raise_(NoRepository, e) self._parse_repofile(repofile) repofile.close() @@ -261,7 +261,7 @@ def __init__(self, access, base, is_group = False): pkgfile = access.openAddress(os.path.join(base, self.PKGDATA_FILENAME)) except Exception as e: access.finish() -raise_(NoRepository, e) + raise_(NoRepository, e) self._parse_packages(pkgfile) pkgfile.close() From d97f4aadcee0c4c98753be31941eb9552cab0be8 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 12 Jul 2022 11:51:09 +0200 Subject: [PATCH 05/56] Futurize fixes: use "raise" not "raise_" when not reraising Signed-off-by: Yann Dirson --- xcp/mount.py | 5 ++--- xcp/xmlunwrap.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/xcp/mount.py b/xcp/mount.py index 647817fa..49b0a2e8 100644 --- a/xcp/mount.py +++ b/xcp/mount.py @@ -23,7 +23,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from future.utils import raise_ import os import os.path import tempfile @@ -53,13 +52,13 @@ def mount(dev, mountpoint, options = None, fstype = None, label = None): rc, out, err = xcp.cmd.runCmd(cmd, with_stdout=True, with_stderr=True) if rc != 0: - raise_(MountException, "out: '%s' err: '%s'" % (out, err)) + raise MountException("out: '%s' err: '%s'" % (out, err)) def bindMount(source, mountpoint): cmd = [ '/bin/mount', '--bind', source, mountpoint] rc, out, err = xcp.cmd.runCmd(cmd, with_stdout=True, with_stderr=True) if rc != 0: - raise_(MountException, "out: '%s' err: '%s'" % (out, err)) + raise MountException("out: '%s' err: '%s'" % (out, err)) def umount(mountpoint, force = False): # -d option also removes the loop device (if present) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 6f50988c..59404211 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -40,7 +40,7 @@ def getElementsByTagName(el, tags, mandatory = False): for tag in tags: matching.extend(el.getElementsByTagName(tag)) if mandatory and len(matching) == 0: - raise_(XmlUnwrapError, "Missing mandatory element %s" % tags[0]) + raise XmlUnwrapError("Missing mandatory element %s" % tags[0]) return matching def getStrAttribute(el, attrs, default = '', mandatory = False): @@ -51,7 +51,7 @@ def getStrAttribute(el, attrs, default = '', mandatory = False): matching.append(val) if len(matching) == 0: if mandatory: - raise_(XmlUnwrapError, "Missing mandatory attribute %s" % attrs[0]) + raise XmlUnwrapError("Missing mandatory attribute %s" % attrs[0]) return default return matching[0] @@ -79,7 +79,7 @@ def getMapAttribute(el, attrs, mapping, default = None): key = getStrAttribute(el, attrs, default, mandatory) if key not in k: - raise_(XmlUnwrapError, "Unexpected key %s for attribute" % key) + raise XmlUnwrapError("Unexpected key %s for attribute" % key) k_list = list(k) return v[k_list.index(key)] From d05898afd7744608a82322fe199cec4bd867449b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:39:44 +0200 Subject: [PATCH 06/56] Un-futurize: replace future.utils.raise_ with six.raise_from Signed-off-by: Yann Dirson --- requirements-dev.txt | 2 +- setup.py | 2 +- xcp/repository.py | 25 ++++++++++++++----------- xcp/xmlunwrap.py | 7 ++++--- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ecda27a2..6bbf9447 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,4 @@ mock pytest pytest-cov # dependencies also in setup.py until they can be used -future +six diff --git a/setup.py b/setup.py index 0fdb3f1b..e7881222 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,6 @@ ], requires=[ 'branding', - 'future', + 'six', ], ) diff --git a/xcp/repository.py b/xcp/repository.py index 63b1277a..98b39683 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -23,12 +23,13 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from future.utils import raise_ import md5 import os.path import xml.dom.minidom import ConfigParser +import six + import xcp.version as version import xcp.xmlunwrap as xmlunwrap @@ -189,7 +190,8 @@ def _getVersion(cls, access, category): repo_ver = version.Version.from_string(ver_str) except Exception as e: - raise_(RepoFormatError, "Failed to open %s: %s" % (cls.TREEINFO_FILENAME, str(e))) + six.raise_from(RepoFormatError("Failed to open %s: %s" % + (cls.TREEINFO_FILENAME, str(e))), e) access.finish() return repo_ver @@ -232,7 +234,8 @@ def findRepositories(cls, access): package_list.append(line.strip()) extra.close() except Exception as e: - raise_(RepoFormatError, "Failed to open %s: %s" % (cls.REPOLIST_FILENAME, str(e))) + six.raise_from(RepoFormatError("Failed to open %s: %s" % + (cls.REPOLIST_FILENAME, str(e))), e) for loc in package_list: if cls.isRepo(access, loc): @@ -253,7 +256,7 @@ def __init__(self, access, base, is_group = False): repofile = access.openAddress(os.path.join(base, self.REPOSITORY_FILENAME)) except Exception as e: access.finish() - raise_(NoRepository, e) + six.raise_from(NoRepository(), e) self._parse_repofile(repofile) repofile.close() @@ -261,7 +264,7 @@ def __init__(self, access, base, is_group = False): pkgfile = access.openAddress(os.path.join(base, self.PKGDATA_FILENAME)) except Exception as e: access.finish() - raise_(NoRepository, e) + six.raise_from(NoRepository(), e) self._parse_packages(pkgfile) pkgfile.close() @@ -290,8 +293,8 @@ def _parse_repofile(self, repofile): # build xml doc object try: xmldoc = xml.dom.minidom.parseString(repofile_contents) - except: - raise_(RepoFormatError, "%s not in XML" % self.REPOSITORY_FILENAME) + except Exception as e: + six.raise_from(RepoFormatError("%s not in XML" % self.REPOSITORY_FILENAME), e) try: repo_node = xmlunwrap.getElementsByTagName(xmldoc, ['repository'], mandatory = True) @@ -314,8 +317,8 @@ def _parse_repofile(self, repofile): del req['build'] assert req['test'] in self.OPER_MAP self.requires.append(req) - except: - raise_(RepoFormatError, "%s format error" % self.REPOSITORY_FILENAME) + except Exception as e: + six.raise_from(RepoFormatError("%s format error" % self.REPOSITORY_FILENAME), e) self.identifier = "%s:%s" % (self.originator, self.name) ver_str = self.version @@ -333,8 +336,8 @@ def _parse_packages(self, pkgfile): # build xml doc object try: xmldoc = xml.dom.minidom.parseString(pkgfile_contents) - except: - raise_(RepoFormatError, "%s not in XML" % self.PKGDATA_FILENAME) + except Exception as e: + six.raise_from(RepoFormatError("%s not in XML" % self.PKGDATA_FILENAME), e) for pkg_node in xmlunwrap.getElementsByTagName(xmldoc, ['package']): pkg = self._create_package(pkg_node) diff --git a/xcp/xmlunwrap.py b/xcp/xmlunwrap.py index 59404211..1487afab 100644 --- a/xcp/xmlunwrap.py +++ b/xcp/xmlunwrap.py @@ -23,7 +23,8 @@ """xmlunwrap - general methods to unwrap XML elements & attributes""" -from future.utils import raise_ +import six + class XmlUnwrapError(Exception): pass @@ -69,8 +70,8 @@ def getIntAttribute(el, attrs, default = None): return default try: int_val = int(val, 0) - except: - raise_(XmlUnwrapError, "Invalid integer value for %s" % attrs[0]) + except Exception as e: + six.raise_from(XmlUnwrapError("Invalid integer value for %s" % attrs[0]), e) return int_val def getMapAttribute(el, attrs, mapping, default = None): From 459f5eff60cb45da4c53867d3aa35f6e54a5c6db Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 12 Jul 2022 11:51:15 +0200 Subject: [PATCH 07/56] Futurize fixes: remove now-unused imports Signed-off-by: Yann Dirson --- xcp/accessor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index ecabb6c5..7763cf96 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -28,7 +28,6 @@ import ftplib import os import tempfile -import types import urllib import urllib2 import urlparse From 6505b3f300b5a30c04e25bb8b8861fc8618c2388 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:02:04 +0200 Subject: [PATCH 08/56] Futurize pylint complements: "print" continuation indentations, line lengths Signed-off-by: Yann Dirson --- xcp/bootloader.py | 22 +++++++++++----------- xcp/cpiofile.py | 3 +-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 441d3804..0602b822 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -468,11 +468,11 @@ def writeExtLinux(self, dst_file = None): if self.serial: if self.serial.get('flow', None) is None: print("serial %s %s" % (self.serial['port'], - self.serial['baud']), file=fh) + self.serial['baud']), file=fh) else: print("serial %s %s %s" % (self.serial['port'], - self.serial['baud'], - self.serial['flow']), file=fh) + self.serial['baud'], + self.serial['flow']), file=fh) if self.default: print("default " + self.default, file=fh) print("prompt 1", file=fh) @@ -486,13 +486,13 @@ def writeExtLinux(self, dst_file = None): print(" # " + m.title, file=fh) if m.tboot: print(" kernel mboot.c32", file=fh) - print(" append %s %s --- %s %s --- %s %s --- %s" % \ - (m.tboot, m.tboot_args, m.hypervisor, m.hypervisor_args, - m.kernel, m.kernel_args, m.initrd), file=fh) + print(" append %s %s --- %s %s --- %s %s --- %s" % + (m.tboot, m.tboot_args, m.hypervisor, m.hypervisor_args, + m.kernel, m.kernel_args, m.initrd), file=fh) elif m.hypervisor: print(" kernel mboot.c32", file=fh) - print(" append %s %s --- %s %s --- %s" % \ - (m.hypervisor, m.hypervisor_args, m.kernel, m.kernel_args, m.initrd), file=fh) + print(" append %s %s --- %s %s --- %s" % + (m.hypervisor, m.hypervisor_args, m.kernel, m.kernel_args, m.initrd), file=fh) else: print(" kernel " + m.kernel, file=fh) print(" append " + m.kernel_args, file=fh) @@ -508,8 +508,8 @@ def writeGrub(self, dst_file = None): print("# location " + self.location, file=fh) if self.serial: - print("serial --unit=%s --speed=%s" % (self.serial['port'], - self.serial['baud']), file=fh) + print("serial --unit=%s --speed=%s" % + (self.serial['port'], self.serial['baud']), file=fh) print("terminal --timeout=10 console serial", file=fh) else: print("terminal console", file=fh) @@ -542,7 +542,7 @@ def writeGrub2(self, dst_file = None): if self.serial: print("serial --unit=%s --speed=%s" % (self.serial['port'], - self.serial['baud']), file=fh) + self.serial['baud']), file=fh) print("terminal_input serial console", file=fh) print("terminal_output serial console", file=fh) if self.default: diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index a259efc3..79ffcd4e 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1306,8 +1306,7 @@ def list(self, verbose=True): print(filemode(cpioinfo.mode), end=' ') print("%d/%d" % (cpioinfo.uid, cpioinfo.gid), end=' ') if cpioinfo.ischr() or cpioinfo.isblk(): - print("%10s" % ("%d,%d" \ - % (cpioinfo.devmajor, cpioinfo.devminor)), end=' ') + print("%10s" % ("%d,%d" % (cpioinfo.devmajor, cpioinfo.devminor)), end=' ') else: print("%10d" % cpioinfo.size, end=' ') print("%d-%02d-%02d %02d:%02d:%02d" \ From 7ae43d5d5a6c00d1cc69963d0bd58df854318051 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:40:49 +0200 Subject: [PATCH 09/56] Futurize pylint complements: whitespace in expressions Signed-off-by: Yann Dirson --- xcp/environ.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/environ.py b/xcp/environ.py index c8901f6d..8223cca8 100644 --- a/xcp/environ.py +++ b/xcp/environ.py @@ -47,8 +47,8 @@ def readInventory(root = '/'): try: fh = open(os.path.join(root, 'etc/xensource-inventory')) - for line in ( x for x in ( y.strip() for y in fh ) - if not x.startswith('#') ): + for line in (x for x in (y.strip() for y in fh) + if not x.startswith('#')): vals = line.split('=', 1) From a88ceb50c90e31a7a4fce9250b98e2d63ca339da Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 17:28:31 +0200 Subject: [PATCH 10/56] Migrate tests from StringIO.StringIO to io.StringIO This API change requires switching literals from str to unicode. Signed-off-by: Yann Dirson --- tests/test_ifrename_dynamic.py | 22 ++++----- tests/test_ifrename_logic.py | 12 ++--- tests/test_ifrename_static.py | 90 +++++++++++++++++----------------- 3 files changed, 59 insertions(+), 65 deletions(-) diff --git a/tests/test_ifrename_dynamic.py b/tests/test_ifrename_dynamic.py index 1eb28a74..6313b861 100644 --- a/tests/test_ifrename_dynamic.py +++ b/tests/test_ifrename_dynamic.py @@ -1,12 +1,10 @@ +from __future__ import unicode_literals import json import logging import unittest from copy import deepcopy -try: - import cStringIO as StringIO -except ImportError: - import StringIO +from io import StringIO from xcp.net.ifrename.dynamic import DynamicRules from xcp.net.ifrename.macpci import MACPCI @@ -16,7 +14,7 @@ class TestLoadAndParse(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): @@ -26,7 +24,7 @@ def tearDown(self): def test_null(self): - fd = StringIO.StringIO("") + fd = StringIO("") dr = DynamicRules(fd=fd) self.assertTrue(dr.load_and_parse()) @@ -36,7 +34,7 @@ def test_null(self): def test_empty(self): - fd = StringIO.StringIO( + fd = StringIO( '{"lastboot":[],"old":[]}' ) dr = DynamicRules(fd=fd) @@ -48,7 +46,7 @@ def test_empty(self): def test_one_invalid(self): - fd = StringIO.StringIO( + fd = StringIO( '{"lastboot":[["","",""]],"old":[]}' ) dr = DynamicRules(fd=fd) @@ -60,7 +58,7 @@ def test_one_invalid(self): def test_one_valid_lastboot(self): - fd = StringIO.StringIO( + fd = StringIO( '{"lastboot":[["01:23:45:67:89:0a","00:10.2","eth2"]],"old":[]}' ) dr = DynamicRules(fd=fd) @@ -74,7 +72,7 @@ def test_one_valid_lastboot(self): def test_one_valid_lastboot2(self): - fd = StringIO.StringIO( + fd = StringIO( '{"lastboot":[],"old":[["01:23:45:67:89:0a","00:10.2","eth2"]]}' ) dr = DynamicRules(fd=fd) @@ -88,7 +86,7 @@ def test_one_valid_lastboot2(self): class TestGenerate(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): @@ -149,7 +147,7 @@ def test_pci_missing(self): class TestSave(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index c74522eb..a1850c20 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -1,13 +1,11 @@ from __future__ import print_function +from __future__ import unicode_literals import logging import sys import unittest from copy import deepcopy -try: - import cStringIO as StringIO -except ImportError: - import StringIO +from io import StringIO from xcp.net.ifrename.logic import * from xcp.logger import LOG, openLog, closeLogs @@ -35,7 +33,7 @@ def apply_transactions(lst, trans): class TestSimpleLogic(unittest.TestCase): def setUp(self): - self.siobuff = StringIO.StringIO() + self.siobuff = StringIO() openLog(self.siobuff, logging.NOTSET) def tearDown(self): @@ -260,7 +258,7 @@ def test_1drule_1eth_already_complete(self): class TestUseCases(unittest.TestCase): def setUp(self): - self.siobuff = StringIO.StringIO() + self.siobuff = StringIO() openLog(self.siobuff, logging.NOTSET) def tearDown(self): @@ -534,7 +532,7 @@ def setUp(self): set to None and a tname set to the 'eth' """ - self.siobuff = StringIO.StringIO() + self.siobuff = StringIO() openLog(self.siobuff) diff --git a/tests/test_ifrename_static.py b/tests/test_ifrename_static.py index 16909142..9b11e380 100644 --- a/tests/test_ifrename_static.py +++ b/tests/test_ifrename_static.py @@ -1,11 +1,9 @@ +from __future__ import unicode_literals import logging import unittest from copy import deepcopy -try: - import cStringIO as StringIO -except ImportError: - import StringIO +from io import StringIO from xcp.net.ifrename.static import StaticRules from xcp.net.ifrename.macpci import MACPCI @@ -15,7 +13,7 @@ class TestLoadAndParse(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): @@ -37,7 +35,7 @@ def test_null(self): def test_empty(self): - fd = StringIO.StringIO("") + fd = StringIO("") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -46,7 +44,7 @@ def test_empty(self): def test_comment(self): - fd = StringIO.StringIO("#comment") + fd = StringIO("#comment") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -55,7 +53,7 @@ def test_comment(self): def test_comment_and_empty(self): - fd = StringIO.StringIO("\n # Another Comment\n ") + fd = StringIO("\n # Another Comment\n ") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -64,7 +62,7 @@ def test_comment_and_empty(self): def test_single_incorrect_mac(self): - fd = StringIO.StringIO('eth0:mac="foo"') + fd = StringIO('eth0:mac="foo"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -73,7 +71,7 @@ def test_single_incorrect_mac(self): def test_single_mac(self): - fd = StringIO.StringIO('eth0:mac="AB:CD:EF:AB:CD:EF"') + fd = StringIO('eth0:mac="AB:CD:EF:AB:CD:EF"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -82,7 +80,7 @@ def test_single_mac(self): def test_single_invalid_pci(self): - fd = StringIO.StringIO('eth0:pci="bar"') + fd = StringIO('eth0:pci="bar"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -91,7 +89,7 @@ def test_single_invalid_pci(self): def test_single_pci(self): - fd = StringIO.StringIO('eth0:pci="0000:00:00.1"') + fd = StringIO('eth0:pci="0000:00:00.1"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -100,7 +98,7 @@ def test_single_pci(self): def test_single_pci_0index(self): - fd = StringIO.StringIO('eth0:pci="0000:00:00.1[0]"') + fd = StringIO('eth0:pci="0000:00:00.1[0]"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -109,7 +107,7 @@ def test_single_pci_0index(self): def test_single_invalid_ppn(self): - fd = StringIO.StringIO('eth0:ppn="baz"') + fd = StringIO('eth0:ppn="baz"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -118,7 +116,7 @@ def test_single_invalid_ppn(self): def test_single_ppn_embedded(self): - fd = StringIO.StringIO('eth0:ppn="em2"') + fd = StringIO('eth0:ppn="em2"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -127,7 +125,7 @@ def test_single_ppn_embedded(self): def test_single_ppn_slot(self): - fd = StringIO.StringIO('eth0:ppn="p2p3"') + fd = StringIO('eth0:ppn="p2p3"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -136,7 +134,7 @@ def test_single_ppn_slot(self): def test_single_oldsytle_ppn_slot(self): # CA-82901 - Accept old-style PPNs but translate them to new-style - fd = StringIO.StringIO('eth0:ppn="pci2p3"') + fd = StringIO('eth0:ppn="pci2p3"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -145,7 +143,7 @@ def test_single_oldsytle_ppn_slot(self): def test_single_label(self): - fd = StringIO.StringIO('eth0:label="somestring"') + fd = StringIO('eth0:label="somestring"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -155,7 +153,7 @@ def test_single_label(self): class TestLoadAndParseGuess(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): @@ -165,7 +163,7 @@ def tearDown(self): def test_single_explicit_label(self): - fd = StringIO.StringIO("eth0=\"foo\"") + fd = StringIO("eth0=\"foo\"") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -174,7 +172,7 @@ def test_single_explicit_label(self): def test_single_implicit_label(self): - fd = StringIO.StringIO("eth0=foo") + fd = StringIO("eth0=foo") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -183,7 +181,7 @@ def test_single_implicit_label(self): def test_single_mac(self): - fd = StringIO.StringIO("eth0=00:00:00:00:00:00") + fd = StringIO("eth0=00:00:00:00:00:00") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -192,7 +190,7 @@ def test_single_mac(self): def test_single_pci(self): - fd = StringIO.StringIO("eth0=0000:00:00.0") + fd = StringIO("eth0=0000:00:00.0") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -201,7 +199,7 @@ def test_single_pci(self): def test_single_pci_index(self): - fd = StringIO.StringIO("eth0=0000:00:00.0[1]") + fd = StringIO("eth0=0000:00:00.0[1]") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -210,7 +208,7 @@ def test_single_pci_index(self): def test_single_ppn_embedded(self): - fd = StringIO.StringIO("eth0=em4") + fd = StringIO("eth0=em4") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -219,7 +217,7 @@ def test_single_ppn_embedded(self): def test_single_ppn_slot(self): - fd = StringIO.StringIO("eth0=p1p2") + fd = StringIO("eth0=p1p2") sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -230,7 +228,7 @@ def test_single_ppn_slot(self): class TestGenerate(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) self.state = [ @@ -251,7 +249,7 @@ def tearDown(self): def test_null(self): - fd = StringIO.StringIO('eth0:label="somestring"') + fd = StringIO('eth0:label="somestring"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) sr.generate([]) @@ -260,7 +258,7 @@ def test_null(self): def test_single_not_matching_state(self): - fd = StringIO.StringIO('eth0:label="somestring"') + fd = StringIO('eth0:label="somestring"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) sr.generate(self.state) @@ -269,7 +267,7 @@ def test_single_not_matching_state(self): def test_single_mac_matching(self): - fd = StringIO.StringIO('eth0:mac="01:23:45:67:89:0a"') + fd = StringIO('eth0:mac="01:23:45:67:89:0a"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -281,7 +279,7 @@ def test_single_mac_matching(self): def test_single_pci_matching(self): - fd = StringIO.StringIO('eth0:pci="0000:00:10.0"') + fd = StringIO('eth0:pci="0000:00:10.0"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -293,7 +291,7 @@ def test_single_pci_matching(self): def test_single_ppn_embedded_matching(self): - fd = StringIO.StringIO('eth0:ppn="em1"') + fd = StringIO('eth0:ppn="em1"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -305,7 +303,7 @@ def test_single_ppn_embedded_matching(self): def test_single_ppn_slot_matching(self): - fd = StringIO.StringIO('eth0:ppn="p2p2"') + fd = StringIO('eth0:ppn="p2p2"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -317,7 +315,7 @@ def test_single_ppn_slot_matching(self): def test_single_label_matching(self): - fd = StringIO.StringIO('eth0:label="Ethernet1"') + fd = StringIO('eth0:label="Ethernet1"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -330,8 +328,8 @@ def test_single_label_matching(self): def test_ppn_quirks(self): # Test case taken from example on CA-75599 - fd = StringIO.StringIO('eth0:ppn="em1"\n' - 'eth1:ppn="em2"') + fd = StringIO('eth0:ppn="em1"\n' + 'eth1:ppn="em2"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -350,7 +348,7 @@ def test_ppn_quirks(self): class TestMultiplePCI(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) self.state = [ MACPCI("c8:cb:b8:d3:0c:ca", "0000:03:00.0", kname="eth0", @@ -370,8 +368,8 @@ def tearDown(self): def test_pci_matching(self): - fd = StringIO.StringIO('eth0:pci="0000:04:00.0"\n' - 'eth1:pci="0000:04:00.0[1]"') + fd = StringIO('eth0:pci="0000:04:00.0"\n' + 'eth1:pci="0000:04:00.0[1]"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -384,8 +382,8 @@ def test_pci_matching(self): def test_pci_matching_invert(self): - fd = StringIO.StringIO('eth0:pci="0000:04:00.0[1]"\n' - 'eth1:pci="0000:04:00.0[0]"') + fd = StringIO('eth0:pci="0000:04:00.0[1]"\n' + 'eth1:pci="0000:04:00.0[0]"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -398,8 +396,8 @@ def test_pci_matching_invert(self): def test_pci_matching_mixed(self): - fd = StringIO.StringIO('eth0:ppn="em3"\n' - 'eth1:pci="0000:04:00.0[1]"') + fd = StringIO('eth0:ppn="em3"\n' + 'eth1:pci="0000:04:00.0[1]"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -412,8 +410,8 @@ def test_pci_matching_mixed(self): def test_pci_missing(self): - fd = StringIO.StringIO('eth0:pci="0000:03:00.0"\n' - 'eth4:pci="0000:05:00.0"') + fd = StringIO('eth0:pci="0000:03:00.0"\n' + 'eth4:pci="0000:05:00.0"') sr = StaticRules(fd = fd) self.assertTrue(sr.load_and_parse()) @@ -427,7 +425,7 @@ def test_pci_missing(self): class TestSave(unittest.TestCase): def setUp(self): - self.logbuf = StringIO.StringIO() + self.logbuf = StringIO() openLog(self.logbuf, logging.NOTSET) def tearDown(self): From 4d3681cf421e60d8175de87e8b338242e4f1c7b9 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 29 Jul 2022 11:59:33 +0200 Subject: [PATCH 11/56] cpiofile: drop CpioFileCompat.write() This implementation of CpioFileCompat.write cannot work with python 2.7, as adding new attributes to a ZipInfo triggers an exception. `tarfile.py` in Python 2.7 has a fix, but this method is clearly not used, so there's no reason to keep it, especially as its use of StringIO should likely switch to BytesIO and we would need to write tests for this. Also note that in python3 the TarFileCompat class is removed. We could probably just drop our CpioFileCompat completely, if we're sure noone uses it, but it likely only exists because there was a TarFileCompat. Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 79ffcd4e..fa9282cd 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1905,16 +1905,7 @@ def read(self, name): return self.cpiofile.extractfile(self.cpiofile.getmember(name)).read() def write(self, filename, arcname=None, compress_type=None): self.cpiofile.add(filename, arcname) - def writestr(self, zinfo, bts): - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - import calendar - zinfo.name = zinfo.filename - zinfo.size = zinfo.file_size - zinfo.mtime = calendar.timegm(zinfo.date_time) - self.cpiofile.addfile(zinfo, StringIO(bts)) + # deleted writestr method def close(self): self.cpiofile.close() #class CpioFileCompat From 8a291689edf4cf1eeb061a159840e4a60a509aa0 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 17:37:04 +0200 Subject: [PATCH 12/56] Switch integer division to py3-compatible "//" ... on locations reported by futurize Signed-off-by: Yann Dirson --- xcp/bootloader.py | 5 +++-- xcp/dom0.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 0602b822..7ae55e2c 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -24,6 +24,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import print_function +from __future__ import division import os import os.path import re @@ -519,7 +520,7 @@ def writeGrub(self, dst_file = None): print("default %d" % i, file=fh) break if self.timeout: - print("timeout %d" % (self.timeout / 10), file=fh) + print("timeout %d" % (self.timeout // 10), file=fh) for label in self.menu_order: m = self.menu[label] @@ -553,7 +554,7 @@ def writeGrub2(self, dst_file = None): else: print("set default='%s'" % str(self.default), file=fh) if self.timeout: - print("set timeout=%d" % (self.timeout / 10), file=fh) + print("set timeout=%d" % (self.timeout // 10), file=fh) boilerplate = getattr(self, 'boilerplate', [])[:] boilerplate.reverse() diff --git a/xcp/dom0.py b/xcp/dom0.py index 66a23a9e..086a683b 100644 --- a/xcp/dom0.py +++ b/xcp/dom0.py @@ -24,6 +24,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import +from __future__ import division import re from . import version @@ -40,7 +41,7 @@ def default_memory_v2(host_mem_kib): # # Add a bit extra to account for this. # - gb = (host_mem_kib + 256 * 1024) / 1024 / 1024 + gb = (host_mem_kib + 256 * 1024) // 1024 // 1024 if gb < 24: return 752 * 1024 @@ -62,7 +63,7 @@ def default_memory_v3(host_mem_kib): # # Add a bit extra to account for this. # - mb = (host_mem_kib + 256 * 1024) / 1024 + mb = (host_mem_kib + 256 * 1024) // 1024 # Give dom0 1 GiB + 5% of host memory, rounded to 16 MiB, limited to 8 GiB return min(1024 + int(mb * 0.05) & ~0xF, 8192) * 1024 From c410bab97703d3ed6c0ac668e79518d62d83d20e Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 10:43:08 +0200 Subject: [PATCH 13/56] xcp.accessor: upgrade urllib usage (futurize) This is the only usage we have of `future`, where we used `six` instead everywhere else. At least `future` has the advantage of being really easy to drop when we cull python2 support. Signed-off-by: Yann Dirson --- requirements-dev.txt | 1 + setup.py | 1 + xcp/accessor.py | 34 ++++++++++++++++++---------------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6bbf9447..b2799686 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,3 +7,4 @@ pytest pytest-cov # dependencies also in setup.py until they can be used six +future diff --git a/setup.py b/setup.py index e7881222..08c8200d 100644 --- a/setup.py +++ b/setup.py @@ -36,5 +36,6 @@ requires=[ 'branding', 'six', + 'future', ], ) diff --git a/xcp/accessor.py b/xcp/accessor.py index 7763cf96..f2cf6631 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -25,11 +25,13 @@ """accessor - provide common interface to access methods""" +from future import standard_library +standard_library.install_aliases() import ftplib import os import tempfile -import urllib -import urllib2 +import urllib.request, urllib.parse, urllib.error +import urllib.request, urllib.error, urllib.parse import urlparse import errno @@ -280,12 +282,12 @@ def start(self): username = self.url_parts.username password = self.url_parts.password if username: - username = urllib.unquote(username) + username = urllib.parse.unquote(username) if password: - password = urllib.unquote(password) + password = urllib.parse.unquote(password) self.ftp.login(username, password) - directory = urllib.unquote(self.url_parts.path[1:]) + directory = urllib.parse.unquote(self.url_parts.path[1:]) if directory != '': logger.debug("Changing to " + directory) self.ftp.cwd(directory) @@ -305,7 +307,7 @@ def access(self, path): try: logger.debug("Testing "+path) self._cleanup() - url = urllib.unquote(path) + url = urllib.parse.unquote(path) if self.ftp.size(url) is not None: return True @@ -330,7 +332,7 @@ def access(self, path): def openAddress(self, address): logger.debug("Opening "+address) self._cleanup() - url = urllib.unquote(address) + url = urllib.parse.unquote(address) self.ftp.voidcmd('TYPE I') s = self.ftp.transfercmd('RETR ' + url).makefile('rb') @@ -339,7 +341,7 @@ def openAddress(self, address): def writeFile(self, in_fh, out_name): self._cleanup() - fname = urllib.unquote(out_name) + fname = urllib.parse.unquote(out_name) logger.debug("Storing as " + fname) self.ftp.storbinary('STOR ' + fname, in_fh) @@ -356,23 +358,23 @@ def __init__(self, baseAddress, ro): if self.url_parts.username: username = self.url_parts.username if username is not None: - username = urllib.unquote(self.url_parts.username) + username = urllib.parse.unquote(self.url_parts.username) password = self.url_parts.password if password is not None: - password = urllib.unquote(self.url_parts.password) - self.passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + password = urllib.parse.unquote(self.url_parts.password) + self.passman = urllib.request.HTTPPasswordMgrWithDefaultRealm() self.passman.add_password(None, self.url_parts.hostname, username, password) - self.authhandler = urllib2.HTTPBasicAuthHandler(self.passman) - self.opener = urllib2.build_opener(self.authhandler) - urllib2.install_opener(self.opener) + self.authhandler = urllib.request.HTTPBasicAuthHandler(self.passman) + self.opener = urllib.request.build_opener(self.authhandler) + urllib.request.install_opener(self.opener) self.baseAddress = rebuild_url(self.url_parts) def openAddress(self, address): try: - urlFile = urllib2.urlopen(os.path.join(self.baseAddress, address)) - except urllib2.HTTPError as e: + urlFile = urllib.request.urlopen(os.path.join(self.baseAddress, address)) + except urllib.error.HTTPError as e: self.lastError = e.code return False return urlFile From 60ecacddc480fbd0bf3697d64c8294bd0e15b928 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:17:42 +0200 Subject: [PATCH 14/56] xcp.accessor: more urllib upgrading These changes come from a `futurize -2` or `2to3` run, but are for some reason not applied by the `urllib` fix itself. Signed-off-by: Yann Dirson --- xcp/accessor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index f2cf6631..be98e545 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -32,7 +32,7 @@ import tempfile import urllib.request, urllib.parse, urllib.error import urllib.request, urllib.error, urllib.parse -import urlparse +import urllib.parse import errno import xcp.mount as mount @@ -252,14 +252,14 @@ def rebuild_url(url_parts): host = url_parts.hostname if url_parts.port: host += ':' + str(url_parts.port) - return urlparse.urlunsplit( + return urllib.parse.urlunsplit( (url_parts.scheme, host, url_parts.path, '', '')) class FTPAccessor(Accessor): def __init__(self, baseAddress, ro): super(FTPAccessor, self).__init__(ro) - self.url_parts = urlparse.urlsplit(baseAddress, allow_fragments=False) + self.url_parts = urllib.parse.urlsplit(baseAddress, allow_fragments=False) self.start_count = 0 self.cleanup = False self.ftp = None @@ -312,7 +312,7 @@ def access(self, path): if self.ftp.size(url) is not None: return True lst = self.ftp.nlst(os.path.dirname(url)) - return os.path.basename(url) in map(os.path.basename, lst) + return os.path.basename(url) in list(map(os.path.basename, lst)) except IOError as e: if e.errno == errno.EIO: self.lastError = 5 @@ -353,7 +353,7 @@ class HTTPAccessor(Accessor): def __init__(self, baseAddress, ro): assert ro super(HTTPAccessor, self).__init__(ro) - self.url_parts = urlparse.urlsplit(baseAddress, allow_fragments=False) + self.url_parts = urllib.parse.urlsplit(baseAddress, allow_fragments=False) if self.url_parts.username: username = self.url_parts.username @@ -391,7 +391,7 @@ def __repr__(self): } def createAccessor(baseAddress, *args): - url_parts = urlparse.urlsplit(baseAddress, allow_fragments=False) + url_parts = urllib.parse.urlsplit(baseAddress, allow_fragments=False) assert url_parts.scheme in SUPPORTED_ACCESSORS.keys() return SUPPORTED_ACCESSORS[url_parts.scheme](baseAddress, *args) From 422993db07c09e5c8f667e3365fb7a7ca305b9da Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 13:50:02 +0200 Subject: [PATCH 15/56] Use new-style classes even in python2 (2to3) Signed-off-by: Yann Dirson --- xcp/cmd.py | 2 +- xcp/repository.py | 2 +- xcp/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/cmd.py b/xcp/cmd.py index ff039484..bbd94656 100644 --- a/xcp/cmd.py +++ b/xcp/cmd.py @@ -56,7 +56,7 @@ def runCmd(command, with_stdout = False, with_stderr = False, inputtext = None): return rv, err return rv -class OutputCache: +class OutputCache(object): def __init__(self): self.cache = {} diff --git a/xcp/repository.py b/xcp/repository.py index 98b39683..7b4c1599 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -33,7 +33,7 @@ import xcp.version as version import xcp.xmlunwrap as xmlunwrap -class Package: +class Package(object): pass class BzippedPackage(Package): diff --git a/xcp/version.py b/xcp/version.py index aaa7fcd6..2daf65bf 100644 --- a/xcp/version.py +++ b/xcp/version.py @@ -25,7 +25,7 @@ """version - version comparison methods""" -class Version: +class Version(object): def __init__(self, ver, build = None): self.ver = ver self.build = build From 764fc9b95f639f703949e06d8479da420b6e74d3 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 13:59:03 +0200 Subject: [PATCH 16/56] futurize: deal with xrange() Changes driven by "futurize -f libfuturize.fixes.fix_xrange_with_import", but not using "from builtins import range", which only brings negligible performance degradation on python2. Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 6 +++--- xcp/net/mac.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index fa9282cd..88d06045 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -116,7 +116,7 @@ def copyfileobj(src, dst, length=None): bufsize = 16 * 1024 blocks, remainder = divmod(length, bufsize) - for _ in xrange(blocks): + for b in range(blocks): buf = src.read(bufsize) if len(buf) < bufsize: raise IOError("end of file reached") @@ -398,7 +398,7 @@ def seek(self, pos=0): """ if pos - self.pos >= 0: blocks, remainder = divmod(pos - self.pos, self.bufsize) - for _ in xrange(blocks): + for i in range(blocks): self.read(self.bufsize) self.read(remainder) else: @@ -1794,7 +1794,7 @@ def _getmember(self, name, cpioinfo=None): else: end = members.index(cpioinfo) - for i in xrange(end - 1, -1, -1): + for i in range(end - 1, -1, -1): if name == members[i].name: return members[i] diff --git a/xcp/net/mac.py b/xcp/net/mac.py index 9b042c8f..b512a565 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -89,7 +89,7 @@ def _set_from_str_octets(self, octets): self.octets = [ int(i, 16) for i in octets ] self.integer = long(sum(t[0] << t[1] for t in - zip(self.octets, xrange(40, -1, -8)))) + zip(self.octets, range(40, -1, -8)))) def _set_from_str_quads(self, quads): """Private helper""" @@ -101,7 +101,7 @@ def _set_from_str_quads(self, quads): self.octets.extend([(quad >> 8) & 0xff, quad & 0xff]) self.integer = long(sum(t[0] << t[1] for t in - zip(self.octets, xrange(40, -1, -8)))) + zip(self.octets, range(40, -1, -8)))) def is_unicast(self): """is this a unicast address?""" From 4728f42af9fdc17914897e256bc433b5d84d235a Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 14:06:14 +0200 Subject: [PATCH 17/56] python3: remove casts from "int" to "long" There has not been any distinction at the python code level between `long` and `int` since 2.2. `2to3 -f long` would replace them with casts to `int` instead, which would just be a no-op. Signed-off-by: Yann Dirson --- xcp/net/mac.py | 8 ++++---- xcp/repository.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/xcp/net/mac.py b/xcp/net/mac.py index b512a565..47e586c0 100644 --- a/xcp/net/mac.py +++ b/xcp/net/mac.py @@ -88,8 +88,8 @@ def _set_from_str_octets(self, octets): raise ValueError("Expected 6 octets, got %d" % len(octets)) self.octets = [ int(i, 16) for i in octets ] - self.integer = long(sum(t[0] << t[1] for t in - zip(self.octets, range(40, -1, -8)))) + self.integer = sum(t[0] << t[1] for t in + zip(self.octets, range(40, -1, -8))) def _set_from_str_quads(self, quads): """Private helper""" @@ -100,8 +100,8 @@ def _set_from_str_quads(self, quads): for quad in ( int(i, 16) for i in quads ): self.octets.extend([(quad >> 8) & 0xff, quad & 0xff]) - self.integer = long(sum(t[0] << t[1] for t in - zip(self.octets, range(40, -1, -8)))) + self.integer = sum(t[0] << t[1] for t in + zip(self.octets, range(40, -1, -8))) def is_unicast(self): """is this a unicast address?""" diff --git a/xcp/repository.py b/xcp/repository.py index 7b4c1599..4bb98f8b 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -46,7 +46,7 @@ def __init__(self, repository, label, size, md5sum, optional, fname, root): self.optional, self.filename, self.destination - ) = ( repository, label, long(size), md5sum, (optional==True), fname, root ) + ) = ( repository, label, size, md5sum, (optional==True), fname, root ) def __repr__(self): return "" % self.label @@ -61,7 +61,7 @@ def __init__(self, repository, label, size, md5sum, optional, fname, options): self.optional, self.filename, self.options - ) = ( repository, label, long(size), md5sum, (optional==True), fname, options ) + ) = ( repository, label, size, md5sum, (optional==True), fname, options ) def __repr__(self): return "" % self.label @@ -76,7 +76,7 @@ def __init__(self, repository, label, size, md5sum, fname, kernel, options): self.filename, self.kernel, self.options - ) = ( repository, label, long(size), md5sum, fname, kernel, options ) + ) = ( repository, label, size, md5sum, fname, kernel, options ) def __repr__(self): return "" % (self.label, self.kernel) @@ -90,7 +90,7 @@ def __init__(self, repository, label, size, md5sum, fname, root): self.md5sum, self.filename, self.destination - ) = ( repository, label, long(size), md5sum, fname, root ) + ) = ( repository, label, size, md5sum, fname, root ) def __repr__(self): return "" % self.label @@ -103,7 +103,7 @@ def __init__(self, repository, label, size, md5sum, fname): self.size, self.md5sum, self.filename - ) = ( repository, label, long(size), md5sum, fname ) + ) = ( repository, label, size, md5sum, fname ) def __repr__(self): return "" % self.label From 05e65da298f6c15d1d5c15c947be68ab7883858a Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 14:37:13 +0200 Subject: [PATCH 18/56] python3: change float cast from "long" to "int" Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 88d06045..d070d3f7 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -303,7 +303,7 @@ def _init_write_gz(self): -self.zlib.MAX_WBITS, self.zlib.DEF_MEM_LEVEL, 0) - timestamp = struct.pack(" Date: Mon, 18 Jul 2022 14:34:57 +0200 Subject: [PATCH 19/56] 2to3: rename "next" methods to "__next__", use six.Iterator for portability Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index d070d3f7..b53b7bd4 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -51,6 +51,8 @@ import struct import copy +import six + if sys.platform == 'mac': # This module needs work for MacOS9, especially in the area of pathname # handling. In many places it is assumed a simple substitution of / by the @@ -917,7 +919,7 @@ def isdev(self): return (stat.S_ISCHR(self.mode) or stat.S_ISBLK(self.mode)) # class CpioInfo -class CpioFile(object): +class CpioFile(six.Iterator): """The CpioFile Class provides an interface to cpio archives. """ @@ -1697,7 +1699,7 @@ def utime(self, cpioinfo, cpiogetpath): raise ExtractError("could not change modification time") #-------------------------------------------------------------------------- - def next(self): + def __next__(self): """Return the next member of the archive as a CpioInfo object, when CpioFile is opened for reading. Return None if there is no more available. @@ -1832,7 +1834,7 @@ def _dbg(self, level, msg): print(msg, file=sys.stderr) # class CpioFile -class CpioIter(object): +class CpioIter(six.Iterator): """Iterator Class. for cpioinfo in CpioFile(...): @@ -1848,7 +1850,7 @@ def __iter__(self): """Return iterator object. """ return self - def next(self): + def __next__(self): """Return the next item using CpioFile's next() method. When all members have been read, set CpioFile as _loaded. """ From 1b549180290ecd03c5bf09cfa58653470f6d2562 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 15:46:55 +0200 Subject: [PATCH 20/56] futurize: convert "iteritems()" to "items()", all negligible perf impact Signed-off-by: Yann Dirson --- xcp/net/ifrename/dynamic.py | 2 +- xcp/net/ifrename/static.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index befab65a..aa9ff077 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -170,7 +170,7 @@ def generate(self, state): LOG.warning("Discovered physical policy naming quirks in provided " "state. Disabling 'method=ppn' generation") - for target, (method, value) in self.formulae.iteritems(): + for target, (method, value) in self.formulae.items(): if method == "mac": diff --git a/xcp/net/ifrename/static.py b/xcp/net/ifrename/static.py index 615c2073..be1e695d 100644 --- a/xcp/net/ifrename/static.py +++ b/xcp/net/ifrename/static.py @@ -175,7 +175,7 @@ def load_and_parse(self): # If we need to guess the method from the value if method == "guess": - for k, v in StaticRules.validators.iteritems(): + for k, v in StaticRules.validators.items(): if v.match(value) is not None: method = k break @@ -222,7 +222,7 @@ def generate(self, state): LOG.warning("Discovered physical policy naming quirks in provided " "state. Disabling 'method=ppn' generation") - for target, (method, value) in self.formulae.iteritems(): + for target, (method, value) in self.formulae.items(): if method == "mac": From 48f6d6b3865f5bb2c9b25ebb47bc2928392523f5 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:29:51 +0200 Subject: [PATCH 21/56] Apply fixes from `2to3 -f map` that are useful even when suboptimal Only exclude a change to `xcp.accessor`. Signed-off-by: Yann Dirson --- xcp/bootloader.py | 4 ++-- xcp/cpiofile.py | 2 +- xcp/net/ifrename/logic.py | 24 +++++++++++------------- xcp/net/ifrename/util.py | 2 +- xcp/repository.py | 6 ++---- xcp/version.py | 2 +- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 7ae55e2c..819a417a 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -152,7 +152,7 @@ def readExtLinux(cls, src_file): # els[2] contains tboot args, hypervisor, # hypervisor args, kernel, # kernel args & initrd - args = map(lambda x: x.strip(), els[2].split('---')) + args = [x.strip() for x in els[2].split('---')] if len(args) == 4: hypervisor = args[1].split(None, 1) kernel = args[2].split(None, 1) @@ -168,7 +168,7 @@ def readExtLinux(cls, src_file): elif 'xen' in els[1]: # els[2] contains hypervisor args, kernel, # kernel args & initrd - args = map(lambda x: x.strip(), els[2].split('---')) + args = [x.strip() for x in els[2].split('---')] if len(args) == 3: kernel = args[1].split(None, 1) if len(kernel) == 2: diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index b53b7bd4..cccb4878 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1893,7 +1893,7 @@ def __init__(self, fpath, mode="r", compression=CPIO_PLAIN): m.file_size = m.size m.date_time = time.gmtime(m.mtime)[:6] def namelist(self): - return map(lambda m: m.name, self.infolist()) + return [m.name for m in self.infolist()] def infolist(self): return filter(lambda m: m.isreg(), self.cpiofile.getmembers()) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index 2328ca54..b4a52267 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -83,7 +83,7 @@ def __rename_nic(nic, name, transactions, cur_state): # Assert that name is valid assert VALID_ETH_NAME.match(name) is not None # Assert that name is not already taken in the current state - assert name not in map(lambda x: x.tname, cur_state) + assert name not in [x.tname for x in cur_state] # Given the previous assert, only un-renamed nics in the current state can # possibly alias the new name @@ -327,7 +327,7 @@ def rename_logic( static_rules, for fn in multinic_functions: newnics = util.get_nics_with_pci(filter(util.needs_renaming, cur_state), fn) - orders = sorted(map(lambda x: x.order, newnics)) + orders = sorted([x.order for x in newnics]) newnics.sort(key = lambda n: n.mac.integer) for nic, neworder in zip(newnics, orders): LOG.debug("NIC '%s' getting new order '%s'" % (nic, neworder)) @@ -336,10 +336,8 @@ def rename_logic( static_rules, # For completely new network cards which we have never seen before, work out # a safe new number to assign it ethnumbers = sorted( - map(lambda x: int(x[3:]), - filter(lambda x: VALID_ETH_NAME.match(x) is not None, - map(lambda x: x.tname or x.kname, - static_rules + cur_state + last_state)))) + [int(x[3:]) for x in filter(lambda x: VALID_ETH_NAME.match(x) is not None, + [x.tname or x.kname for x in static_rules + cur_state + last_state])]) if len(ethnumbers): nextethnum = ethnumbers[-1]+1 else: @@ -353,7 +351,7 @@ def rename_logic( static_rules, LOG.info("Renaming brand new nic '%s'" % (nic,)) if ( VALID_ETH_NAME.match(nic.kname) is not None and - nic.kname not in map(lambda x: x.tname, cur_state) ): + nic.kname not in [x.tname for x in cur_state] ): # User has been messing around with state files but not the udev # rules. If the eth name is still free, give it @@ -411,13 +409,13 @@ def rename( static_rules, "'eth'" % (e, )) # Verify no two static rules refer to the same eth name - _ = frozenset( map(lambda x: x.tname, static_rules) ) + _ = frozenset( [x.tname for x in static_rules] ) if len(_) != len(static_rules): raise StaticRuleError("Some static rules alias the same " "eth name") # Verify no two static rules refer to the same mac address - _ = frozenset( map(lambda x: x.mac, static_rules) ) + _ = frozenset( [x.mac for x in static_rules] ) if len(_) != len(static_rules): raise StaticRuleError("Some static rules alias the same MAC " "address") @@ -445,13 +443,13 @@ def rename( static_rules, # Verify no two entries of current state refer to the same eth name - _ = frozenset( map(lambda x: x.kname, cur_state) ) + _ = frozenset( [x.kname for x in cur_state] ) if len(_) != len(cur_state): raise CurrentStateError("Some entries of current state alias the " "same eth name") # Verify no two entries of current state refer to the same mac address - _ = frozenset( map(lambda x: x.mac, cur_state) ) + _ = frozenset( [x.mac for x in cur_state] ) if len(_) != len(cur_state): raise CurrentStateError("Some entries of current state alias the " "same MAC address") @@ -474,13 +472,13 @@ def rename( static_rules, # Verify no two entries of last state refer to the same eth name - _ = frozenset( map(lambda x: x.tname, last_state) ) + _ = frozenset( [x.tname for x in last_state] ) if len(_) != len(last_state): raise LastStateError("Some entries of last state alias the " "same eth name") # Verify no two entries of last state refer to the same mac address - _ = frozenset( map(lambda x: x.mac, last_state) ) + _ = frozenset( [x.mac for x in last_state] ) if len(_) != len(last_state): raise LastStateError("Some entries of last state alias the " "same MAC address") diff --git a/xcp/net/ifrename/util.py b/xcp/net/ifrename/util.py index 72c89fa3..3a7326f5 100644 --- a/xcp/net/ifrename/util.py +++ b/xcp/net/ifrename/util.py @@ -50,7 +50,7 @@ def get_nic_with_kname(nics, kname): def tname_free(nics, name): """Check that name is not taken by any nics""" - return name not in map(lambda x: x.tname, nics) + return name not in [x.tname for x in nics] def get_nic_with_mac(nics, mac): """Search for nic with mac""" diff --git a/xcp/repository.py b/xcp/repository.py index 4bb98f8b..32399a88 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -170,8 +170,7 @@ def __init__(self, access, base = ""): def isRepo(cls, access, base): """ Return whether there is a repository at base address 'base' accessible using accessor.""" - return False not in map(lambda x: access.access(os.path.join(base, x)), - [cls.TREEINFO_FILENAME, cls.REPOMD_FILENAME]) + return False not in [access.access(os.path.join(base, x)) for x in [cls.TREEINFO_FILENAME, cls.REPOMD_FILENAME]] @classmethod def _getVersion(cls, access, category): @@ -368,8 +367,7 @@ def _create_package(self, node): def isRepo(cls, access, base): """ Return whether there is a repository at base address 'base' accessible using accessor.""" - return False not in map(lambda x: access.access(os.path.join(base, x)), - [cls.REPOSITORY_FILENAME, cls.PKGDATA_FILENAME]) + return False not in [access.access(os.path.join(base, x)) for x in [cls.REPOSITORY_FILENAME, cls.PKGDATA_FILENAME]] @classmethod def getRepoVer(cls, access): diff --git a/xcp/version.py b/xcp/version.py index 2daf65bf..37f9e30a 100644 --- a/xcp/version.py +++ b/xcp/version.py @@ -52,7 +52,7 @@ def from_string(cls, ver_str): if '-' in ver_str: ver_str, build = ver_str.split('-', 1) - ver = map(cls.intify, ver_str.split('.')) + ver = list(map(cls.intify, ver_str.split('.'))) return cls(ver, build) From e67cf34777d3c6b8a5c9b36d278c135725f50a1a Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 12:13:01 +0200 Subject: [PATCH 22/56] Apply all fixes from `2to3 -f filter` that are useful even when suboptimal Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 3 +-- xcp/net/ifrename/dynamic.py | 4 ++-- xcp/net/ifrename/logic.py | 10 ++++------ xcp/pci.py | 12 +++++------- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index cccb4878..d03a2ab7 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1895,8 +1895,7 @@ def __init__(self, fpath, mode="r", compression=CPIO_PLAIN): def namelist(self): return [m.name for m in self.infolist()] def infolist(self): - return filter(lambda m: m.isreg(), - self.cpiofile.getmembers()) + return [m for m in self.cpiofile.getmembers() if m.isreg()] def printdir(self): self.cpiofile.list() def testzip(self): diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index aa9ff077..0528baa7 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -262,8 +262,8 @@ def validate(entry): % (entry, e)) return False - lastboot = filter(validate, self.lastboot) - old = filter(validate, self.old) + lastboot = list(filter(validate, self.lastboot)) + old = list(filter(validate, self.old)) try: res += json.dumps({"lastboot": lastboot, "old": old}, diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index b4a52267..21cd628d 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -88,7 +88,7 @@ def __rename_nic(nic, name, transactions, cur_state): # Given the previous assert, only un-renamed nics in the current state can # possibly alias the new name aliased = util.get_nic_with_kname( - filter(lambda x: x.tname is None, cur_state), name) + [x for x in cur_state if x.tname is None], name) if aliased is None: # Using this rule will not alias another currently present NIC @@ -325,7 +325,7 @@ def rename_logic( static_rules, if len(multinic_functions): LOG.debug("New multi-nic logic - attempting to re-order") for fn in multinic_functions: - newnics = util.get_nics_with_pci(filter(util.needs_renaming, cur_state), + newnics = util.get_nics_with_pci(list(filter(util.needs_renaming, cur_state)), fn) orders = sorted([x.order for x in newnics]) newnics.sort(key = lambda n: n.mac.integer) @@ -336,8 +336,7 @@ def rename_logic( static_rules, # For completely new network cards which we have never seen before, work out # a safe new number to assign it ethnumbers = sorted( - [int(x[3:]) for x in filter(lambda x: VALID_ETH_NAME.match(x) is not None, - [x.tname or x.kname for x in static_rules + cur_state + last_state])]) + [int(x[3:]) for x in [x for x in [x.tname or x.kname for x in static_rules + cur_state + last_state] if VALID_ETH_NAME.match(x) is not None]]) if len(ethnumbers): nextethnum = ethnumbers[-1]+1 else: @@ -422,8 +421,7 @@ def rename( static_rules, if len(cur_state): # Filter out iBFT NICs - cur_state = filter(lambda x: VALID_IBFT_NAME.match(x.kname) is None, - cur_state) + cur_state = [x for x in cur_state if VALID_IBFT_NAME.match(x.kname) is None] # Verify types and properties of the list for e in cur_state: diff --git a/xcp/pci.py b/xcp/pci.py index 367e7dc5..85f82ec7 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -260,8 +260,7 @@ def __init__(self): stdout = subprocess.PIPE) for l in cmd.stdout: line = l.rstrip() - el = filter(lambda x: not x.startswith('-'), - line.replace('"','').split()) + el = [x for x in line.replace('"','').split() if not x.startswith('-')] self.devs[el[0]] = {'id': el[0], 'class': el[1][:2], 'subclass': el[1][2:], @@ -279,11 +278,11 @@ def findByClass(self, cls, subcls = None): [class1, class2, ... classN]""" if subcls: assert isinstance(cls, str) - return filter(lambda x: x['class'] == cls and - x['subclass'] == subcls, self.devs.values()) + return [x for x in self.devs.values() if x['class'] == cls and + x['subclass'] == subcls] else: assert isinstance(cls, list) - return filter(lambda x: x['class'] in cls, self.devs.values()) + return [x for x in self.devs.values() if x['class'] in cls] def findRelatedFunctions(self, dev): """ return other devices that share the same bus & slot""" @@ -291,8 +290,7 @@ def slot(dev): left, _ = dev.rsplit('.', 1) return left - return filter(lambda x: x != dev and slot(x) == slot(dev), - self.devs.keys()) + return [x for x in self.devs.keys() if x != dev and slot(x) == slot(dev)] def pci_sbdfi_to_nic(sbdfi, nics): From eede266b3cfb94f9985ffb47ef01bc5b79f9504c Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:45:52 +0200 Subject: [PATCH 23/56] Futurize fixes: use generator comprehensions when list ones are not needed 2to3 uses list comprehensions to replace `map` calls in many places where they're not needed, and would yield worse perf than `map`. Signed-off-by: Yann Dirson --- xcp/net/ifrename/logic.py | 28 +++++++++++++++------------- xcp/net/ifrename/util.py | 2 +- xcp/repository.py | 6 ++++-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index 21cd628d..19b5b5b7 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -83,12 +83,12 @@ def __rename_nic(nic, name, transactions, cur_state): # Assert that name is valid assert VALID_ETH_NAME.match(name) is not None # Assert that name is not already taken in the current state - assert name not in [x.tname for x in cur_state] + assert name not in (x.tname for x in cur_state) # Given the previous assert, only un-renamed nics in the current state can # possibly alias the new name aliased = util.get_nic_with_kname( - [x for x in cur_state if x.tname is None], name) + (x for x in cur_state if x.tname is None), name) if aliased is None: # Using this rule will not alias another currently present NIC @@ -325,9 +325,8 @@ def rename_logic( static_rules, if len(multinic_functions): LOG.debug("New multi-nic logic - attempting to re-order") for fn in multinic_functions: - newnics = util.get_nics_with_pci(list(filter(util.needs_renaming, cur_state)), - fn) - orders = sorted([x.order for x in newnics]) + newnics = util.get_nics_with_pci((x for x in cur_state if util.needs_renaming(x)), fn) + orders = sorted(x.order for x in newnics) newnics.sort(key = lambda n: n.mac.integer) for nic, neworder in zip(newnics, orders): LOG.debug("NIC '%s' getting new order '%s'" % (nic, neworder)) @@ -336,7 +335,10 @@ def rename_logic( static_rules, # For completely new network cards which we have never seen before, work out # a safe new number to assign it ethnumbers = sorted( - [int(x[3:]) for x in [x for x in [x.tname or x.kname for x in static_rules + cur_state + last_state] if VALID_ETH_NAME.match(x) is not None]]) + int(x[3:]) + for x in (x for x in (x.tname or x.kname + for x in static_rules + cur_state + last_state) + if VALID_ETH_NAME.match(x) is not None)) if len(ethnumbers): nextethnum = ethnumbers[-1]+1 else: @@ -350,7 +352,7 @@ def rename_logic( static_rules, LOG.info("Renaming brand new nic '%s'" % (nic,)) if ( VALID_ETH_NAME.match(nic.kname) is not None and - nic.kname not in [x.tname for x in cur_state] ): + nic.kname not in (x.tname for x in cur_state) ): # User has been messing around with state files but not the udev # rules. If the eth name is still free, give it @@ -408,13 +410,13 @@ def rename( static_rules, "'eth'" % (e, )) # Verify no two static rules refer to the same eth name - _ = frozenset( [x.tname for x in static_rules] ) + _ = frozenset(x.tname for x in static_rules) if len(_) != len(static_rules): raise StaticRuleError("Some static rules alias the same " "eth name") # Verify no two static rules refer to the same mac address - _ = frozenset( [x.mac for x in static_rules] ) + _ = frozenset(x.mac for x in static_rules) if len(_) != len(static_rules): raise StaticRuleError("Some static rules alias the same MAC " "address") @@ -441,13 +443,13 @@ def rename( static_rules, # Verify no two entries of current state refer to the same eth name - _ = frozenset( [x.kname for x in cur_state] ) + _ = frozenset(x.kname for x in cur_state) if len(_) != len(cur_state): raise CurrentStateError("Some entries of current state alias the " "same eth name") # Verify no two entries of current state refer to the same mac address - _ = frozenset( [x.mac for x in cur_state] ) + _ = frozenset(x.mac for x in cur_state) if len(_) != len(cur_state): raise CurrentStateError("Some entries of current state alias the " "same MAC address") @@ -470,13 +472,13 @@ def rename( static_rules, # Verify no two entries of last state refer to the same eth name - _ = frozenset( [x.tname for x in last_state] ) + _ = frozenset(x.tname for x in last_state) if len(_) != len(last_state): raise LastStateError("Some entries of last state alias the " "same eth name") # Verify no two entries of last state refer to the same mac address - _ = frozenset( [x.mac for x in last_state] ) + _ = frozenset(x.mac for x in last_state) if len(_) != len(last_state): raise LastStateError("Some entries of last state alias the " "same MAC address") diff --git a/xcp/net/ifrename/util.py b/xcp/net/ifrename/util.py index 3a7326f5..0aad71f1 100644 --- a/xcp/net/ifrename/util.py +++ b/xcp/net/ifrename/util.py @@ -50,7 +50,7 @@ def get_nic_with_kname(nics, kname): def tname_free(nics, name): """Check that name is not taken by any nics""" - return name not in [x.tname for x in nics] + return name not in (x.tname for x in nics) def get_nic_with_mac(nics, mac): """Search for nic with mac""" diff --git a/xcp/repository.py b/xcp/repository.py index 32399a88..e83e14f2 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -170,7 +170,8 @@ def __init__(self, access, base = ""): def isRepo(cls, access, base): """ Return whether there is a repository at base address 'base' accessible using accessor.""" - return False not in [access.access(os.path.join(base, x)) for x in [cls.TREEINFO_FILENAME, cls.REPOMD_FILENAME]] + return False not in (access.access(os.path.join(base, x)) + for x in [cls.TREEINFO_FILENAME, cls.REPOMD_FILENAME]) @classmethod def _getVersion(cls, access, category): @@ -367,7 +368,8 @@ def _create_package(self, node): def isRepo(cls, access, base): """ Return whether there is a repository at base address 'base' accessible using accessor.""" - return False not in [access.access(os.path.join(base, x)) for x in [cls.REPOSITORY_FILENAME, cls.PKGDATA_FILENAME]] + return False not in (access.access(os.path.join(base, x)) + for x in [cls.REPOSITORY_FILENAME, cls.PKGDATA_FILENAME]) @classmethod def getRepoVer(cls, access): From 610144ae155f80067896fcb5f7d4c5a959531d94 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:49:31 +0200 Subject: [PATCH 24/56] Futurize cleanups: use list comprehension not `list(map())` or `list(filter())` Looks like a bug/limitation in `2to3`. Signed-off-by: Yann Dirson --- xcp/net/ifrename/dynamic.py | 4 ++-- xcp/version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/net/ifrename/dynamic.py b/xcp/net/ifrename/dynamic.py index 0528baa7..33040ffd 100644 --- a/xcp/net/ifrename/dynamic.py +++ b/xcp/net/ifrename/dynamic.py @@ -262,8 +262,8 @@ def validate(entry): % (entry, e)) return False - lastboot = list(filter(validate, self.lastboot)) - old = list(filter(validate, self.old)) + lastboot = [x for x in self.lastboot if validate(x)] + old = [x for x in self.old if validate(x)] try: res += json.dumps({"lastboot": lastboot, "old": old}, diff --git a/xcp/version.py b/xcp/version.py index 37f9e30a..e19d3c96 100644 --- a/xcp/version.py +++ b/xcp/version.py @@ -52,7 +52,7 @@ def from_string(cls, ver_str): if '-' in ver_str: ver_str, build = ver_str.split('-', 1) - ver = list(map(cls.intify, ver_str.split('.'))) + ver = [cls.intify(i) for i in ver_str.split('.')] return cls(ver, build) From 7b03e6a6276deec39c9b5061c2bd5b61954fbbd2 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 12:28:15 +0200 Subject: [PATCH 25/56] Futurize cleanups: xcp.net.ifrename.logic: simplify nested comprehensions Signed-off-by: Yann Dirson --- xcp/net/ifrename/logic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index 19b5b5b7..ce7fbcd1 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -336,9 +336,8 @@ def rename_logic( static_rules, # a safe new number to assign it ethnumbers = sorted( int(x[3:]) - for x in (x for x in (x.tname or x.kname - for x in static_rules + cur_state + last_state) - if VALID_ETH_NAME.match(x) is not None)) + for x in (x.tname or x.kname for x in static_rules + cur_state + last_state) + if VALID_ETH_NAME.match(x) is not None) if len(ethnumbers): nextethnum = ethnumbers[-1]+1 else: From 62bb70a42c3492812924780582049fc035e3cb58 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 12:28:51 +0200 Subject: [PATCH 26/56] Futurize cleanups: line-wrapping fixes Signed-off-by: Yann Dirson --- xcp/pci.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xcp/pci.py b/xcp/pci.py index 85f82ec7..dc8d93d4 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -278,8 +278,7 @@ def findByClass(self, cls, subcls = None): [class1, class2, ... classN]""" if subcls: assert isinstance(cls, str) - return [x for x in self.devs.values() if x['class'] == cls and - x['subclass'] == subcls] + return [x for x in self.devs.values() if x['class'] == cls and x['subclass'] == subcls] else: assert isinstance(cls, list) return [x for x in self.devs.values() if x['class'] in cls] From 1a33f752879b991385ccb2f446ba1cf68c95fb21 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 11:18:53 +0200 Subject: [PATCH 27/56] Futurize cleanups: cleanup duplicated imports Signed-off-by: Yann Dirson --- xcp/accessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index be98e545..c1318002 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -30,8 +30,8 @@ import ftplib import os import tempfile -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse +import urllib.request +import urllib.error import urllib.parse import errno From e3dd1cc1e1bad1002b67d2e2477eb861d9fd0a2c Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:13:54 +0200 Subject: [PATCH 28/56] Futurize pylint complements: xcp.accessor: bring consistency to "openAddress" arguments Pylint would report: xcp/accessor.py:377: W0221: (arguments-differ), HTTPAccessor.openAddress: Parameters differ from overridden 'openAddress' method Signed-off-by: Yann Dirson --- xcp/accessor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index c1318002..a63ed6f7 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -69,7 +69,7 @@ def access(self, name): return True - def openAddress(self, name): + def openAddress(self, address): """should be overloaded""" pass @@ -97,9 +97,9 @@ def __init__(self, location, ro): super(FilesystemAccessor, self).__init__(ro) self.location = location - def openAddress(self, addr): + def openAddress(self, address): try: - file = open(os.path.join(self.location, addr), 'r') + file = open(os.path.join(self.location, address), 'r') except OSError as e: if e.errno == errno.EIO: self.lastError = 5 From 60f35c501b14cd40a3d95c0d275f4c3cf099d860 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:14:49 +0200 Subject: [PATCH 29/56] Futurize pylint complements: disable warnings triggered by "import future" Signed-off-by: Yann Dirson --- xcp/accessor.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index a63ed6f7..2891a6bc 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -25,14 +25,16 @@ """accessor - provide common interface to access methods""" +# pylint: disable=wrong-import-position,wrong-import-order from future import standard_library standard_library.install_aliases() + import ftplib import os import tempfile -import urllib.request -import urllib.error -import urllib.parse +import urllib.request # pylint: disable=import-error +import urllib.error # pylint: disable=import-error +import urllib.parse # pylint: disable=import-error import errno import xcp.mount as mount From b12aa7cc3b9881931eb931f5e06490d9adc8fa5d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:18:14 +0200 Subject: [PATCH 30/56] Futurize pylint complements: test_cpio: mark unused import as expected This one is only for checking feature availability. Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 4cb13825..defde234 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -46,7 +46,7 @@ def setUp(self): check_call("gzip -c < archive.cpio > archive.cpio.gz") check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") try: - import lzma + import lzma # pylint: disable=unused-variable self.doXZ = subprocess.call("xz --check=crc32 --lzma2=dict=1MiB < archive.cpio > archive.cpio.xz", shell=True) == 0 except Exception as ex: # FIXME will issue warning even if test_xz is not requested From 6d14ba81a6970c65e156ef3e3944cc8ba857b009 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:34:17 +0200 Subject: [PATCH 31/56] Futurize pylint complements: xcp.accessor: avoid redefining "file" builtin Signed-off-by: Yann Dirson --- xcp/accessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 2891a6bc..70306933 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -101,7 +101,7 @@ def __init__(self, location, ro): def openAddress(self, address): try: - file = open(os.path.join(self.location, address), 'r') + filehandle = open(os.path.join(self.location, address), 'r') except OSError as e: if e.errno == errno.EIO: self.lastError = 5 @@ -117,7 +117,7 @@ def openAddress(self, address): except Exception as e: self.lastError = 500 return False - return file + return filehandle class MountingAccessor(FilesystemAccessor): def __init__(self, mount_types, mount_source, mount_options = None): From a78340c53373d544f225d014d85f34aef9aeff62 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:37:42 +0200 Subject: [PATCH 32/56] Futurize pylint complements: Avoid useless ".keys()" when enumerating dicts Signed-off-by: Yann Dirson --- xcp/accessor.py | 2 +- xcp/net/ifrename/static.py | 3 +-- xcp/pci.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 70306933..4d092607 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -395,5 +395,5 @@ def __repr__(self): def createAccessor(baseAddress, *args): url_parts = urllib.parse.urlsplit(baseAddress, allow_fragments=False) - assert url_parts.scheme in SUPPORTED_ACCESSORS.keys() + assert url_parts.scheme in SUPPORTED_ACCESSORS return SUPPORTED_ACCESSORS[url_parts.scheme](baseAddress, *args) diff --git a/xcp/net/ifrename/static.py b/xcp/net/ifrename/static.py index be1e695d..c1b9b100 100644 --- a/xcp/net/ifrename/static.py +++ b/xcp/net/ifrename/static.py @@ -298,8 +298,7 @@ def write(self, header = True): if header: res += SAVE_HEADER - keys = list(set(( x for x in self.formulae.keys() - if x.startswith("eth") ))) + keys = list(set((x for x in self.formulae if x.startswith("eth")))) keys.sort(key=lambda x: int(x[3:])) for target in keys: diff --git a/xcp/pci.py b/xcp/pci.py index dc8d93d4..cff6d506 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -289,7 +289,7 @@ def slot(dev): left, _ = dev.rsplit('.', 1) return left - return [x for x in self.devs.keys() if x != dev and slot(x) == slot(dev)] + return [x for x in self.devs if x != dev and slot(x) == slot(dev)] def pci_sbdfi_to_nic(sbdfi, nics): From 5da778b6fdf08988047b48a69b266427da2b01dd Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:43:05 +0200 Subject: [PATCH 33/56] Futurize pylint complements: xcp.repository: honor singleton-comparison convention Signed-off-by: Yann Dirson --- xcp/repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcp/repository.py b/xcp/repository.py index e83e14f2..14877d5c 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -46,7 +46,7 @@ def __init__(self, repository, label, size, md5sum, optional, fname, root): self.optional, self.filename, self.destination - ) = ( repository, label, size, md5sum, (optional==True), fname, root ) + ) = (repository, label, size, md5sum, optional is True, fname, root) def __repr__(self): return "" % self.label @@ -61,7 +61,7 @@ def __init__(self, repository, label, size, md5sum, optional, fname, options): self.optional, self.filename, self.options - ) = ( repository, label, size, md5sum, (optional==True), fname, options ) + ) = (repository, label, size, md5sum, optional is True, fname, options) def __repr__(self): return "" % self.label From e6b3689c83141b6ceb225e0f687f5ee4b2f1603b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 20 Jul 2022 11:44:51 +0200 Subject: [PATCH 34/56] Futurize pylint complements: xcp.repository: locally disable too-few-public-methods Signed-off-by: Yann Dirson --- xcp/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/repository.py b/xcp/repository.py index 14877d5c..c084d1d8 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -33,7 +33,7 @@ import xcp.version as version import xcp.xmlunwrap as xmlunwrap -class Package(object): +class Package(object): # pylint: disable=too-few-public-methods pass class BzippedPackage(Package): From 3d35879d3fb21707b82b95b7485259c5f8f4e7e3 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:02:04 +0200 Subject: [PATCH 35/56] Futurize pylint complements: "print" continuation indentations, line lengths Signed-off-by: Yann Dirson --- tests/test_cpio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index defde234..628c54ca 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -47,7 +47,8 @@ def setUp(self): check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") try: import lzma # pylint: disable=unused-variable - self.doXZ = subprocess.call("xz --check=crc32 --lzma2=dict=1MiB < archive.cpio > archive.cpio.xz", shell=True) == 0 + self.doXZ = subprocess.call("xz --check=crc32 --lzma2=dict=1MiB" + " < archive.cpio > archive.cpio.xz", shell=True) == 0 except Exception as ex: # FIXME will issue warning even if test_xz is not requested warnings.warn("will not test cpio.xz: %s" % ex) From 84a013f0a61b24561856edeb1197675a53d0a9f4 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 15 Jul 2022 15:40:49 +0200 Subject: [PATCH 36/56] Futurize pylint complements: whitespace in expressions Signed-off-by: Yann Dirson --- xcp/accessor.py | 2 +- xcp/net/ifrename/logic.py | 4 ++-- xcp/pci.py | 2 +- xcp/repository.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/xcp/accessor.py b/xcp/accessor.py index 4d092607..6d057927 100644 --- a/xcp/accessor.py +++ b/xcp/accessor.py @@ -120,7 +120,7 @@ def openAddress(self, address): return filehandle class MountingAccessor(FilesystemAccessor): - def __init__(self, mount_types, mount_source, mount_options = None): + def __init__(self, mount_types, mount_source, mount_options=None): ro = isinstance(mount_options, list) and 'ro' in mount_options super(MountingAccessor, self).__init__(None, ro) diff --git a/xcp/net/ifrename/logic.py b/xcp/net/ifrename/logic.py index ce7fbcd1..e2a3484b 100644 --- a/xcp/net/ifrename/logic.py +++ b/xcp/net/ifrename/logic.py @@ -350,8 +350,8 @@ def rename_logic( static_rules, key=lambda x: x.order): LOG.info("Renaming brand new nic '%s'" % (nic,)) - if ( VALID_ETH_NAME.match(nic.kname) is not None and - nic.kname not in (x.tname for x in cur_state) ): + if (VALID_ETH_NAME.match(nic.kname) is not None and + nic.kname not in (x.tname for x in cur_state)): # User has been messing around with state files but not the udev # rules. If the eth name is still free, give it diff --git a/xcp/pci.py b/xcp/pci.py index cff6d506..1c8e081d 100644 --- a/xcp/pci.py +++ b/xcp/pci.py @@ -260,7 +260,7 @@ def __init__(self): stdout = subprocess.PIPE) for l in cmd.stdout: line = l.rstrip() - el = [x for x in line.replace('"','').split() if not x.startswith('-')] + el = [x for x in line.replace('"', '').split() if not x.startswith('-')] self.devs[el[0]] = {'id': el[0], 'class': el[1][:2], 'subclass': el[1][2:], diff --git a/xcp/repository.py b/xcp/repository.py index c084d1d8..ca284647 100644 --- a/xcp/repository.py +++ b/xcp/repository.py @@ -76,7 +76,7 @@ def __init__(self, repository, label, size, md5sum, fname, kernel, options): self.filename, self.kernel, self.options - ) = ( repository, label, size, md5sum, fname, kernel, options ) + ) = (repository, label, size, md5sum, fname, kernel, options) def __repr__(self): return "" % (self.label, self.kernel) @@ -90,7 +90,7 @@ def __init__(self, repository, label, size, md5sum, fname, root): self.md5sum, self.filename, self.destination - ) = ( repository, label, size, md5sum, fname, root ) + ) = (repository, label, size, md5sum, fname, root) def __repr__(self): return "" % self.label @@ -103,7 +103,7 @@ def __init__(self, repository, label, size, md5sum, fname): self.size, self.md5sum, self.filename - ) = ( repository, label, size, md5sum, fname ) + ) = (repository, label, size, md5sum, fname) def __repr__(self): return "" % self.label From 4cd6d2be1f0547be6cebfde9fc78855131373528 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 17:00:36 +0200 Subject: [PATCH 37/56] xcp.cpiofile: fix bad indentation Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index d03a2ab7..3c5dcce2 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1112,8 +1112,7 @@ def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9): fileobj = file(name, mode + "b") try: - t = cls.cpioopen(name, mode, - gzip.GzipFile(name, mode, compresslevel, fileobj)) + t = cls.cpioopen(name, mode, gzip.GzipFile(name, mode, compresslevel, fileobj)) except IOError: raise ReadError("not a gzip file") t._extfileobj = False From 0ccda70706ba29df90cf2f057d65f0fcb1accccb Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 17:07:40 +0200 Subject: [PATCH 38/56] xcp: get rid of useless blackslash line continuations Signed-off-by: Yann Dirson --- xcp/bootloader.py | 3 +-- xcp/cpiofile.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/xcp/bootloader.py b/xcp/bootloader.py index 819a417a..a1d19709 100644 --- a/xcp/bootloader.py +++ b/xcp/bootloader.py @@ -146,8 +146,7 @@ def readExtLinux(cls, src_file): title = l[1:].lstrip() elif keywrd == 'kernel' and len(els) > 1: kernel = els[1] - elif keywrd == 'append' and len(els) > 1 and \ - kernel == 'mboot.c32': + elif keywrd == 'append' and len(els) > 1 and kernel == 'mboot.c32': if 'tboot' in els[1]: # els[2] contains tboot args, hypervisor, # hypervisor args, kernel, diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 3c5dcce2..a3905786 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1310,8 +1310,7 @@ def list(self, verbose=True): print("%10s" % ("%d,%d" % (cpioinfo.devmajor, cpioinfo.devminor)), end=' ') else: print("%10d" % cpioinfo.size, end=' ') - print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(cpioinfo.mtime)[:6], end=' ') + print("%d-%02d-%02d %02d:%02d:%02d" % time.localtime(cpioinfo.mtime)[:6], end=' ') print(cpioinfo.name) From 9060d4b89cf29ad8054498a87817faafffccbe76 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 17:21:50 +0200 Subject: [PATCH 39/56] test_cpio: always use hashlib, came with python 2.5 Signed-off-by: Yann Dirson --- tests/test_cpio.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 628c54ca..57c15188 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -1,4 +1,5 @@ from __future__ import print_function +from hashlib import md5 import os import shutil import subprocess @@ -7,11 +8,6 @@ from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED -try: - from hashlib import md5 -except: - from md5 import md5 - def writeRandomFile(fn, size, start='', add='a'): f = open(fn, 'wb') m = md5() From 9a5bb61e30f3e757e2b21e837bda28d7258b36d4 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 16:45:37 +0200 Subject: [PATCH 40/56] test_cpio: use context manager to avoid forgetting to close some files Signed-off-by: Yann Dirson --- tests/test_cpio.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 57c15188..b43e1764 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -37,7 +37,8 @@ def setUp(self): shutil.rmtree('archive', True) os.mkdir('archive') writeRandomFile('archive/data', 10491) - self.md5data = md5(open('archive/data').read()).hexdigest() + with open('archive/data') as fd: + self.md5data = md5(fd.read()).hexdigest() check_call("find archive | cpio -o -H newc > archive.cpio") check_call("gzip -c < archive.cpio > archive.cpio.gz") check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") @@ -78,7 +79,8 @@ def archiveCreate(self, fn, fmt='w'): os.unlink(fn) arc = CpioFile.open(fn, fmt) f = arc.getcpioinfo('archive/data') - arc.addfile(f, open('archive/data')) + with open('archive/data') as fd: + arc.addfile(f, fd) # test recursively add "." os.chdir('archive') arc.add(".") From ce2af04898735753f742e6e18a8d1a9df21a6487 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 16:46:15 +0200 Subject: [PATCH 41/56] test_cpio: use context manager for other "open" calls for consistency Signed-off-by: Yann Dirson --- tests/test_cpio.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index b43e1764..5af86b92 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -9,18 +9,17 @@ from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED def writeRandomFile(fn, size, start='', add='a'): - f = open(fn, 'wb') - m = md5() - m.update(start) - assert(len(add) != 0) - while size > 0: - d = m.digest() - if size < len(d): - d=d[:size] - f.write(d) - size -= len(d) - m.update(add) - f.close() + with open(fn, 'wb') as f: + m = md5() + m.update(start) + assert(len(add) != 0) + while size > 0: + d = m.digest() + if size < len(d): + d=d[:size] + f.write(d) + size -= len(d) + m.update(add) def check_call(cmd): @@ -89,10 +88,9 @@ def archiveCreate(self, fn, fmt='w'): arc.close() # special case for XZ, test check type (crc32) if fmt.endswith('xz'): - f = open(fn, 'rb') - f.seek(6) - self.assertEqual(f.read(2), '\x00\x01') - f.close() + with open(fn, 'rb') as f: + f.seek(6) + self.assertEqual(f.read(2), '\x00\x01') self.archiveExtract(fn) def doArchive(self, fn, fmt=None): From 49bf4760946f5b9c2e3f71c848f0ab376eeaedcf Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 27 Jul 2022 11:07:55 +0200 Subject: [PATCH 42/56] tests: avoid useless try/except Tests already fail on uncaught exception, and those except blocks are just adding dead code to tests. Signed-off-by: Yann Dirson --- tests/test_ifrename_dynamic.py | 5 +---- tests/test_ifrename_logic.py | 22 +++------------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/tests/test_ifrename_dynamic.py b/tests/test_ifrename_dynamic.py index 6313b861..1cc95e39 100644 --- a/tests/test_ifrename_dynamic.py +++ b/tests/test_ifrename_dynamic.py @@ -160,10 +160,7 @@ def test_one_invalid_lastboot(self): dr = DynamicRules() dr.lastboot = [["foo", "bar", "baz"]] - try: - json.loads(dr.write(False)) - except Exception: - self.fail() + json.loads(dr.write(False)) def test_one_ibft_lastboot(self): diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index a1850c20..670aaade 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -554,22 +554,8 @@ def tearDown(self): self.siobuff.close() - def assertNotRaises(self, excp, fn, *argl, **kwargs): - """Because unittest.TestCase seems to be missing this functionality""" - try: - fn(*argl, **kwargs) - except excp as e: - self.fail("function raised %s unexpectedly: %s" - % (excp, e)) - def test_srule_eth_unaliased(self): - - self.assertNotRaises(StaticRuleError, - rename, - [self.s111], - [], - [], - []) + rename([self.s111], [], [], []) def test_srule_eth_alias(self): """ @@ -636,8 +622,7 @@ def test_laststate_input(self): self.assertRaises(LastStateError, rename, [], [], [self.c123], []) - self.assertNotRaises(LastStateError, rename, - [], [], [self.s123], []) + rename([], [], [self.s123], []) def test_oldstate_input(self): """ @@ -648,5 +633,4 @@ def test_oldstate_input(self): self.assertRaises(OldStateError, rename, [], [], [], [self.c123]) - self.assertNotRaises(OldStateError, rename, - [], [], [], [self.s123]) + rename([], [], [], [self.s123]) From 8437deb184a100dbf20e8efa8452a2bc6715067b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 27 Jul 2022 11:17:47 +0200 Subject: [PATCH 43/56] tests: use TestCase.assertRaises as context manager Makes for much more readable tests. Signed-off-by: Yann Dirson --- tests/test_ifrename_logic.py | 24 ++++++++++++------------ tests/test_mac.py | 30 ++++++++++++++++++++---------- tests/test_pci.py | 13 ++++++++----- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/tests/test_ifrename_logic.py b/tests/test_ifrename_logic.py index 670aaade..3bb7a911 100644 --- a/tests/test_ifrename_logic.py +++ b/tests/test_ifrename_logic.py @@ -568,8 +568,8 @@ def test_srule_eth_alias(self): [self.s112, self.s222, self.s331] ] for i in srule_inputs: - self.assertRaises(StaticRuleError, rename, - i, [], [], []) + with self.assertRaises(StaticRuleError): + rename(i, [], [], []) def test_srule_mac_alias(self): """ @@ -582,8 +582,8 @@ def test_srule_mac_alias(self): [self.s211, self.s222, self.s133] ] for i in srule_inputs: - self.assertRaises(StaticRuleError, rename, - i, [], [], []) + with self.assertRaises(StaticRuleError): + rename(i, [], [], []) def test_curstate_eth_alias(self): """ @@ -596,8 +596,8 @@ def test_curstate_eth_alias(self): [self.c112, self.c222, self.c331] ] for i in curstate_inputs: - self.assertRaises(CurrentStateError, rename, - [], i, [], []) + with self.assertRaises(CurrentStateError): + rename([], i, [], []) def test_curstate_mac_alias(self): """ @@ -610,8 +610,8 @@ def test_curstate_mac_alias(self): [self.c211, self.c222, self.c133] ] for i in curstate_inputs: - self.assertRaises(CurrentStateError, rename, - [], i, [], []) + with self.assertRaises(CurrentStateError): + rename([], i, [], []) def test_laststate_input(self): """ @@ -619,8 +619,8 @@ def test_laststate_input(self): (xx:xx.x,xx:xx:xx:xx:xx:xx)->eth## """ - self.assertRaises(LastStateError, rename, - [], [], [self.c123], []) + with self.assertRaises(LastStateError): + rename([], [], [self.c123], []) rename([], [], [self.s123], []) @@ -630,7 +630,7 @@ def test_oldstate_input(self): (xx:xx.x,xx:xx:xx:xx:xx:xx)->eth## """ - self.assertRaises(OldStateError, rename, - [], [], [], [self.c123]) + with self.assertRaises(OldStateError): + rename([], [], [], [self.c123]) rename([], [], [], [self.s123]) diff --git a/tests/test_mac.py b/tests/test_mac.py index 310e17e3..7976a3bf 100644 --- a/tests/test_mac.py +++ b/tests/test_mac.py @@ -11,18 +11,21 @@ class TestInvalidMAC(unittest.TestCase): def test_null_str(self): val = "" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_non_str(self): for val in [None, [], {}]: - self.assertRaises(TypeError, MAC, val) + with self.assertRaises(TypeError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_colon_too_few_octets(self): val = "00:00:00:00:00" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_colon_invalid_octets(self): @@ -32,27 +35,32 @@ def test_colon_invalid_octets(self): "12:34:56g:78:90:ab" "12:34::78:90:ab" ]: - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_colon_too_many_octets(self): val = "00:00:00:00:00:00:00" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dash_too_few_octetes(self): val = "00-00-00-00-00" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dash_too_many_octets(self): val = "00-00-00-00-00-00-00" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dotquad_too_few_quads(self): val = "0000.0000" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dotquad_invalid_quads(self): @@ -62,12 +70,14 @@ def test_dotquad_invalid_quads(self): "abcd.efgh.ijkl", "1234.-5678.90Ab" ]: - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) def test_dotquad_too_many_quads(self): val = "0000.0000.0000.0000" - self.assertRaises(ValueError, MAC, val) + with self.assertRaises(ValueError): + MAC(val) self.assertFalse(MAC.is_valid(val)) class TestValidMAC(unittest.TestCase): diff --git a/tests/test_pci.py b/tests/test_pci.py index ac03c375..1895e88a 100644 --- a/tests/test_pci.py +++ b/tests/test_pci.py @@ -7,11 +7,14 @@ class TestInvalid(unittest.TestCase): def test_invalid_types(self): - - self.assertRaises(TypeError, PCI, 0) - self.assertRaises(TypeError, PCI, (0,)) - self.assertRaises(TypeError, PCI, []) - self.assertRaises(TypeError, PCI, {}) + with self.assertRaises(TypeError): + PCI(0) + with self.assertRaises(TypeError): + PCI((0,)) + with self.assertRaises(TypeError): + PCI([]) + with self.assertRaises(TypeError): + PCI({}) def test_invalid_format(self): pass From c445fd85f3c28d46cb95a1a8f628b862f610b601 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 18 Jul 2022 16:51:52 +0200 Subject: [PATCH 44/56] Mark bytes literals and streams as such Signed-off-by: Yann Dirson --- tests/test_cpio.py | 8 ++--- xcp/cpiofile.py | 87 +++++++++++++++++++++++----------------------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 5af86b92..651a4204 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -8,7 +8,7 @@ from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED -def writeRandomFile(fn, size, start='', add='a'): +def writeRandomFile(fn, size, start=b'', add=b'a'): with open(fn, 'wb') as f: m = md5() m.update(start) @@ -36,7 +36,7 @@ def setUp(self): shutil.rmtree('archive', True) os.mkdir('archive') writeRandomFile('archive/data', 10491) - with open('archive/data') as fd: + with open('archive/data', 'rb') as fd: self.md5data = md5(fd.read()).hexdigest() check_call("find archive | cpio -o -H newc > archive.cpio") check_call("gzip -c < archive.cpio > archive.cpio.gz") @@ -78,7 +78,7 @@ def archiveCreate(self, fn, fmt='w'): os.unlink(fn) arc = CpioFile.open(fn, fmt) f = arc.getcpioinfo('archive/data') - with open('archive/data') as fd: + with open('archive/data', 'rb') as fd: arc.addfile(f, fd) # test recursively add "." os.chdir('archive') @@ -90,7 +90,7 @@ def archiveCreate(self, fn, fmt='w'): if fmt.endswith('xz'): with open(fn, 'rb') as f: f.seek(6) - self.assertEqual(f.read(2), '\x00\x01') + self.assertEqual(f.read(2), b'\x00\x01') self.archiveExtract(fn) def doArchive(self, fn, fmt=None): diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index a3905786..c2728940 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -72,9 +72,9 @@ # cpio constants #--------------------------------------------------------- MAGIC_NEWC = 0x070701 # magic for SVR4 portable format (no CRC) -TRAILER_NAME = "TRAILER!!!" # filename in final member +TRAILER_NAME = b"TRAILER!!!" # filename in final member WORDSIZE = 4 # pad size -NUL = "\0" # the null character +NUL = b"\0" # the null character BLOCKSIZE = 512 # length of processing blocks HEADERSIZE_SVR4 = 110 # length of fixed header @@ -255,7 +255,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.comptype = comptype self.fileobj = fileobj self.bufsize = bufsize - self.buf = "" + self.buf = b"" self.pos = 0 self.closed = False @@ -265,7 +265,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): except ImportError: raise CompressionError("zlib module is not available") self.zlib = zlib - self.crc = zlib.crc32("") + self.crc = zlib.crc32(b"") if mode == "r": self._init_read_gz() else: @@ -277,7 +277,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): except ImportError: raise CompressionError("bz2 module is not available") if mode == "r": - self.dbuf = "" + self.dbuf = b"" self.cmp = bz2.BZ2Decompressor() else: self.cmp = bz2.BZ2Compressor() @@ -288,7 +288,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): except ImportError: raise CompressionError("lzma module is not available") if mode == "r": - self.dbuf = "" + self.dbuf = b"" self.cmp = lzma.LZMADecompressor() else: self.cmp = lzma.LZMACompressor() @@ -306,7 +306,7 @@ def _init_write_gz(self): self.zlib.DEF_MEM_LEVEL, 0) timestamp = struct.pack(" Date: Tue, 19 Jul 2022 15:40:45 +0200 Subject: [PATCH 45/56] test_cpio: add some docstrings Signed-off-by: Yann Dirson --- tests/test_cpio.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 651a4204..4773ef55 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -9,6 +9,7 @@ from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED def writeRandomFile(fn, size, start=b'', add=b'a'): + "Create a pseudo-random reproducible file from seeds `start` amd `add`" with open(fn, 'wb') as f: m = md5() m.update(start) From 0d5d52a33a150fab0d2c4162db3802aacc9406b2 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 16:21:02 +0200 Subject: [PATCH 46/56] test_cpio: generate reproducible archives This helps understanding what really differs between 2 platforms. Signed-off-by: Yann Dirson --- tests/test_cpio.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 4773ef55..b6568c88 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -39,7 +39,12 @@ def setUp(self): writeRandomFile('archive/data', 10491) with open('archive/data', 'rb') as fd: self.md5data = md5(fd.read()).hexdigest() - check_call("find archive | cpio -o -H newc > archive.cpio") + # fixed timestamps for cpio reproducibility + os.utime('archive/data', (0, 0)) + os.utime('archive', (0, 0)) + + check_call( + "find archive | cpio --reproducible -o -H newc > archive.cpio") check_call("gzip -c < archive.cpio > archive.cpio.gz") check_call("bzip2 -c < archive.cpio > archive.cpio.bz2") try: From 33c559ccd2e292c97bc00647e0ea7fd037c154de Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 16:22:43 +0200 Subject: [PATCH 47/56] test_cpio: avoid overwriting original archives with "archiveCreate()" When hunting for platform difference, it helps being able to tell that a difference comes from the original archives, now reproducible, or from the ones generated by the code under test. Signed-off-by: Yann Dirson --- tests/test_cpio.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index b6568c88..e41bd445 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -57,7 +57,7 @@ def setUp(self): self.doXZ = False def tearDown(self): - check_call("rm -rf archive archive.cpio* archive2") + check_call("rm -rf archive archive.cpio* archive2 archive2.cpio*") # TODO check with file (like 'r:*') # TODO use cat to check properly for pipes @@ -81,7 +81,8 @@ def archiveExtract(self, fn, fmt='r|*'): arc.close() def archiveCreate(self, fn, fmt='w'): - os.unlink(fn) + if os.path.exists(fn): + os.unlink(fn) arc = CpioFile.open(fn, fmt) f = arc.getcpioinfo('archive/data') with open('archive/data', 'rb') as fd: @@ -101,9 +102,11 @@ def archiveCreate(self, fn, fmt='w'): def doArchive(self, fn, fmt=None): self.archiveExtract(fn) - self.archiveCreate(fn, fmt is None and 'w' or 'w|%s' % fmt ) + fn2 = "archive2" + fn[len("archive"):] + print("creating %s" % fn2) + self.archiveCreate(fn2, fmt is None and 'w' or 'w|%s' % fmt) if not fmt is None: - self.archiveExtract(fn, 'r|%s' % fmt) + self.archiveExtract(fn2, 'r|%s' % fmt) def test_plain(self): self.doArchive('archive.cpio') From 67394aab0443822e15582d37b110892281e12861 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 16:46:35 +0200 Subject: [PATCH 48/56] test_cpio: improve test readability Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index e41bd445..e8e200e0 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -105,7 +105,7 @@ def doArchive(self, fn, fmt=None): fn2 = "archive2" + fn[len("archive"):] print("creating %s" % fn2) self.archiveCreate(fn2, fmt is None and 'w' or 'w|%s' % fmt) - if not fmt is None: + if fmt is not None: self.archiveExtract(fn2, 'r|%s' % fmt) def test_plain(self): From 225541b7a57ef77015f43b3d5de29bfa80baac9a Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 17:13:11 +0200 Subject: [PATCH 49/56] test_cpio: expect different xz flags depending on python version References: * xz file-format spec: https://tukaani.org/xz/xz-file-format.txt * python3 defaults: https://docs.python.org/3/library/lzma.html#lzma.LZMACompressor * python2 defaults: Help on class LZMAFile in module lzma: class LZMAFile(__builtin__.object) | LZMAFile(name [, mode='r', buffering=0, memlimit=-1, | options={'format':'xz', 'check':'crc32', 'level':6, 'extreme':False, | 'dict_size':23, 'lc':3 'lp':0, 'pb':2, 'mode':2, | 'nice_len':128, 'mf':'bt4', 'depth':0]) -> file object Signed-off-by: Yann Dirson --- tests/test_cpio.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index e8e200e0..0ddd2ce8 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -1,6 +1,7 @@ from __future__ import print_function from hashlib import md5 import os +import sys import shutil import subprocess import unittest @@ -97,7 +98,12 @@ def archiveCreate(self, fn, fmt='w'): if fmt.endswith('xz'): with open(fn, 'rb') as f: f.seek(6) - self.assertEqual(f.read(2), b'\x00\x01') + # check stream flags + if sys.version_info < (3, 0): + expected_flags = b'\x00\x01' # pylzma defaults to CRC32 + else: + expected_flags = b'\x00\x04' # python3 defaults to CRC64 + self.assertEqual(f.read(2), expected_flags) self.archiveExtract(fn) def doArchive(self, fn, fmt=None): From ccf5cc948333e4b860b3bbf540e2c3c9e47e7580 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 17:15:28 +0200 Subject: [PATCH 50/56] test_cpio: check xz magic, not just the flags Just safer than just checking for 2 single-digit bytes. Signed-off-by: Yann Dirson --- tests/test_cpio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 0ddd2ce8..101a05de 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -97,7 +97,8 @@ def archiveCreate(self, fn, fmt='w'): # special case for XZ, test check type (crc32) if fmt.endswith('xz'): with open(fn, 'rb') as f: - f.seek(6) + # check xz magic + self.assertEqual(f.read(6), b"\xfd7zXZ\0") # check stream flags if sys.version_info < (3, 0): expected_flags = b'\x00\x01' # pylzma defaults to CRC32 From 8466dbd87128fbbe1e5e1b247ddea77c44916715 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 19 Jul 2022 17:16:32 +0200 Subject: [PATCH 51/56] cpio: fix check for xz file magic ignoring the 6th byte Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index c2728940..f6e0dd6b 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -480,7 +480,7 @@ def getcomptype(self): return "gz" if self.buf.startswith(b"BZh91"): return "bz2" - if self.buf.startswith(b"\xfd7zXZ"): + if self.buf.startswith(b"\xfd7zXZ\0"): return "xz" return "cpio" From cb25e3ca033745fd7aef9c7a686b4a11452be9f8 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Mon, 8 Aug 2022 16:14:32 +0200 Subject: [PATCH 52/56] test_cpio: remove unused import (pylint on python3) Signed-off-by: Yann Dirson --- tests/test_cpio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cpio.py b/tests/test_cpio.py index 101a05de..c3543c89 100644 --- a/tests/test_cpio.py +++ b/tests/test_cpio.py @@ -7,7 +7,7 @@ import unittest import warnings -from xcp.cpiofile import CpioFile, CpioInfo, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED +from xcp.cpiofile import CpioFile, CpioFileCompat, CPIO_PLAIN, CPIO_GZIPPED def writeRandomFile(fn, size, start=b'', add=b'a'): "Create a pseudo-random reproducible file from seeds `start` amd `add`" From 39d23a966290559b0a1b6e5ce5e88bd8253891bd Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 10 Aug 2022 12:22:39 +0200 Subject: [PATCH 53/56] cpiofile: revert unnecessary deviations from original tarfile.py Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index f6e0dd6b..d9c556e2 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#!/usr/bin/env python # -*- coding: iso-8859-1 -*- #------------------------------------------------------------------- # cpiofile.py @@ -37,7 +37,7 @@ __version__ = "0.1" __author__ = "Simon Rowe" -__credits__ = "Lars Gustäbel" +__credits__ = "Lars Gustäbel, Gustavo Niemeyer, Niels Gustäbel, Richard Townsend." #--------- # Imports @@ -666,10 +666,6 @@ def readsparsesection(self, size): # return NUL * size #class _FileInFile -SEEK_SET = 0 -SEEK_CUR = 1 -SEEK_END = 2 - class ExFileObject(object): """File-like object for reading an archive member. Is returned by CpioFile.extractfile(). @@ -763,20 +759,20 @@ def tell(self): return self.position - def seek(self, pos, whence=SEEK_SET): + def seek(self, pos, whence=os.SEEK_SET): """Seek to a position in the file. """ if self.closed: raise ValueError("I/O operation on closed file") - if whence == SEEK_SET: + if whence == os.SEEK_SET: self.position = min(max(pos, 0), self.size) - elif whence == SEEK_CUR: + elif whence == os.SEEK_CUR: if pos < 0: self.position = max(self.position + pos, 0) else: self.position = min(self.position + pos, self.size) - elif whence == SEEK_END: + elif whence == os.SEEK_END: self.position = max(min(self.size + pos, self.size), 0) else: raise ValueError("Invalid argument") @@ -1195,11 +1191,6 @@ def close(self): self.fileobj.write(buf) self.offset += len(buf) -# blocks, remainder = divmod(self.offset, BLOCKSIZE) -# if remainder > 0: -# self.fileobj.write((BLOCKSIZE - remainder) * NUL) -# self.offset += (BLOCKSIZE - remainder) - if not self._extfileobj: self.fileobj.close() self.closed = True @@ -1312,7 +1303,14 @@ def list(self, verbose=True): print("%10d" % cpioinfo.size, end=' ') print("%d-%02d-%02d %02d:%02d:%02d" % time.localtime(cpioinfo.mtime)[:6], end=' ') - print(cpioinfo.name) + print(cpioinfo.name, end="") + + if verbose: + if cpioinfo.issym(): + print("->", cpioinfo.linkname, end="") + if cpioinfo.islnk(): + print("link to", cpioinfo.linkname, end="") + print() def add(self, name, arcname=None, recursive=True): """Add the file `name' to the archive. `name' may be any type of file @@ -1457,7 +1455,6 @@ def extract(self, member, path=""): # Prepare the link cpioget for makelink(). if cpioinfo.islnk(): -# cpioinfo._link_cpioget = os.path.join(path, cpioinfo.linkname) cpioinfo._link_path = path try: @@ -1492,7 +1489,12 @@ def extractfile(self, member): else: cpioinfo = self.getmember(member) - if cpioinfo.issym(): + if cpioinfo.isreg(): + return self.fileobject(self, cpioinfo) + + elif cpioinfo.islnk(): + return self.fileobject(self, self._datamember(cpioinfo)) + elif cpioinfo.issym(): if isinstance(self.fileobj, _Stream): # A small but ugly workaround for the case that someone tries # to extract a symlink as a file-object from a non-seekable @@ -1502,10 +1504,6 @@ def extractfile(self, member): # A symlink's file object is its cpioget's file object. return self.extractfile(self._getmember(cpioinfo.linkname, cpioinfo)) - elif cpioinfo.islnk(): - return self.fileobject(self, self._datamember(cpioinfo)) - elif cpioinfo.isreg(): - return self.fileobject(self, cpioinfo) else: # If there's no data associated with the member (directory, chrdev, # blkdev, etc.), return None instead of a file object. From 268f84f1274846201582f34e13c71e4767b5dc63 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 10 Aug 2022 12:24:20 +0200 Subject: [PATCH 54/56] cpiofile: revert overeager tar->cpio substitution on "target" Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 99 +++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index d9c556e2..aeb35272 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -1453,7 +1453,7 @@ def extract(self, member, path=""): else: cpioinfo = self.getmember(member) - # Prepare the link cpioget for makelink(). + # Prepare the link target for makelink(). if cpioinfo.islnk(): cpioinfo._link_path = path @@ -1477,7 +1477,7 @@ def extractfile(self, member): """Extract a member from the archive as a file object. `member' may be a filename or a CpioInfo object. If `member' is a regular file, a file-like object is returned. If `member' is a link, a file-like - object is constructed from the link's cpioget. If `member' is none of + object is constructed from the link's target. If `member' is none of the above, None is returned. The file-like object is read-only and provides the following methods: read(), readline(), readlines(), seek() and tell() @@ -1501,7 +1501,7 @@ def extractfile(self, member): # stream of cpio blocks. raise StreamError("cannot extract symlink as file object") else: - # A symlink's file object is its cpioget's file object. + # A symlink's file object is its target's file object. return self.extractfile(self._getmember(cpioinfo.linkname, cpioinfo)) else: @@ -1509,17 +1509,17 @@ def extractfile(self, member): # blkdev, etc.), return None instead of a file object. return None - def _extract_member(self, cpioinfo, cpiogetpath): + def _extract_member(self, cpioinfo, targetpath): """Extract the CpioInfo object cpioinfo to a physical - file called cpiogetpath. + file called targetpath. """ # Fetch the CpioInfo object for the given name # and build the destination pathname, replacing # forward slashes to platform specific separators. - cpiogetpath = os.path.normpath(cpiogetpath) + targetpath = os.path.normpath(targetpath) # Create all upper directories. - upperdirs = os.path.dirname(cpiogetpath) + upperdirs = os.path.dirname(targetpath) if upperdirs and not os.path.exists(upperdirs): ti = CpioInfo() ti.name = upperdirs @@ -1538,39 +1538,39 @@ def _extract_member(self, cpioinfo, cpiogetpath): self._dbg(1, cpioinfo.name) if cpioinfo.isreg(): - self.makefile(cpioinfo, cpiogetpath) + self.makefile(cpioinfo, targetpath) elif cpioinfo.isdir(): - self.makedir(cpioinfo, cpiogetpath) + self.makedir(cpioinfo, targetpath) elif cpioinfo.isfifo(): - self.makefifo(cpioinfo, cpiogetpath) + self.makefifo(cpioinfo, targetpath) elif cpioinfo.ischr() or cpioinfo.isblk(): - self.makedev(cpioinfo, cpiogetpath) + self.makedev(cpioinfo, targetpath) elif cpioinfo.issym(): - self.makesymlink(cpioinfo, cpiogetpath) + self.makesymlink(cpioinfo, targetpath) else: - self.makefile(cpioinfo, cpiogetpath) + self.makefile(cpioinfo, targetpath) - self.chown(cpioinfo, cpiogetpath) + self.chown(cpioinfo, targetpath) if not cpioinfo.issym(): - self.chmod(cpioinfo, cpiogetpath) - self.utime(cpioinfo, cpiogetpath) + self.chmod(cpioinfo, targetpath) + self.utime(cpioinfo, targetpath) #-------------------------------------------------------------------------- # Below are the different file methods. They are called via # _extract_member() when extract() is called. They can be replaced in a # subclass to implement other functionality. - def makedir(self, cpioinfo, cpiogetpath): - """Make a directory called cpiogetpath. + def makedir(self, cpioinfo, targetpath): + """Make a directory called targetpath. """ try: - os.mkdir(cpiogetpath) + os.mkdir(targetpath) except EnvironmentError as e: if e.errno != errno.EEXIST: raise - def makefile(self, cpioinfo, cpiogetpath): - """Make a file called cpiogetpath. + def makefile(self, cpioinfo, targetpath): + """Make a file called targetpath. """ extractinfo = None if cpioinfo.nlink == 1: @@ -1579,7 +1579,8 @@ def makefile(self, cpioinfo, cpiogetpath): if cpioinfo.ino in self.inodes: # actual file exists, create link # FIXME handle platforms that don't support hardlinks - os.link(os.path.join(cpioinfo._link_path, six.ensure_text(self.inodes[cpioinfo.ino][0])), cpiogetpath) + os.link(os.path.join(cpioinfo._link_path, + six.ensure_text(self.inodes[cpioinfo.ino][0])), targetpath) else: extractinfo = self._datamember(cpioinfo) @@ -1589,21 +1590,21 @@ def makefile(self, cpioinfo, cpiogetpath): if extractinfo: source = self.extractfile(extractinfo) - cpioget = file(cpiogetpath, "wb") - copyfileobj(source, cpioget) + target = file(targetpath, "wb") + copyfileobj(source, target) source.close() - cpioget.close() + target.close() - def makefifo(self, cpioinfo, cpiogetpath): - """Make a fifo called cpiogetpath. + def makefifo(self, cpioinfo, targetpath): + """Make a fifo called targetpath. """ if hasattr(os, "mkfifo"): - os.mkfifo(cpiogetpath) + os.mkfifo(targetpath) else: raise ExtractError("fifo not supported by system") - def makedev(self, cpioinfo, cpiogetpath): - """Make a character or block device called cpiogetpath. + def makedev(self, cpioinfo, targetpath): + """Make a character or block device called targetpath. """ if not hasattr(os, "mknod") or not hasattr(os, "makedev"): raise ExtractError("special devices not supported by system") @@ -1614,25 +1615,25 @@ def makedev(self, cpioinfo, cpiogetpath): else: mode |= stat.S_IFCHR - os.mknod(cpiogetpath, mode, + os.mknod(targetpath, mode, os.makedev(cpioinfo.devmajor, cpioinfo.devminor)) - def makesymlink(self, cpioinfo, cpiogetpath): + def makesymlink(self, cpioinfo, targetpath): # FIXME handle platforms that don't support symlinks - os.symlink(cpioinfo.linkname, cpiogetpath) + os.symlink(cpioinfo.linkname, targetpath) - def makelink(self, cpioinfo, cpiogetpath): - """Make a (symbolic) link called cpiogetpath. If it cannot be created + def makelink(self, cpioinfo, targetpath): + """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. """ linkpath = cpioinfo.linkname try: if cpioinfo.issym(): - os.symlink(linkpath, cpiogetpath) + os.symlink(linkpath, targetpath) else: # See extract(). - os.link(cpioinfo._link_cpioget, cpiogetpath) + os.link(cpioinfo._link_target, targetpath) except AttributeError: if cpioinfo.issym(): linkpath = os.path.join(os.path.dirname(cpioinfo.name), @@ -1640,16 +1641,16 @@ def makelink(self, cpioinfo, cpiogetpath): linkpath = normpath(linkpath) try: - self._extract_member(self.getmember(linkpath), cpiogetpath) + self._extract_member(self.getmember(linkpath), targetpath) except (EnvironmentError, KeyError): linkpath = os.path.normpath(linkpath) try: - shutil.copy2(linkpath, cpiogetpath) + shutil.copy2(linkpath, targetpath) except EnvironmentError: raise IOError("link could not be created") - def chown(self, cpioinfo, cpiogetpath): - """Set owner of cpiogetpath according to cpioinfo. + def chown(self, cpioinfo, targetpath): + """Set owner of targetpath according to cpioinfo. """ if PWD and hasattr(os, "geteuid") and os.geteuid() == 0: # We have to be root to do so. @@ -1663,24 +1664,24 @@ def chown(self, cpioinfo, cpiogetpath): u = os.getuid() try: if cpioinfo.issym() and hasattr(os, "lchown"): - os.lchown(cpiogetpath, u, g) + os.lchown(targetpath, u, g) else: if sys.platform != "os2emx": - os.chown(cpiogetpath, u, g) + os.chown(targetpath, u, g) except EnvironmentError: raise ExtractError("could not change owner") - def chmod(self, cpioinfo, cpiogetpath): - """Set file permissions of cpiogetpath according to cpioinfo. + def chmod(self, cpioinfo, targetpath): + """Set file permissions of targetpath according to cpioinfo. """ if hasattr(os, 'chmod'): try: - os.chmod(cpiogetpath, cpioinfo.mode) + os.chmod(targetpath, cpioinfo.mode) except EnvironmentError: raise ExtractError("could not change mode") - def utime(self, cpioinfo, cpiogetpath): - """Set modification time of cpiogetpath according to cpioinfo. + def utime(self, cpioinfo, targetpath): + """Set modification time of targetpath according to cpioinfo. """ if not hasattr(os, 'utime'): return @@ -1689,7 +1690,7 @@ def utime(self, cpioinfo, cpiogetpath): # to use utime() on directories. return try: - os.utime(cpiogetpath, (cpioinfo.mtime, cpioinfo.mtime)) + os.utime(targetpath, (cpioinfo.mtime, cpioinfo.mtime)) except EnvironmentError: raise ExtractError("could not change modification time") From 8439094bf28b8735cc004efb35f805cdf86067bf Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 31 Aug 2022 10:53:10 +0200 Subject: [PATCH 55/56] cpiofile: catch use of non-binary streams early Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index aeb35272..9eff7ac3 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -50,6 +50,7 @@ import time import struct import copy +import io import six @@ -959,6 +960,7 @@ def __init__(self, name=None, mode="r", fileobj=None): self.name = None if name: self.name = os.path.abspath(name) + assert not isinstance(fileobj, io.TextIOBase) self.fileobj = fileobj # Init datastructures From 19021749d8cf45b1d69e99ae27251fc4d7008dfd Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 31 Aug 2022 10:54:46 +0200 Subject: [PATCH 56/56] cpiofile: hack support for text IO to archive member Really hackish in case the code gets used. Caught/tested using non-included test_cpiofile.py derived from cpython's test_tarfile.py Signed-off-by: Yann Dirson --- xcp/cpiofile.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/xcp/cpiofile.py b/xcp/cpiofile.py index 9eff7ac3..7b2623fa 100755 --- a/xcp/cpiofile.py +++ b/xcp/cpiofile.py @@ -710,6 +710,8 @@ def read(self, size=None): self.position += len(buf) return buf + # FIXME no universal-newlines, a TextIOWrapper would help but hey + # we're not using cpio archives on non-unices, right ? def readline(self, size=-1): """Read one entire line from the file. If size is present and non-negative, return a string with at most that @@ -718,16 +720,16 @@ def readline(self, size=-1): if self.closed: raise ValueError("I/O operation on closed file") - if "\n" in self.buffer: - pos = self.buffer.find("\n") + 1 + if b"\n" in self.buffer: + pos = self.buffer.find(b"\n") + 1 else: buffers = [self.buffer] while True: buf = self.fileobj.read(self.blocksize) buffers.append(buf) - if not buf or "\n" in buf: - self.buffer = "".join(buffers) - pos = self.buffer.find("\n") + 1 + if not buf or b"\n" in buf: + self.buffer = b"".join(buffers) + pos = self.buffer.find(b"\n") + 1 if pos == 0: # no newline found. pos = len(self.buffer) @@ -739,7 +741,7 @@ def readline(self, size=-1): buf = self.buffer[:pos] self.buffer = self.buffer[pos:] self.position += len(buf) - return buf + return six.ensure_text(buf) def readlines(self): """Return a list with all remaining lines. @@ -1794,8 +1796,9 @@ def _getmember(self, name, cpioinfo=None): else: end = members.index(cpioinfo) + encoded_name = six.ensure_binary(name) for i in range(end - 1, -1, -1): - if name == members[i].name: + if encoded_name == members[i].name: return members[i] def _load(self):