From 352a2f83492aee21e1fbed774bae85b4c5be2f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= Date: Thu, 28 Sep 2023 23:00:17 -0400 Subject: [PATCH 1/4] Added __init__.py to get real submodules, cleaned up keyring message --- dcclab/analysis/__init__.py | 0 dcclab/database/database.py | 4 ++-- dcclab/image/image.py | 1 - dcclab/ml/__init__.py | 0 dcclab/utils/cafeine.py | 10 ++++------ setup.py | 2 +- 6 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 dcclab/analysis/__init__.py create mode 100644 dcclab/ml/__init__.py diff --git a/dcclab/analysis/__init__.py b/dcclab/analysis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dcclab/database/database.py b/dcclab/database/database.py index 475f847e..b897e574 100644 --- a/dcclab/database/database.py +++ b/dcclab/database/database.py @@ -218,7 +218,7 @@ def connect(self): pwd = keyring.get_password(serviceName, self.mysqlUser) if pwd is None: raise Exception(""" Set the password in the system password manager on the command line with: - python -m keyring set {0} {1}""".format(serviceName, self.mysqlUser)) +{2} -m keyring set {0} {1}""".format(serviceName, self.mysqlUser, sys.executable)) else: pwd = None @@ -524,7 +524,7 @@ def connect(self): pwd = keyring.get_password(serviceName, self.mysqlUser) if pwd is None: raise Exception(""" Set the password in the system password manager on the command line with: - python -m keyring set {0} {1}""".format(serviceName, self.mysqlUser)) +{2} -m keyring set {0} {1}""".format(serviceName, self.mysqlUser, sys.executable)) else: pwd = None diff --git a/dcclab/image/image.py b/dcclab/image/image.py index a3a87cf7..c50250ed 100644 --- a/dcclab/image/image.py +++ b/dcclab/image/image.py @@ -5,7 +5,6 @@ import PIL.Image import matplotlib.pyplot as plt import os -import re from typing import List, Union diff --git a/dcclab/ml/__init__.py b/dcclab/ml/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dcclab/utils/cafeine.py b/dcclab/utils/cafeine.py index a5b64132..26dd0d2c 100644 --- a/dcclab/utils/cafeine.py +++ b/dcclab/utils/cafeine.py @@ -2,7 +2,7 @@ import time import keyring from contextlib import redirect_stderr - +import sys class Cafeine: def __init__(self, username='dcclab'): @@ -12,11 +12,9 @@ def startMySQLTunnel(self, remote_bind_address="127.0.0.1"): password = keyring.get_password("cafeine2-ssh", "dcclab") if password is None: - print(""" - Run the following command in the terminal to add the dcclab password to your Keychain: - keyring set cafeine2-ssh dcclab - (you will be prompted for the password) - """) + print("""Run the following command in the terminal to add the dcclab password to your Keychain: +{0} -m keyring set cafeine2-ssh dcclab +(you will be prompted for the password)""".format(sys.executable)) exit(1) import io diff --git a/setup.py b/setup.py index bd2dc6ed..621a1f84 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ ============= rm dist/*; python setup.py sdist --formats=gztar,zip or -rm dist/*; python setup.py sdist bdist_wheel; python -m twine upload dist/* +rm dist/*; python3 setup.py sdist; python3 -m twine upload dist/* """ From 21d9e520ab43e0f02dd659dd32def3f8d2097a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= Date: Thu, 28 Sep 2023 23:18:15 -0400 Subject: [PATCH 2/4] Several imports were incorrect when I moved to sub-modules But those files (labpca for instance, computengine, ml/dataset ) were not tested, so i did not catch that. --- dcclab/analysis/__init__.py | 2 ++ dcclab/analysis/computeengine.py | 3 +-- dcclab/database/labdatadb.py | 1 - dcclab/image/cziUtil.py | 2 -- dcclab/ml/dataset.py | 1 - dcclab/utils/pathPattern.py | 2 +- setup.py | 5 ++++- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dcclab/analysis/__init__.py b/dcclab/analysis/__init__.py index e69de29b..917b7610 100644 --- a/dcclab/analysis/__init__.py +++ b/dcclab/analysis/__init__.py @@ -0,0 +1,2 @@ +from .computeengine import * +from .labpca import * diff --git a/dcclab/analysis/computeengine.py b/dcclab/analysis/computeengine.py index 92a1dd73..8f6594a4 100644 --- a/dcclab/analysis/computeengine.py +++ b/dcclab/analysis/computeengine.py @@ -2,9 +2,8 @@ from threading import Thread from queue import Empty import time -from json import dumps import signal -from database import * +from dcclab.database import * """ A class to run many tasks in parallel when they are mostly independent. This engine is perfectly diff --git a/dcclab/database/labdatadb.py b/dcclab/database/labdatadb.py index d36f4ade..b898d2c4 100644 --- a/dcclab/database/labdatadb.py +++ b/dcclab/database/labdatadb.py @@ -2,7 +2,6 @@ import numpy as np import pandas as pd import re -from collections.abc import Iterable class LabdataDB(MySQLDatabase): """ diff --git a/dcclab/image/cziUtil.py b/dcclab/image/cziUtil.py index f6401a72..6957564a 100644 --- a/dcclab/image/cziUtil.py +++ b/dcclab/image/cziUtil.py @@ -6,9 +6,7 @@ import xml.etree.ElementTree as ET import os import fnmatch -import multiprocessing import warnings -from concurrent.futures import ThreadPoolExecutor """ Python script containing utility functions to be used for handling .czi image. diff --git a/dcclab/ml/dataset.py b/dcclab/ml/dataset.py index 259ce0f6..da9b3e08 100644 --- a/dcclab/ml/dataset.py +++ b/dcclab/ml/dataset.py @@ -2,7 +2,6 @@ from typing import List import pandas as pd import numpy as np -import json import os """ diff --git a/dcclab/utils/pathPattern.py b/dcclab/utils/pathPattern.py index 77575e54..a617f2d5 100644 --- a/dcclab/utils/pathPattern.py +++ b/dcclab/utils/pathPattern.py @@ -1,6 +1,6 @@ import re import os -from pathlib import Path, PureWindowsPath, PurePosixPath +from pathlib import Path, PurePosixPath class PathPattern: diff --git a/setup.py b/setup.py index 621a1f84..45c8ac9e 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,10 @@ license='MIT', keywords='image analysis stack movies', packages=setuptools.find_packages(), - install_requires=['keyring', 'sshtunnel', 'mysql-connector-python', 'requests', 'matplotlib','numpy','scikit-image','scipy','czifile >= 2019.6.18','tifffile','read-lif','tables','xlwt','xlrd','seaborn','imagecodecs','deprecated'], + install_requires=['keyring', 'gsheet-keyring', 'sshtunnel', 'mysql-connector-python', + 'requests', 'matplotlib','numpy','scikit-image','scipy', + 'czifile >= 2019.6.18','tifffile','read-lif','tables','xlwt', + 'xlrd','seaborn','imagecodecs','deprecated'], python_requires='>=3', package_data={ # If any package contains *.txt or *.rst files, include them: From d724685f0ab087d7a04037301110e8e3b51f20cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= Date: Thu, 28 Sep 2023 23:48:35 -0400 Subject: [PATCH 3/4] Create testComputeEngine.py --- dcclab/tests/testComputeEngine.py | 70 +++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 dcclab/tests/testComputeEngine.py diff --git a/dcclab/tests/testComputeEngine.py b/dcclab/tests/testComputeEngine.py new file mode 100644 index 00000000..d9cbdd59 --- /dev/null +++ b/dcclab/tests/testComputeEngine.py @@ -0,0 +1,70 @@ +import env +import unittest +from dcclab.analysis import * + +def calculateFactorial(inputQueue, outputQueue): + try: + value = inputQueue.get_nowait() + product = 1 + for i in range(value): + product *= (i+1) + outputQueue.put( (value, product) ) + except Empty as err: + pass # not an error + +def slowCalculation(inputQueue, outputQueue): + try: + value = inputQueue.get_nowait() + time.sleep(3) + outputQueue.put( value ) + except Empty as err: + pass # not an error + +def processSimple(queue): + while not queue.empty(): + try: + (n, nfactorial) = queue.get_nowait() + print('Just finished calculating {0}!'.format(n)) + except Empty as err: + break # we are done + + +class MyTestCase(env.DCCLabTestCase): + def testThreads1(self): + N = 11 + print("Calculating n! for numbers 0 to {0} (every calculation is independent)".format(N - 1)) + print("======================================================================") + + print("Using threads: fast startup time appropriate for quick calculations") + engine = ComputeEngine(useThreads=True) + for i in range(N): + engine.inputQueue.put(i) + engine.compute(target=calculateFactorial) + + def testProcesses1(self): + N = 11 + print("Using processes: long startup time appropriate for longer calculations") + engine = ComputeEngine(useThreads=False) + for i in range(N): + engine.inputQueue.put(i) + engine.compute(target=calculateFactorial) + + def testThreadsProcessTask(self): + N = 11 + print("Using threads and replacing the processTaskResult function") + engine = ComputeEngine(useThreads=True) + for i in range(N): + engine.inputQueue.put(i) + engine.compute(target=calculateFactorial, processTaskResults=processSimple) + @unittest.skip + def testProcessesVeryLong(self): + N = 11 + print("Using processes with very long calculations and timeout") + engine = ComputeEngine(useThreads=False) + for i in range(N): + engine.inputQueue.put(i) + engine.compute(target=slowCalculation, timeoutInSeconds=2) + + +if __name__ == '__main__': + unittest.main() From b5d3b23fe8a998a959f347a5bc27f52cadec3cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= Date: Thu, 28 Sep 2023 23:59:37 -0400 Subject: [PATCH 4/4] new version --- dcclab/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dcclab/__init__.py b/dcclab/__init__.py index 6c866958..39b53c10 100644 --- a/dcclab/__init__.py +++ b/dcclab/__init__.py @@ -13,5 +13,5 @@ from .utils import * from .DCCExceptions import * -__version__ = "1.1.1" +__version__ = "1.1.2" __author__ = "Daniel Cote " diff --git a/setup.py b/setup.py index 45c8ac9e..7aa2dfbd 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setuptools.setup( name="dcclab", - version="1.1.1", + version="1.1.2", url="https://github.com/DCC-Lab/PyDCCLab", author="Daniel Côté, Gabriel Genest, Mathieu Fournier, Ludovick Bégin", author_email="dccote@cervo.ulaval.ca",