diff --git a/libsubmit/channels/channel_base.py b/libsubmit/channels/channel_base.py index dcf081c..d0eabfb 100644 --- a/libsubmit/channels/channel_base.py +++ b/libsubmit/channels/channel_base.py @@ -1,7 +1,8 @@ from abc import ABCMeta, abstractmethod, abstractproperty +import six - -class Channel(metaclass=ABCMeta): +@six.add_metaclass(ABCMeta) +class Channel(): """ Define the interface to all channels. Channels are usually called via the execute_wait function. For channels that execute remotely, a push_file function allows you to copy over files. diff --git a/libsubmit/channels/errors.py b/libsubmit/channels/errors.py index 23fb3b6..19cdc6d 100644 --- a/libsubmit/channels/errors.py +++ b/libsubmit/channels/errors.py @@ -25,7 +25,7 @@ class BadHostKeyException(ChannelError): ''' def __init__(self, e, hostname): - super().__init__() + super(BadHostKeyException, self).__init__() self.reason = "SSH channel could not be created since server's host keys could not be verified" self.hostname = hostname self.e = e @@ -41,7 +41,7 @@ class BadScriptPath(ChannelError): ''' def __init__(self, e, hostname): - super().__init__() + super(BadScriptPath, self).__init__() self.reason = "Inaccessible remote script dir. Specify script_dir" self.hostname = hostname self.e = e @@ -57,7 +57,7 @@ class BadPermsScriptPath(ChannelError): ''' def __init__(self, e, hostname): - super().__init__() + super(BadPermsScriptPath, self).__init__() self.reason = "User does not have permissions to access the script_dir" self.hostname = hostname self.e = e @@ -74,7 +74,7 @@ class FileExists(ChannelError): ''' def __init__(self, e, hostname, filename=None): - super().__init__() + super(FileExists, self).__init__() self.reason = "File name collision in channel transport phase:" + filename self.hostname = hostname self.e = e @@ -90,7 +90,7 @@ class AuthException(ChannelError): ''' def __init__(self, e, hostname): - super().__init__() + super(AuthException, self).__init__() self.reason = "Authentication to remote server failed" self.hostname = hostname self.e = e @@ -106,7 +106,7 @@ class SSHException(ChannelError): ''' def __init__(self, e, hostname): - super().__init__() + super(SSHException, self).__init__() self.reason = "Error connecting or establishing an SSH session" self.hostname = hostname self.e = e @@ -122,7 +122,7 @@ class FileCopyException(ChannelError): ''' def __init__(self, e, hostname): - super().__init__() + super(FileCopyException, self).__init__() self.reason = "File copy failed due to {0}".format(e) self.hostname = hostname self.e = e diff --git a/libsubmit/launchers/launchers.py b/libsubmit/launchers/launchers.py index 60ac38d..8e0c14b 100644 --- a/libsubmit/launchers/launchers.py +++ b/libsubmit/launchers/launchers.py @@ -1,9 +1,11 @@ from abc import ABCMeta, abstractmethod +import six from libsubmit.utils import RepresentationMixin -class Launcher(RepresentationMixin, metaclass=ABCMeta): +@six.add_metaclass(ABCMeta) +class Launcher(RepresentationMixin): """ Launcher base class to enforce launcher interface """ @abstractmethod diff --git a/libsubmit/providers/aws/aws.py b/libsubmit/providers/aws/aws.py index 580e57d..d8c29a7 100644 --- a/libsubmit/providers/aws/aws.py +++ b/libsubmit/providers/aws/aws.py @@ -1,3 +1,4 @@ +# -*- coding: future_fstrings -*- import json import logging import os @@ -593,8 +594,8 @@ def submit(self, command='sleep 1', blocksize=1, job_name="parsl.auto"): wrapped_cmd = self.launcher(command, self.tasks_per_node, self.nodes_per_block) - [instance, *rest] = self.spin_up_instance(command=wrapped_cmd, job_name=job_name) - + instance_data = self.spin_up_instance(command=wrapped_cmd, job_name=job_name) + instance, rest = instance_data[0], instance_data[1:] if not instance: logger.error("Failed to submit request to EC2") return None diff --git a/libsubmit/providers/azure/azure.py b/libsubmit/providers/azure/azure.py index 7567f53..36f3bd9 100644 --- a/libsubmit/providers/azure/azure.py +++ b/libsubmit/providers/azure/azure.py @@ -131,7 +131,8 @@ def submit(self, command='sleep 1', blocksize=1, job_name="parsl.auto"): """ job_name = "parsl.auto.{0}".format(time.time()) - [instance, *rest] = self.deployer.deploy(command=command, job_name=job_name, blocksize=1) + instance_data = self.deployer.deploy(command=command, job_name=job_name, blocksize=1) + instance, rest = instance_data[0], instance_data[1:] if not instance: logger.error("Failed to submit request to Azure") diff --git a/libsubmit/providers/cobalt/cobalt.py b/libsubmit/providers/cobalt/cobalt.py index 97a7e7b..f28e8aa 100644 --- a/libsubmit/providers/cobalt/cobalt.py +++ b/libsubmit/providers/cobalt/cobalt.py @@ -75,7 +75,7 @@ def __init__(self, overrides='', launcher=AprunLauncher(), cmd_timeout=10): - super().__init__(label, + super(CobaltProvider, self).__init__(label, channel=channel, script_dir=script_dir, nodes_per_block=nodes_per_block, @@ -104,7 +104,7 @@ def _status(self): jobs_missing = list(self.resources.keys()) - retcode, stdout, stderr = super().execute_wait("qstat -u $USER") + retcode, stdout, stderr = super(CobaltProvider, self).execute_wait("qstat -u $USER") # Execute_wait failed. Do no update. if retcode != 0: @@ -193,7 +193,7 @@ def submit(self, command, blocksize, job_name="parsl.auto"): self.nodes_per_block, queue_opt, wtime_to_minutes(self.walltime), account_opt, channel_script_path) logger.debug("Executing {}".format(command)) - retcode, stdout, stderr = super().execute_wait(command) + retcode, stdout, stderr = super(CobaltProvider, self).execute_wait(command) # TODO : FIX this block if retcode != 0: @@ -226,7 +226,7 @@ def cancel(self, job_ids): """ job_id_list = ' '.join(job_ids) - retcode, stdout, stderr = super().execute_wait("qdel {0}".format(job_id_list)) + retcode, stdout, stderr = super(CobaltProvider, self).execute_wait("qdel {0}".format(job_id_list)) rets = None if retcode == 0: for jid in job_ids: diff --git a/libsubmit/providers/condor/condor.py b/libsubmit/providers/condor/condor.py index 44fa0e4..8694bf8 100644 --- a/libsubmit/providers/condor/condor.py +++ b/libsubmit/providers/condor/condor.py @@ -82,7 +82,7 @@ def __init__(self, launcher=SingleNodeLauncher(), requirements=''): - super().__init__(label, + super(CondorProvider, self).__init__(label, channel, script_dir, nodes_per_block, @@ -115,7 +115,7 @@ def _status(self): job_id_list = ' '.join(self.resources.keys()) cmd = "condor_q {0} -af:jr JobStatus".format(job_id_list) - retcode, stdout, stderr = super().execute_wait(cmd) + retcode, stdout, stderr = super(CondorProvider, self).execute_wait(cmd) """ Example output: @@ -230,7 +230,7 @@ def submit(self, command, blocksize, job_name="parsl.auto"): channel_script_path = self.channel.push_file(script_path, self.channel.script_dir) cmd = "condor_submit {0}".format(channel_script_path) - retcode, stdout, stderr = super().execute_wait(cmd, 3) + retcode, stdout, stderr = super(CondorProvider, self).execute_wait(cmd, 3) logger.debug("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip()) job_id = [] diff --git a/libsubmit/providers/grid_engine/grid_engine.py b/libsubmit/providers/grid_engine/grid_engine.py index 2cf87aa..d6f31b5 100644 --- a/libsubmit/providers/grid_engine/grid_engine.py +++ b/libsubmit/providers/grid_engine/grid_engine.py @@ -78,7 +78,7 @@ def __init__(self, walltime="00:10:00", overrides='', launcher=SingleNodeLauncher()): - super().__init__(label, + super(GridEngineProvider, self).__init__(label, channel, script_dir, nodes_per_block, @@ -151,7 +151,7 @@ def submit(self, command="", blocksize=1, job_name="parsl.auto"): channel_script_path = self.channel.push_file(script_path, self.channel.script_dir) cmd = "qsub -terse {0}".format(channel_script_path) - retcode, stdout, stderr = super().execute_wait(cmd, 10) + retcode, stdout, stderr = super(GridEngineProvider, self).execute_wait(cmd, 10) if retcode == 0: for line in stdout.split('\n'): @@ -179,7 +179,7 @@ def _status(self): cmd = "qstat" - retcode, stdout, stderr = super().execute_wait(cmd) + retcode, stdout, stderr = super(GridEngineProvider, self).execute_wait(cmd) # Execute_wait failed. Do no update if retcode != 0: @@ -217,7 +217,7 @@ def cancel(self, job_ids): job_id_list = ' '.join(job_ids) cmd = "qdel {}".format(job_id_list) - retcode, stdout, stderr = super().execute_wait(cmd, 3) + retcode, stdout, stderr = super(GridEngineProvider, self).execute_wait(cmd, 3) rets = None if retcode == 0: diff --git a/libsubmit/providers/kubernetes/kube.py b/libsubmit/providers/kubernetes/kube.py index dea305c..3b86b2d 100644 --- a/libsubmit/providers/kubernetes/kube.py +++ b/libsubmit/providers/kubernetes/kube.py @@ -7,6 +7,12 @@ from libsubmit.error import * from libsubmit.providers.provider_base import ExecutionProvider +# Compatibility with python2.7, FileNotFoundError is not defined +try: + FileNotFoundError +except NameError: + FileNotFoundError = IOError + try: from kubernetes import client, config config.load_kube_config() diff --git a/libsubmit/providers/provider_base.py b/libsubmit/providers/provider_base.py index dc7f616..417fd07 100644 --- a/libsubmit/providers/provider_base.py +++ b/libsubmit/providers/provider_base.py @@ -1,7 +1,9 @@ from abc import ABCMeta, abstractmethod, abstractproperty +import six -class ExecutionProvider(metaclass=ABCMeta): +@six.add_metaclass(ABCMeta) +class ExecutionProvider(): """ Define the strict interface for all Execution Provider .. code:: python diff --git a/libsubmit/providers/slurm/slurm.py b/libsubmit/providers/slurm/slurm.py index 5424cc9..3000b4a 100644 --- a/libsubmit/providers/slurm/slurm.py +++ b/libsubmit/providers/slurm/slurm.py @@ -1,3 +1,4 @@ +# -*- coding: future_fstrings -*- import logging import os import time @@ -83,7 +84,7 @@ def __init__(self, overrides='', cmd_timeout=10, launcher=SingleNodeLauncher()): - super().__init__(label, + super(SlurmProvider, self).__init__(label, channel, script_dir, nodes_per_block, @@ -110,7 +111,7 @@ def _status(self): job_id_list = ','.join(self.resources.keys()) cmd = "squeue --job {0}".format(job_id_list) - retcode, stdout, stderr = super().execute_wait(cmd) + retcode, stdout, stderr = super(SlurmProvider, self).execute_wait(cmd) # Execute_wait failed. Do no update if retcode != 0: @@ -179,7 +180,7 @@ def submit(self, command, blocksize, job_name="parsl.auto"): channel_script_path = self.channel.push_file(script_path, self.channel.script_dir) - retcode, stdout, stderr = super().execute_wait("sbatch {0}".format(channel_script_path)) + retcode, stdout, stderr = super(SlurmProvider, self).execute_wait("sbatch {0}".format(channel_script_path)) job_id = None if retcode == 0: @@ -203,7 +204,7 @@ def cancel(self, job_ids): ''' job_id_list = ' '.join(job_ids) - retcode, stdout, stderr = super().execute_wait("scancel {0}".format(job_id_list)) + retcode, stdout, stderr = super(SlurmProvider, self).execute_wait("scancel {0}".format(job_id_list)) rets = None if retcode == 0: for jid in job_ids: diff --git a/libsubmit/providers/torque/torque.py b/libsubmit/providers/torque/torque.py index c468692..906ef88 100644 --- a/libsubmit/providers/torque/torque.py +++ b/libsubmit/providers/torque/torque.py @@ -83,7 +83,7 @@ def __init__(self, parallelism=1, launcher=AprunLauncher(), walltime="00:20:00"): - super().__init__(label, + super(TorqueProvider, self).__init__(label, channel, script_dir, nodes_per_block, diff --git a/requirements.txt b/requirements.txt index 0f0b728..6b199bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,6 @@ google-api-python-client google-auth nbsphinx kubernetes>=6.0.0 +six +configparser +future-fstrings \ No newline at end of file diff --git a/setup.py b/setup.py index ccc0979..60f15c6 100755 --- a/setup.py +++ b/setup.py @@ -4,11 +4,17 @@ exec(f.read()) install_requires = [ - 'paramiko' + 'paramiko', + 'six', + 'configparser', + 'future-fstrings', ] tests_require = [ 'paramiko', + 'six', + 'configparser', + 'future-fstrings', 'mock>=1.0.0', 'nose', 'pytest' @@ -40,6 +46,7 @@ # Licence, must match with licence above 'License :: OSI Approved :: Apache Software License', # Python versions supported + 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', ],