Remove py 3.6/3.7 support and add 3.11

Author: saville

.github/workflows/build.yaml

@@ -12,10 +12,10 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-        - 3.7
         - 3.8
         - 3.9
         - '3.10'
+        - '3.11'
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

.pylintrc

@@ -65,87 +65,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=print-statement,
-        parameter-unpacking,
-        unpacking-in-except,
-        old-raise-syntax,
-        backtick,
-        long-suffix,
-        old-ne-operator,
-        old-octal-literal,
-        import-star-module-level,
-        non-ascii-bytes-literal,
-        raw-checker-failed,
-        bad-inline-option,
-        locally-disabled,
-        file-ignored,
-        suppressed-message,
-        useless-suppression,
-        deprecated-pragma,
-        use-symbolic-message-instead,
-        apply-builtin,
-        basestring-builtin,
-        buffer-builtin,
-        cmp-builtin,
-        coerce-builtin,
-        execfile-builtin,
-        file-builtin,
-        long-builtin,
-        raw_input-builtin,
-        reduce-builtin,
-        standarderror-builtin,
-        unicode-builtin,
-        xrange-builtin,
-        coerce-method,
-        delslice-method,
-        getslice-method,
-        setslice-method,
-        no-absolute-import,
-        old-division,
-        dict-iter-method,
-        dict-view-method,
-        next-method-called,
-        metaclass-assignment,
-        indexing-exception,
-        raising-string,
-        reload-builtin,
-        oct-method,
-        hex-method,
-        nonzero-method,
-        cmp-method,
-        input-builtin,
-        round-builtin,
-        intern-builtin,
-        unichr-builtin,
-        map-builtin-not-iterating,
-        zip-builtin-not-iterating,
-        range-builtin-not-iterating,
-        filter-builtin-not-iterating,
-        using-cmp-argument,
-        eq-without-hash,
-        div-method,
-        idiv-method,
-        rdiv-method,
-        exception-message-attribute,
-        invalid-str-codec,
-        sys-max-int,
-        bad-python3-import,
-        deprecated-string-function,
-        deprecated-str-translate-call,
-        deprecated-itertools-function,
-        deprecated-types-field,
-        next-method-defined,
-        dict-items-not-iterating,
-        dict-keys-not-iterating,
-        dict-values-not-iterating,
-        deprecated-operator-function,
-        deprecated-urllib-function,
-        xreadlines-attribute,
-        deprecated-sys-function,
-        exception-escape,
-        comprehension-escape,
-        bare-except,
-        logging-fstring-interpolation,
+disable=logging-fstring-interpolation,
         unnecessary-pass,
         wrong-import-order,
         too-few-public-methods,
@@ -595,5 +515,5 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

CHANGELOG.rst

@@ -5,6 +5,14 @@
 .. contents::
    :local:
 
+3.0
+###
+
+* 3.0
+
+  * Add python 3.11 support
+  * Remove python 3.6 and 3.7 support
+
 2.0
 ###
 

buildrunner/__init__.py

@@ -219,7 +219,7 @@ def __init__(
 
         # print out env vars
         # pylint: disable=consider-iterating-dictionary
-        key_len = max([len(key) for key in self.env.keys()])
+        key_len = max(len(key) for key in self.env.keys())
         for key in sorted(self.env.keys()):
             val = self.env[key]
             LOGGER.debug(f'Environment: {key!s:>{key_len}}: {val}')
@@ -274,7 +274,8 @@ def log(self):
 
             try:
                 log_file_path = os.path.join(self.build_results_dir, 'build.log')
-                self._log_file = open(log_file_path, 'w')
+                # pylint: disable=consider-using-with
+                self._log_file = open(log_file_path, 'w', encoding='utf8')
                 self._log = ConsoleLogger(self.colorize_log, self._log_file)
 
                 self.add_artifact(
@@ -385,6 +386,7 @@ def get_local_files_from_alias(self, file_alias):
                     return local_path
 
                 # need to put the contents in a tmp file and return the path
+                # pylint: disable=consider-using-with
                 _fileobj = tempfile.NamedTemporaryFile(
                     delete=False,
                     dir=self.global_config.get_temp_dir(),
@@ -470,6 +472,7 @@ def _exclude_working_dir(tarinfo):
             self.log.write('Creating source archive\n')
             _fileobj = None
             try:
+                # pylint: disable=consider-using-with
                 _fileobj = tempfile.NamedTemporaryFile(
                     delete=False,
                     dir=self.global_config.get_temp_dir(),

buildrunner/config.py

@@ -54,6 +54,7 @@ def _raise_exception_jinja(message):
         """
         Raises an exception from a jinja template.
         """
+        # pylint: disable=broad-exception-raised
         raise Exception(message)
 
     @staticmethod

buildrunner/docker/builder.py

@@ -43,6 +43,7 @@ def __init__(
             if os.path.exists(dockerfile):
                 self.dockerfile = dockerfile
             else:
+                # pylint: disable=consider-using-with
                 df_file = tempfile.NamedTemporaryFile(delete=False, dir=self.temp_dir)
                 try:
                     df_file.write(dockerfile.encode('utf-8'))
@@ -88,16 +89,16 @@ def build(self, console=None, nocache=False, cache_from=None, rm=True, pull=True
         if buildargs is None:
             buildargs = {}
         # create our own tar file, injecting the appropriate paths
+        # pylint: disable=consider-using-with
         _fileobj = tempfile.NamedTemporaryFile(dir=self.temp_dir)
-        tfile = tarfile.open(mode='w', fileobj=_fileobj)
-        if self.path:
-            tfile.add(self.path, arcname='.')
-        if self.inject:
-            for to_inject, dest in self.inject.items():
-                tfile.add(to_inject, arcname=dest)
-        if self.dockerfile:
-            tfile.add(self.dockerfile, arcname='./Dockerfile')
-        tfile.close()
+        with tarfile.open(mode='w', fileobj=_fileobj) as tfile:
+            if self.path:
+                tfile.add(self.path, arcname='.')
+            if self.inject:
+                for to_inject, dest in self.inject.items():
+                    tfile.add(to_inject, arcname=dest)
+            if self.dockerfile:
+                tfile.add(self.dockerfile, arcname='./Dockerfile')
         _fileobj.seek(0)
 
         # Always add default registry to build args

buildrunner/fetch/github.py

@@ -35,7 +35,7 @@ def v3_fetch_file(parsed_url, config):
         'users',
         username,
     ))
-    resp = requests.get(url, auth=auth)
+    resp = requests.get(url, auth=auth, timeout=180)
     if resp.status_code != 200:
         raise BuildRunnerProtocolError(f'Failed authenticating {username} on {endpoint}')
 
@@ -46,7 +46,7 @@ def v3_fetch_file(parsed_url, config):
     ubuild = [endpoint, version, 'repos', fpath[1], fpath[2], 'contents']
     ubuild.extend(fpath[3:])
     url = '/'.join(ubuild)
-    resp = requests.get(url, auth=auth)
+    resp = requests.get(url, auth=auth, timeout=180)
     if resp.status_code != 200:
         raise BuildRunnerProtocolError(f'Failed fetching URL: {url}')
     json_c = resp.json()

buildrunner/provisioners/salt.py

@@ -32,7 +32,7 @@ def provision(self, runner):
         # see if salt is installed, bootstrap if it isn't
         if runner.run('salt-call -h') != 0:
             # pull bootstrap and run as a script
-            bootstrap_response = requests.get('http://bootstrap.saltstack.org')
+            bootstrap_response = requests.get('http://bootstrap.saltstack.org', timeout=600)
             if bootstrap_response.status_code != 200:
                 raise BuildRunnerProvisionerError(
                     "Unable to get salt bootstrap"

buildrunner/sshagent/__init__.py

@@ -24,7 +24,8 @@
     SSHException,
 )
 from paramiko.agent import AgentSSH
-from paramiko.common import asbytes, io_sleep
+from paramiko.common import io_sleep
+from paramiko.util import asbytes
 from paramiko.message import Message
 
 from buildrunner.errors import (
@@ -224,6 +225,7 @@ def _try_connect(self, ssh_host, ssh_port):
             time.sleep(backoff)
         else:
             self.log.write(f'Unable to successfully connect to {ssh_host}')
+            # pylint: disable=broad-exception-raised
             raise Exception(f'Unable to successfully connect to {ssh_host}')
 
     def stop(self):

buildrunner/steprunner/tasks/run.py

@@ -226,6 +226,7 @@ def _retrieve_artifacts(self, console=None):  # pylint: disable=too-many-locals
                     log=self.step_runner.log,
                 )
                 if exit_code != 0:
+                    # pylint: disable=broad-exception-raised
                     raise Exception(
                         "Error gathering artifacts--unable to change ownership"
                     )
@@ -274,6 +275,7 @@ def _archive_dir(
                         log=self.step_runner.log,
                     )
                     if exit_code != 0:
+                        # pylint: disable=broad-exception-raised
                         raise Exception(
                             f"Error gathering artifact {artifact_file}",
                         )
@@ -383,6 +385,7 @@ def _archive_file(
             workdir=workdir,
         )
         if exit_code != 0:
+            # pylint: disable=broad-exception-raised
             raise Exception(
                 f"Error gathering artifact {artifact_file}",
             )

buildrunner/utils.py

@@ -201,6 +201,7 @@ def write(self, output: Union[bytes, str], color=None):
         # colorize stdout
         if color and self.colorize_log:
             # Colorize stdout
+            # pylint: disable=invalid-character-esc
             _stdout = f'[01;3{color}m{_stdout}\033[00;00m'
         self.stdout.write(_stdout)
 
@@ -339,6 +340,7 @@ def _acquire_flock_open(
     :param exclusive: config exclusive lock (True) or shared lock (False), defaults to True
     :return: opened file object if successful else None
     """
+    # pylint: disable=unspecified-encoding,consider-using-with
     file_obj = open(lock_file, mode)
     pid = os.getpid()
     lock_file_obj = None

requirements.in

@@ -1,17 +1,12 @@
-# Last version to support python 3.6
-GitPython<=3.1.18
-Jinja2==2.11.2
-MarkupSafe==1.1.1
-PyYAML==6.0
-bcrypt==3.2.0
-decorator==5.1.0
-docker==4.4.4
-fabric==2.5.0
-paramiko==2.10.3
-requests==2.27.0
-twine==3.2.0
+Jinja2>=2.11.2
+PyYAML>=6.0
+bcrypt>=3.2.0
+decorator>=5.1.0
+docker>=6.1.2
+fabric>=2.5.0
+paramiko>=2.10.3
+requests>=2.27.0
+twine>=3.2.0
 vcsinfo>=2.1.105
 graphlib-backport>=1.0.3
-# Needed for python 3.6
-typing-extensions<4.2.0
 timeout-decorator>=0.5.0