update to python 3.11.6; add formal support for python 3.12 (#163)

* update to python 3.11.6; support python 3.12

* avoid missing RLock attribute for 3.11.x with x<6

.codecov.yml

@@ -23,27 +23,19 @@ coverage:
 fixes:
   # reduces pip-installed path to git root and
   # remove dist-name from setup-installed path
-  - "*/python2.7/site-packages/::py2.7/"
-  - "*/python3.7/site-packages/::py3.7/"
   - "*/python3.8/site-packages/::py3.8/"
   - "*/python3.9/site-packages/::py3.9/"
   - "*/python3.10/site-packages/::py3.10/"
   - "*/python3.11/site-packages/::py3.11/"
   - "*/python3.12/site-packages/::py3.12/"
-  - "*/pypy2.7-*/site-packages/::pypy2.7/"
-  - "*/pypy3.7-*/site-packages/::pypy3.7/"
   - "*/pypy3.8-*/site-packages/::pypy3.8/"
   - "*/pypy3.9-*/site-packages/::pypy3.9/"
   - "*/pypy3.10-*/site-packages/::pypy3.10/"
-  - "*/python2.7/site-packages/multiprocess-*::py2.7/"
-  - "*/python3.7/site-packages/multiprocess-*::py3.7/"
   - "*/python3.8/site-packages/multiprocess-*::py3.8/"
   - "*/python3.9/site-packages/multiprocess-*::py3.9/"
   - "*/python3.10/site-packages/multiprocess-*::py3.10/"
   - "*/python3.11/site-packages/multiprocess-*::py3.11/"
   - "*/python3.12/site-packages/multiprocess-*::py3.12/"
-  - "*/pypy2.7-*/site-packages/multiprocess-*::pypy2.7/"
-  - "*/pypy3.7-*/site-packages/multiprocess-*::pypy3.7/"
   - "*/pypy3.8-*/site-packages/multiprocess-*::pypy3.8/"
   - "*/pypy3.9-*/site-packages/multiprocess-*::pypy3.9/"
   - "*/pypy3.10-*/site-packages/multiprocess-*::pypy3.10/"

README.md

@@ -137,7 +137,7 @@ to execute Python in parallel. You can run the test suite with
 ``multiprocessing`` interface, the examples and documentation found at
 http://docs.python.org/library/multiprocessing.html also apply to
 ``multiprocess`` if one will ``import multiprocessing as multiprocess``.
-See https://github.com/uqfoundation/multiprocess/tree/master/py3.11/examples
+See https://github.com/uqfoundation/multiprocess/tree/master/py3.12/examples
 for a set of examples that demonstrate some basic use cases and benchmarking
 for running Python code in parallel. Please feel free to submit a ticket on
 github, or ask a question on stackoverflow (**@Mike McKerns**). If you would

py3.11/Modules/_multiprocess/semaphore.c

@@ -516,12 +516,12 @@ _multiprocess_SemLock_impl(PyTypeObject *type, int kind, int value,
     return result;
 
   failure:
-    if (handle != SEM_FAILED)
-        SEM_CLOSE(handle);
-    PyMem_Free(name_copy);
     if (!PyErr_Occurred()) {
         _PyMp_SetError(NULL, MP_STANDARD_ERROR);
     }
+    if (handle != SEM_FAILED)
+        SEM_CLOSE(handle);
+    PyMem_Free(name_copy);
     return NULL;
 }
 
@@ -556,8 +556,9 @@ _multiprocess_SemLock__rebuild_impl(PyTypeObject *type, SEM_HANDLE handle,
     if (name != NULL) {
         handle = sem_open(name, 0);
         if (handle == SEM_FAILED) {
+            PyErr_SetFromErrno(PyExc_OSError);
             PyMem_Free(name_copy);
-            return PyErr_SetFromErrno(PyExc_OSError);
+            return NULL;
         }
     }
 #endif

py3.11/README_MODS

@@ -803,3 +803,184 @@ diff Python-3.11.4/Lib/test/_test_multiprocessing.py Python-3.11.5/Lib/test/_tes
 >         self.assertEqual(rc, 0)
 >         self.assertFalse(err, msg=err.decode('utf-8'))
 > 
+# ----------------------------------------------------------------------
+diff Python-3.11.5/Modules/_multiprocessing/semaphore.c Python-3.11.6/Modules/_multiprocessing/semaphore.c
+519,521d518
+<     if (handle != SEM_FAILED)
+<         SEM_CLOSE(handle);
+<     PyMem_Free(name_copy);
+524a522,524
+>     if (handle != SEM_FAILED)
+>         SEM_CLOSE(handle);
+>     PyMem_Free(name_copy);
+558a559
+>             PyErr_SetFromErrno(PyExc_OSError);
+560c561
+<             return PyErr_SetFromErrno(PyExc_OSError);
+---
+>             return NULL;
+diff Python-3.11.5/Lib/multiprocessing/connection.py Python-3.11.6/Lib/multiprocessing/connection.py
+11a12
+> import errno
+273a275
+>         _send_ov = None
+275a278,281
+>             ov = self._send_ov
+>             if ov is not None:
+>                 # Interrupt WaitForMultipleObjects() in _send_bytes()
+>                 ov.cancel()
+278a285,288
+>             if self._send_ov is not None:
+>                 # A connection should only be used by a single thread
+>                 raise ValueError("concurrent send_bytes() calls "
+>                                  "are not supported")
+279a290
+>             self._send_ov = ov
+288a300
+>                 self._send_ov = None
+289a302,306
+>             if err == _winapi.ERROR_OPERATION_ABORTED:
+>                 # close() was called by another thread while
+>                 # WaitForMultipleObjects() was waiting for the overlapped
+>                 # operation.
+>                 raise OSError(errno.EPIPE, "handle is closed")
+diff Python-3.11.5/Lib/multiprocessing/popen_spawn_win32.py Python-3.11.6/Lib/multiprocessing/popen_spawn_win32.py
+16a17
+> # Exit code used by Popen.terminate()
+125,126c126,130
+<             except OSError:
+<                 if self.wait(timeout=1.0) is None:
+---
+>             except PermissionError:
+>                 # ERROR_ACCESS_DENIED (winerror 5) is received when the
+>                 # process already died.
+>                 code = _winapi.GetExitCodeProcess(int(self._handle))
+>                 if code == _winapi.STILL_ACTIVE:
+127a132,134
+>                 self.returncode = code
+>             else:
+>                 self.returncode = -signal.SIGTERM
+diff Python-3.11.5/Lib/multiprocessing/resource_tracker.py Python-3.11.6/Lib/multiprocessing/resource_tracker.py
+53a54,57
+> class ReentrantCallError(RuntimeError):
+>     pass
+> 
+> 
+57c61
+<         self._lock = threading.Lock()
+---
+>         self._lock = threading.RLock()
+60a65,72
+>     def _reentrant_call_error(self):
+>         # gh-109629: this happens if an explicit call to the ResourceTracker
+>         # gets interrupted by a garbage collection, invoking a finalizer (*)
+>         # that itself calls back into ResourceTracker.
+>         #   (*) for example the SemLock finalizer
+>         raise ReentrantCallError(
+>             "Reentrant call into the multiprocessing resource tracker")
+> 
+62a75,78
+>             # This should not happen (_stop() isn't called by a finalizer)
+>             # but we check for it anyway.
+>             if self._lock._recursion_count() > 1:
+>                 return self._reentrant_call_error()
+83a100,102
+>             if self._lock._recursion_count() > 1:
+>                 # The code below is certainly not reentrant-safe, so bail out
+>                 return self._reentrant_call_error()
+162c181,191
+<         self.ensure_running()
+---
+>         try:
+>             self.ensure_running()
+>         except ReentrantCallError:
+>             # The code below might or might not work, depending on whether
+>             # the resource tracker was already running and still alive.
+>             # Better warn the user.
+>             # (XXX is warnings.warn itself reentrant-safe? :-)
+>             warnings.warn(
+>                 f"ResourceTracker called reentrantly for resource cleanup, "
+>                 f"which is unsupported. "
+>                 f"The {rtype} object {name!r} might leak.")
+178a208
+> 
+diff Python-3.11.5/Lib/multiprocessing/synchronize.py Python-3.11.6/Lib/multiprocessing/synchronize.py
+53,54c53,54
+<         self.is_fork_ctx = ctx.get_start_method() == 'fork'
+<         unlink_now = sys.platform == 'win32' or self.is_fork_ctx
+---
+>         self._is_fork_ctx = ctx.get_start_method() == 'fork'
+>         unlink_now = sys.platform == 'win32' or self._is_fork_ctx
+106c106
+<             if self.is_fork_ctx:
+---
+>             if self._is_fork_ctx:
+117a118,119
+>         # Ensure that deserialized SemLock can be serialized again (gh-108520).
+>         self._is_fork_ctx = False
+diff Python-3.11.5/Lib/test/_test_multiprocessing.py Python-3.11.6/Lib/test/_test_multiprocessing.py 
+81c81
+<     # bpo-45200: Skip multiprocessing tests if Python is built with ASAN to
+---
+>     # gh-89363: Skip multiprocessing tests if Python is built with ASAN to
+330a331
+>     @support.requires_resource('cpu')
+558,559c559
+<         if os.name != 'nt':
+<             self.assertEqual(exitcode, -signal.SIGTERM)
+---
+>         self.assertEqual(exitcode, -signal.SIGTERM)
+564a565,566
+>         else:
+>             self.assertEqual(exitcode, -signal.SIGTERM)
+675a678
+>     @support.requires_resource('walltime')
+4472a4476
+>     @support.requires_resource('cpu')
+4919a4924
+>     @support.requires_resource('walltime')
+4947a4953
+>     @support.requires_resource('walltime')
+5369a5376,5403
+>     @classmethod
+>     def _put_one_in_queue(cls, queue):
+>         queue.put(1)
+> 
+>     @classmethod
+>     def _put_two_and_nest_once(cls, queue):
+>         queue.put(2)
+>         process = multiprocessing.Process(target=cls._put_one_in_queue, args=(queue,))
+>         process.start()
+>         process.join()
+> 
+>     def test_nested_startmethod(self):
+>         # gh-108520: Regression test to ensure that child process can send its
+>         # arguments to another process
+>         queue = multiprocessing.Queue()
+> 
+>         process = multiprocessing.Process(target=self._put_two_and_nest_once, args=(queue,))
+>         process.start()
+>         process.join()
+> 
+>         results = []
+>         while not queue.empty():
+>             results.append(queue.get())
+> 
+>         # gh-109706: queue.put(1) can write into the queue before queue.put(2),
+>         # there is no synchronization in the test.
+>         self.assertSetEqual(set(results), set([2, 1]))
+> 
+6052c6086,6087
+< def install_tests_in_module_dict(remote_globs, start_method):
+---
+> def install_tests_in_module_dict(remote_globs, start_method,
+>                                  only_type=None, exclude_types=False):
+6064a6100,6103
+>                 if only_type and type_ != only_type:
+>                     continue
+>                 if exclude_types:
+>                     continue
+6074a6114,6116
+>             if only_type:
+>                 continue
+> 

py3.11/multiprocess/connection.py

@@ -9,6 +9,7 @@
 
 __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
 
+import errno
 import io
 import os
 import sys
@@ -274,12 +275,22 @@ class PipeConnection(_ConnectionBase):
         with FILE_FLAG_OVERLAPPED.
         """
         _got_empty_message = False
+        _send_ov = None
 
         def _close(self, _CloseHandle=_winapi.CloseHandle):
+            ov = self._send_ov
+            if ov is not None:
+                # Interrupt WaitForMultipleObjects() in _send_bytes()
+                ov.cancel()
             _CloseHandle(self._handle)
 
         def _send_bytes(self, buf):
+            if self._send_ov is not None:
+                # A connection should only be used by a single thread
+                raise ValueError("concurrent send_bytes() calls "
+                                 "are not supported")
             ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
+            self._send_ov = ov
             try:
                 if err == _winapi.ERROR_IO_PENDING:
                     waitres = _winapi.WaitForMultipleObjects(
@@ -289,7 +300,13 @@ def _send_bytes(self, buf):
                 ov.cancel()
                 raise
             finally:
+                self._send_ov = None
                 nwritten, err = ov.GetOverlappedResult(True)
+            if err == _winapi.ERROR_OPERATION_ABORTED:
+                # close() was called by another thread while
+                # WaitForMultipleObjects() was waiting for the overlapped
+                # operation.
+                raise OSError(errno.EPIPE, "handle is closed")
             assert err == 0
             assert nwritten == len(buf)
 

py3.11/multiprocess/popen_spawn_win32.py

@@ -14,6 +14,7 @@
 #
 #
 
+# Exit code used by Popen.terminate()
 TERMINATE = 0x10000
 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
 WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
@@ -122,9 +123,15 @@ def terminate(self):
         if self.returncode is None:
             try:
                 _winapi.TerminateProcess(int(self._handle), TERMINATE)
-            except OSError:
-                if self.wait(timeout=1.0) is None:
+            except PermissionError:
+                # ERROR_ACCESS_DENIED (winerror 5) is received when the
+                # process already died.
+                code = _winapi.GetExitCodeProcess(int(self._handle))
+                if code == _winapi.STILL_ACTIVE:
                     raise
+                self.returncode = code
+            else:
+                self.returncode = -signal.SIGTERM
 
     kill = terminate
 

py3.11/multiprocess/resource_tracker.py

@@ -54,15 +54,31 @@
     })
 
 
+class ReentrantCallError(RuntimeError):
+    pass
+
+
 class ResourceTracker(object):
 
     def __init__(self):
-        self._lock = threading.Lock()
+        self._lock = threading.RLock()
         self._fd = None
         self._pid = None
 
+    def _reentrant_call_error(self):
+        # gh-109629: this happens if an explicit call to the ResourceTracker
+        # gets interrupted by a garbage collection, invoking a finalizer (*)
+        # that itself calls back into ResourceTracker.
+        #   (*) for example the SemLock finalizer
+        raise ReentrantCallError(
+            "Reentrant call into the multiprocess resource tracker")
+
     def _stop(self):
         with self._lock:
+            # This should not happen (_stop() isn't called by a finalizer)
+            # but we check for it anyway.
+            if getattr(self._lock, "_recursion_count", int)() > 1:
+                return self._reentrant_call_error()
             if self._fd is None:
                 # not running
                 return
@@ -84,6 +100,9 @@ def ensure_running(self):
         This can be run from any process.  Usually a child process will use
         the resource created by its parent.'''
         with self._lock:
+            if getattr(self._lock, "_recursion_count", int)() > 1:
+                # The code below is certainly not reentrant-safe, so bail out
+                return self._reentrant_call_error()
             if self._fd is not None:
                 # resource tracker was launched before, is it still running?
                 if self._check_alive():
@@ -162,7 +181,17 @@ def unregister(self, name, rtype):
         self._send('UNREGISTER', name, rtype)
 
     def _send(self, cmd, name, rtype):
-        self.ensure_running()
+        try:
+            self.ensure_running()
+        except ReentrantCallError:
+            # The code below might or might not work, depending on whether
+            # the resource tracker was already running and still alive.
+            # Better warn the user.
+            # (XXX is warnings.warn itself reentrant-safe? :-)
+            warnings.warn(
+                f"ResourceTracker called reentrantly for resource cleanup, "
+                f"which is unsupported. "
+                f"The {rtype} object {name!r} might leak.")
         msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii')
         if len(msg) > 512:
             # posix guarantees that writes to a pipe of less than PIPE_BUF
@@ -179,6 +208,7 @@ def _send(self, cmd, name, rtype):
 unregister = _resource_tracker.unregister
 getfd = _resource_tracker.getfd
 
+
 def main(fd):
     '''Run resource tracker.'''
     # protect the process from ^C and "killall python" etc