Merge remote-tracking branch 'upstream/main' into pythonrun

Author: iritkatriel

Include/cpython/pyerrors.h

@@ -112,6 +112,10 @@ PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause(
 
 /* In exceptions.c */
 
+PyAPI_FUNC(int) _PyException_AddNote(
+     PyObject *exc,
+     PyObject *note);
+
 /* Helper that attempts to replace the current exception with one of the
  * same type but with a prefix added to the exception text. The resulting
  * exception description looks like:

Lib/asyncio/tasks.py

@@ -25,7 +25,6 @@
 from . import exceptions
 from . import futures
 from . import timeouts
-from .coroutines import _is_coroutine
 
 # Helper to generate new task names
 # This uses itertools.count() instead of a "+= 1" operation because the latter
@@ -635,11 +634,14 @@ def ensure_future(coro_or_future, *, loop=None):
             raise ValueError('The future belongs to a different loop than '
                             'the one specified as the loop argument')
         return coro_or_future
-    called_wrap_awaitable = False
+    should_close = True
     if not coroutines.iscoroutine(coro_or_future):
         if inspect.isawaitable(coro_or_future):
+            async def _wrap_awaitable(awaitable):
+                return await awaitable
+
             coro_or_future = _wrap_awaitable(coro_or_future)
-            called_wrap_awaitable = True
+            should_close = False
         else:
             raise TypeError('An asyncio.Future, a coroutine or an awaitable '
                             'is required')
@@ -649,23 +651,11 @@ def ensure_future(coro_or_future, *, loop=None):
     try:
         return loop.create_task(coro_or_future)
     except RuntimeError:
-        if not called_wrap_awaitable:
+        if should_close:
             coro_or_future.close()
         raise
 
 
-@types.coroutine
-def _wrap_awaitable(awaitable):
-    """Helper for asyncio.ensure_future().
-
-    Wraps awaitable (an object with __await__) into a coroutine
-    that will later be wrapped in a Task by ensure_future().
-    """
-    return (yield from awaitable.__await__())
-
-_wrap_awaitable._is_coroutine = _is_coroutine
-
-
 class _GatheringFuture(futures.Future):
     """Helper for gather().
 

Lib/concurrent/futures/process.py

@@ -366,6 +366,11 @@ def run(self):
             if self.is_shutting_down():
                 self.flag_executor_shutting_down()
 
+                # When only canceled futures remain in pending_work_items, our
+                # next call to wait_result_broken_or_wakeup would hang forever.
+                # This makes sure we have some running futures or none at all.
+                self.add_call_item_to_queue()
+
                 # Since no new work items can be added, it is safe to shutdown
                 # this thread if there are no pending work items.
                 if not self.pending_work_items:

Lib/test/test_asyncio/test_tasks.py

@@ -8,6 +8,7 @@
 import re
 import sys
 import traceback
+import types
 import unittest
 from unittest import mock
 from types import GenericAlias
@@ -274,6 +275,20 @@ async def coro():
         loop.run_until_complete(fut)
         self.assertEqual(fut.result(), 'ok')
 
+    def test_ensure_future_task_awaitable(self):
+        class Aw:
+            def __await__(self):
+                return asyncio.sleep(0, result='ok').__await__()
+
+        loop = asyncio.new_event_loop()
+        self.set_event_loop(loop)
+        task = asyncio.ensure_future(Aw(), loop=loop)
+        loop.run_until_complete(task)
+        self.assertTrue(task.done())
+        self.assertEqual(task.result(), 'ok')
+        self.assertIsInstance(task.get_coro(), types.CoroutineType)
+        loop.close()
+
     def test_ensure_future_neither(self):
         with self.assertRaises(TypeError):
             asyncio.ensure_future('ok')

Lib/test/test_capi/test_exceptions.py

@@ -169,5 +169,25 @@ class Broken(Exception, metaclass=Meta):
         with self.assertRaises(ZeroDivisionError) as e:
             _testcapi.exc_set_object(Broken, Broken())
 
+    def test_set_object_and_fetch(self):
+        class Broken(Exception):
+            def __init__(self, *arg):
+                raise ValueError("Broken __init__")
+
+        exc = _testcapi.exc_set_object_fetch(Broken, 'abcd')
+        self.assertIsInstance(exc, ValueError)
+        self.assertEqual(exc.__notes__[0],
+                         "Normalization failed: type=Broken args='abcd'")
+
+        class BadArg:
+            def __repr__(self):
+                raise TypeError('Broken arg type')
+
+        exc = _testcapi.exc_set_object_fetch(Broken, BadArg())
+        self.assertIsInstance(exc, ValueError)
+        self.assertEqual(exc.__notes__[0],
+                         'Normalization failed: type=Broken args=<unknown>')
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_concurrent_futures.py

@@ -14,6 +14,7 @@
 from logging.handlers import QueueHandler
 import os
 import queue
+import signal
 import sys
 import threading
 import time
@@ -397,6 +398,33 @@ def test_hang_gh83386(self):
         self.assertFalse(err)
         self.assertEqual(out.strip(), b"apple")
 
+    def test_hang_gh94440(self):
+        """shutdown(wait=True) doesn't hang when a future was submitted and
+        quickly canceled right before shutdown.
+
+        See https://github.com/python/cpython/issues/94440.
+        """
+        if not hasattr(signal, 'alarm'):
+            raise unittest.SkipTest(
+                "Tested platform does not support the alarm signal")
+
+        def timeout(_signum, _frame):
+            raise RuntimeError("timed out waiting for shutdown")
+
+        kwargs = {}
+        if getattr(self, 'ctx', None):
+            kwargs['mp_context'] = self.get_context()
+        executor = self.executor_type(max_workers=1, **kwargs)
+        executor.submit(int).result()
+        old_handler = signal.signal(signal.SIGALRM, timeout)
+        try:
+            signal.alarm(5)
+            executor.submit(int).cancel()
+            executor.shutdown(wait=True)
+        finally:
+            signal.alarm(0)
+            signal.signal(signal.SIGALRM, old_handler)
+
 
 class ThreadPoolShutdownTest(ThreadPoolMixin, ExecutorShutdownTest, BaseTestCase):
     def test_threads_terminate(self):

Lib/webbrowser.py

@@ -542,11 +542,15 @@ def register_standard_browsers():
         # First try to use the default Windows browser
         register("windows-default", WindowsDefault)
 
-        # Detect some common Windows browsers, fallback to IE
-        iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"),
-                                "Internet Explorer\\IEXPLORE.EXE")
+        # Detect some common Windows browsers, fallback to Microsoft Edge
+        # location in 64-bit Windows
+        edge64 = os.path.join(os.environ.get("PROGRAMFILES(x86)", "C:\\Program Files (x86)"),
+                              "Microsoft\\Edge\\Application\\msedge.exe")
+        # location in 32-bit Windows
+        edge32 = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"),
+                              "Microsoft\\Edge\\Application\\msedge.exe")
         for browser in ("firefox", "firebird", "seamonkey", "mozilla",
-                        "netscape", "opera", iexplore):
+                        "opera", edge64, edge32):
             if shutil.which(browser):
                 register(browser, None, BackgroundBrowser(browser))
     else:

Misc/ACKS

@@ -1385,6 +1385,7 @@ Thomas Perl
 Mathieu Perreault
 Mark Perrego
 Trevor Perrin
+Yonatan Perry
 Gabriel de Perthuis
 Tim Peters
 Benjamin Peterson

Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst

@@ -0,0 +1 @@
+Add note to exception raised in ``PyErr_SetObject`` when normalization fails.

Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst

@@ -0,0 +1,2 @@
+Fix a :mod:`concurrent.futures.process` bug where ``ProcessPoolExecutor`` shutdown
+could hang after a future has been quickly submitted and canceled.

Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst

@@ -0,0 +1 @@
+:meth:`asyncio.Task.get_coro` now always returns a coroutine when wrapping an awaitable object. Patch by Kumar Aditya.

Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst

@@ -0,0 +1 @@
+Update :mod:`webbrowser` to fall back to Microsoft Edge instead of Internet Explorer.

Modules/_testcapi/exceptions.c

@@ -92,6 +92,26 @@ exc_set_object(PyObject *self, PyObject *args)
     return NULL;
 }
 
+static PyObject *
+exc_set_object_fetch(PyObject *self, PyObject *args)
+{
+    PyObject *exc;
+    PyObject *obj;
+    PyObject *type;
+    PyObject *value;
+    PyObject *tb;
+
+    if (!PyArg_ParseTuple(args, "OO:exc_set_object", &exc, &obj)) {
+        return NULL;
+    }
+
+    PyErr_SetObject(exc, obj);
+    PyErr_Fetch(&type, &value, &tb);
+    Py_XDECREF(type);
+    Py_XDECREF(tb);
+    return value;
+}
+
 static PyObject *
 raise_exception(PyObject *self, PyObject *args)
 {
@@ -262,6 +282,7 @@ static PyMethodDef test_methods[] = {
     {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc),
      METH_VARARGS | METH_KEYWORDS},
     {"exc_set_object",          exc_set_object,                  METH_VARARGS},
+    {"exc_set_object_fetch",    exc_set_object_fetch,            METH_VARARGS},
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",       raise_memoryerror,               METH_NOARGS},
     {"set_exc_info",            test_set_exc_info,               METH_VARARGS},

Objects/exceptions.c

@@ -3749,6 +3749,21 @@ _PyExc_Fini(PyInterpreterState *interp)
     _PyExc_FiniTypes(interp);
 }
 
+int
+_PyException_AddNote(PyObject *exc, PyObject *note)
+{
+    if (!PyExceptionInstance_Check(exc)) {
+        PyErr_Format(PyExc_TypeError,
+                     "exc must be an exception, not '%s'",
+                     Py_TYPE(exc)->tp_name);
+        return -1;
+    }
+    PyObject *r = BaseException_add_note(exc, note);
+    int res = r == NULL ? -1 : 0;
+    Py_XDECREF(r);
+    return res;
+}
+
 /* Helper to do the equivalent of "raise X from Y" in C, but always using
  * the current exception rather than passing one in.
  *

Python/bltinmodule.c

@@ -3098,9 +3098,6 @@ _PyBuiltin_Init(PyInterpreterState *interp)
     }
     Py_DECREF(debug);
 
-    /* m_copy of Py_None means it is copied some other way. */
-    builtinsmodule.m_base.m_copy = Py_NewRef(Py_None);
-
     return mod;
 #undef ADD_TO_ALL
 #undef SETBUILTIN

Python/errors.c

@@ -135,6 +135,28 @@ _PyErr_GetTopmostException(PyThreadState *tstate)
     return exc_info;
 }
 
+static PyObject *
+get_normalization_failure_note(PyThreadState *tstate, PyObject *exception, PyObject *value)
+{
+    PyObject *args = PyObject_Repr(value);
+    if (args == NULL) {
+        _PyErr_Clear(tstate);
+        args = PyUnicode_FromFormat("<unknown>");
+    }
+    PyObject *note;
+    const char *tpname = ((PyTypeObject*)exception)->tp_name;
+    if (args == NULL) {
+        _PyErr_Clear(tstate);
+        note = PyUnicode_FromFormat("Normalization failed: type=%s", tpname);
+    }
+    else {
+        note = PyUnicode_FromFormat("Normalization failed: type=%s args=%S",
+                                    tpname, args);
+        Py_DECREF(args);
+    }
+    return note;
+}
+
 void
 _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value)
 {
@@ -160,19 +182,27 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value)
     Py_XINCREF(value);
     if (!is_subclass) {
         /* We must normalize the value right now */
-        PyObject *fixed_value;
 
         /* Issue #23571: functions must not be called with an
             exception set */
         _PyErr_Clear(tstate);
 
-        fixed_value = _PyErr_CreateException(exception, value);
-        Py_XDECREF(value);
+        PyObject *fixed_value = _PyErr_CreateException(exception, value);
         if (fixed_value == NULL) {
+            PyObject *exc = _PyErr_GetRaisedException(tstate);
+            assert(PyExceptionInstance_Check(exc));
+
+            PyObject *note = get_normalization_failure_note(tstate, exception, value);
+            Py_XDECREF(value);
+            if (note != NULL) {
+                /* ignore errors in _PyException_AddNote - they will be overwritten below */
+                _PyException_AddNote(exc, note);
+                Py_DECREF(note);
+            }
+            _PyErr_SetRaisedException(tstate, exc);
             return;
         }
-
-        value = fixed_value;
+        Py_XSETREF(value, fixed_value);
     }
 
     exc_value = _PyErr_GetTopmostException(tstate)->exc_value;