Merge branch 'main' into small_int_immortal

Author: eendebakpt

Lib/test/support/strace_helper.py

@@ -71,6 +71,27 @@ def sections(self):
 
         return sections
 
+def _filter_memory_call(call):
+    # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory.
+    # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias.
+    if call.syscall == "mmap" and "MAP_ANON" in call.args[3]:
+        return True
+
+    if call.syscall in ("munmap", "mprotect"):
+        return True
+
+    return False
+
+
+def filter_memory(syscalls):
+    """Filter out memory allocation calls from File I/O calls.
+
+    Some calls (mmap, munmap, etc) can be used on files or to just get a block
+    of memory. Use this function to filter out the memory related calls from
+    other calls."""
+
+    return [call for call in syscalls if not _filter_memory_call(call)]
+
 
 @support.requires_subprocess()
 def strace_python(code, strace_flags, check=True):
@@ -93,8 +114,6 @@ def _make_error(reason, details):
             "-c",
             textwrap.dedent(code),
             __run_using_command=[_strace_binary] + strace_flags,
-            # Don't want to trace our JIT's own mmap and mprotect calls:
-            PYTHON_JIT="0",
         )
     except OSError as err:
         return _make_error("Caught OSError", err)
@@ -145,9 +164,14 @@ def get_events(code, strace_flags, prelude, cleanup):
     return all_sections['code']
 
 
-def get_syscalls(code, strace_flags, prelude="", cleanup=""):
+def get_syscalls(code, strace_flags, prelude="", cleanup="",
+                 ignore_memory=True):
     """Get the syscalls which a given chunk of python code generates"""
     events = get_events(code, strace_flags, prelude=prelude, cleanup=cleanup)
+
+    if ignore_memory:
+        events = filter_memory(events)
+
     return [ev.syscall for ev in events]
 
 
@@ -177,5 +201,5 @@ def requires_strace():
     return unittest.skipUnless(_can_strace(), "Requires working strace")
 
 
-__all__ = ["get_events", "get_syscalls", "requires_strace", "strace_python",
-           "StraceEvent", "StraceResult"]
+__all__ = ["filter_memory", "get_events", "get_syscalls", "requires_strace",
+           "strace_python", "StraceEvent", "StraceResult"]

Lib/test/test_fileio.py

@@ -364,8 +364,7 @@ def testErrnoOnClosedReadinto(self, f):
 
     @strace_helper.requires_strace()
     def test_syscalls_read(self):
-        """Check that the set of system calls produced by the I/O stack is what
-        is expected for various read cases.
+        """Check set of system calls during common I/O patterns
 
         It's expected as bits of the I/O implementation change, this will need
         to change. The goal is to catch changes that unintentionally add
@@ -383,6 +382,11 @@ def check_readall(name, code, prelude="", cleanup="",
                                                       prelude=prelude,
                                                       cleanup=cleanup)
 
+                # Some system calls (ex. mmap) can be used for both File I/O and
+                # memory allocation. Filter out the ones used for memory
+                # allocation.
+                syscalls = strace_helper.filter_memory(syscalls)
+
                 # The first call should be an open that returns a
                 # file descriptor (fd). Afer that calls may vary. Once the file
                 # is opened, check calls refer to it by fd as the filename

Lib/test/test_tools/i18n_data/messages.pot

@@ -15,53 +15,75 @@ msgstr ""
 "Generated-By: pygettext.py 1.5\n"
 
 
-#: messages.py:5
+#: messages.py:16
 msgid ""
 msgstr ""
 
-#: messages.py:8 messages.py:9
+#: messages.py:19 messages.py:20
 msgid "parentheses"
 msgstr ""
 
-#: messages.py:12
+#: messages.py:23
 msgid "Hello, world!"
 msgstr ""
 
-#: messages.py:15
+#: messages.py:26
 msgid ""
 "Hello,\n"
 "    multiline!\n"
 msgstr ""
 
-#: messages.py:29
+#: messages.py:46 messages.py:89 messages.py:90 messages.py:93 messages.py:94
+#: messages.py:99
+msgid "foo"
+msgid_plural "foos"
+msgstr[0] ""
+msgstr[1] ""
+
+#: messages.py:47
+msgid "something"
+msgstr ""
+
+#: messages.py:50
 msgid "Hello, {}!"
 msgstr ""
 
-#: messages.py:33
+#: messages.py:54
 msgid "1"
 msgstr ""
 
-#: messages.py:33
+#: messages.py:54
 msgid "2"
 msgstr ""
 
-#: messages.py:34 messages.py:35
+#: messages.py:55 messages.py:56
 msgid "A"
 msgstr ""
 
-#: messages.py:34 messages.py:35
+#: messages.py:55 messages.py:56
 msgid "B"
 msgstr ""
 
-#: messages.py:36
+#: messages.py:57
 msgid "set"
 msgstr ""
 
-#: messages.py:42
+#: messages.py:63
 msgid "nested string"
 msgstr ""
 
-#: messages.py:47
+#: messages.py:68
 msgid "baz"
 msgstr ""
 
+#: messages.py:91 messages.py:92 messages.py:95 messages.py:96
+msgctxt "context"
+msgid "foo"
+msgid_plural "foos"
+msgstr[0] ""
+msgstr[1] ""
+
+#: messages.py:100
+msgid "domain foo"
+msgstr ""
+

Lib/test/test_tools/i18n_data/messages.py

@@ -1,5 +1,16 @@
 # Test message extraction
-from gettext import gettext as _
+from gettext import (
+    gettext,
+    ngettext,
+    pgettext,
+    npgettext,
+    dgettext,
+    dngettext,
+    dpgettext,
+    dnpgettext
+)
+
+_ = gettext
 
 # Empty string
 _("")
@@ -21,13 +32,23 @@
 _(None)
 _(1)
 _(False)
-_(x="kwargs are not allowed")
+_(("invalid"))
+_(["invalid"])
+_({"invalid"})
+_("string"[3])
+_("string"[:3])
+_({"string": "foo"})
+
+# pygettext does not allow keyword arguments, but both xgettext and pybabel do
+_(x="kwargs work!")
+
+# Unusual, but valid arguments
 _("foo", "bar")
 _("something", x="something else")
 
 # .format()
 _("Hello, {}!").format("world")  # valid
-_("Hello, {}!".format("world"))  # invalid
+_("Hello, {}!".format("world"))  # invalid, but xgettext and pybabel extract the first string
 
 # Nested structures
 _("1"), _("2")
@@ -62,3 +83,28 @@ def _(x):
 
 def _(x="don't extract me"):
     pass
+
+
+# Other gettext functions
+gettext("foo")
+ngettext("foo", "foos", 1)
+pgettext("context", "foo")
+npgettext("context", "foo", "foos", 1)
+dgettext("domain", "foo")
+dngettext("domain", "foo", "foos", 1)
+dpgettext("domain", "context", "foo")
+dnpgettext("domain", "context", "foo", "foos", 1)
+
+# Complex arguments
+ngettext("foo", "foos", 42 + (10 - 20))
+dgettext(["some", {"complex"}, ("argument",)], "domain foo")
+
+# Invalid calls which are not extracted
+gettext()
+ngettext('foo')
+pgettext('context')
+npgettext('context', 'foo')
+dgettext('domain')
+dngettext('domain', 'foo')
+dpgettext('domain', 'context')
+dnpgettext('domain', 'context', 'foo')

Lib/test/test_tools/test_i18n.py

@@ -332,14 +332,14 @@ def test_calls_in_fstring_with_multiple_args(self):
         msgids = self.extract_docstrings_from_str(dedent('''\
         f"{_('foo', 'bar')}"
         '''))
-        self.assertNotIn('foo', msgids)
+        self.assertIn('foo', msgids)
         self.assertNotIn('bar', msgids)
 
     def test_calls_in_fstring_with_keyword_args(self):
         msgids = self.extract_docstrings_from_str(dedent('''\
         f"{_('foo', bar='baz')}"
         '''))
-        self.assertNotIn('foo', msgids)
+        self.assertIn('foo', msgids)
         self.assertNotIn('bar', msgids)
         self.assertNotIn('baz', msgids)
 

Lib/test/translationdata/argparse/msgids.txt

@@ -8,6 +8,8 @@ argument %(argument_name)s: %(message)s
 argument '%(argument_name)s' is deprecated
 can't open '%(filename)s': %(error)s
 command '%(parser_name)s' is deprecated
+conflicting option string: %s
+expected %s argument
 expected at least one argument
 expected at most one argument
 expected one argument

Lib/test/translationdata/optparse/msgids.txt

@@ -1,3 +1,4 @@
+%(option)s option requires %(number)d argument
 %prog [options]
 %s option does not take a value
 Options

Misc/NEWS.d/next/Library/2024-11-21-16-23-16.gh-issue-127065.cfL1zd.rst

@@ -0,0 +1,2 @@
+Fix crash when calling a :func:`operator.methodcaller` instance from
+multiple threads in the free threading build.

Misc/NEWS.d/next/Tests/2024-11-20-18-49-01.gh-issue-127076.DHnXxo.rst

@@ -0,0 +1,2 @@
+Filter out memory-related ``mmap``, ``munmap``, and ``mprotect``  calls from
+file-related ones when testing :mod:`io` behavior using strace.

Misc/NEWS.d/next/Tools-Demos/2024-11-16-20-47-20.gh-issue-126700.ayrHv4.rst

@@ -0,0 +1 @@
+Add support for multi-argument :mod:`gettext` functions in :program:`pygettext.py`.

Modules/_operator.c

@@ -1602,6 +1602,7 @@ typedef struct {
     vectorcallfunc vectorcall;
 } methodcallerobject;
 
+#ifndef Py_GIL_DISABLED
 static int _methodcaller_initialize_vectorcall(methodcallerobject* mc)
 {
     PyObject* args = mc->xargs;
@@ -1664,6 +1665,7 @@ methodcaller_vectorcall(
             (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET,
             mc->vectorcall_kwnames);
 }
+#endif
 
 
 /* AC 3.5: variable number of arguments, not currently support by AC */
@@ -1703,7 +1705,14 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     mc->vectorcall_args = 0;
 
 
+#ifdef Py_GIL_DISABLED
+    // gh-127065: The current implementation of methodcaller_vectorcall
+    // is not thread-safe because it modifies the `vectorcall_args` array,
+    // which is shared across calls.
+    mc->vectorcall = NULL;
+#else
     mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall;
+#endif
 
     PyObject_GC_Track(mc);
     return (PyObject *)mc;