Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Doc/library/stdtypes.rst
Original file line number Diff line number Diff line change
Expand Up @@ -2870,6 +2870,14 @@ expression support in the :mod:`re` module).
You can use :meth:`str.maketrans` to create a translation map from
character-to-character mappings in different formats.

The following example uses a mapping to replace ``'a'`` with ``'X'``,
``'b'`` with ``'Y'``, and delete ``'c'``:

.. doctest::

>>> 'abc123'.translate({ord('a'): 'X', ord('b'): 'Y', ord('c'): None})
'XY123'

See also the :mod:`codecs` module for a more flexible approach to custom
character mappings.

Expand Down
4 changes: 2 additions & 2 deletions Include/internal/mimalloc/mimalloc/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -608,8 +608,8 @@ struct mi_heap_s {

#if (MI_DEBUG)
// use our own assertion to print without memory allocation
mi_decl_noreturn mi_decl_cold mi_decl_throw
void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func);
mi_decl_noreturn mi_decl_cold
void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_decl_throw;
#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))
#else
#define mi_assert(x)
Expand Down
4 changes: 4 additions & 0 deletions Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3116,6 +3116,10 @@ def get_signal_name(exitcode):
except KeyError:
pass

# Format Windows exit status as hexadecimal
if 0xC0000000 <= exitcode:
return f"0x{exitcode:X}"

return None

class BrokenIter:
Expand Down
4 changes: 1 addition & 3 deletions Lib/test/test_cppext/extension.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,8 @@
#ifdef TEST_INTERNAL_C_API
// gh-135906: Check for compiler warnings in the internal C API
# include "internal/pycore_frame.h"
// mimalloc emits many compiler warnings when Python is built in debug
// mode (when MI_DEBUG is not zero).
// mimalloc emits compiler warnings when Python is built on Windows.
# if !defined(Py_DEBUG) && !defined(MS_WINDOWS)
# if !defined(MS_WINDOWS)
# include "internal/pycore_backoff.h"
# include "internal/pycore_cell.h"
# endif
Expand Down
38 changes: 38 additions & 0 deletions Lib/test/test_external_inspection.py
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,44 @@ def foo():
finally:
_cleanup_sockets(client_socket, server_socket)

@skip_if_not_supported
@unittest.skipIf(
sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
"Test only runs on Linux with process_vm_readv support",
)
def test_self_trace_after_ctypes_import(self):
"""Test that RemoteUnwinder works on the same process after _ctypes import.

When _ctypes is imported, it may call dlopen on the libpython shared
library, creating a duplicate mapping in the process address space.
The remote debugging code must skip these uninitialized duplicate
mappings and find the real PyRuntime. See gh-144563.
"""
# Run the test in a subprocess to avoid side effects
script = textwrap.dedent("""\
import os
import _remote_debugging

# Should work before _ctypes import
unwinder = _remote_debugging.RemoteUnwinder(os.getpid())

import _ctypes

# Should still work after _ctypes import (gh-144563)
unwinder = _remote_debugging.RemoteUnwinder(os.getpid())
""")

result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
timeout=SHORT_TIMEOUT,
)
self.assertEqual(
result.returncode, 0,
f"stdout: {result.stdout}\nstderr: {result.stderr}"
)

@skip_if_not_supported
@unittest.skipIf(
sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
Expand Down
1 change: 1 addition & 0 deletions Lib/test/test_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,7 @@ def test_get_signal_name(self):
(128 + int(signal.SIGABRT), 'SIGABRT'),
(3221225477, "STATUS_ACCESS_VIOLATION"),
(0xC00000FD, "STATUS_STACK_OVERFLOW"),
(0xC0000906, "0xC0000906"),
):
self.assertEqual(support.get_signal_name(exitcode), expected,
exitcode)
Expand Down
12 changes: 6 additions & 6 deletions Mac/BuildScript/build-installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,9 @@ def library_recipes():

result.extend([
dict(
name="OpenSSL 3.5.4",
url="https://github.com/openssl/openssl/releases/download/openssl-3.5.4/openssl-3.5.4.tar.gz",
checksum="967311f84955316969bdb1d8d4b983718ef42338639c621ec4c34fddef355e99",
name="OpenSSL 3.5.5",
url="https://github.com/openssl/openssl/releases/download/openssl-3.5.5/openssl-3.5.5.tar.gz",
checksum="b28c91532a8b65a1f983b4c28b7488174e4a01008e29ce8e69bd789f28bc2a89",
buildrecipe=build_universal_openssl,
configure=None,
install=None,
Expand All @@ -264,10 +264,10 @@ def library_recipes():
tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch']

else:
tcl_tk_ver='9.0.2'
tcl_checksum='e074c6a8d9ba2cddf914ba97b6677a552d7a52a3ca102924389a05ccb249b520'
tcl_tk_ver='9.0.3'
tcl_checksum='2537ba0c86112c8c953f7c09d33f134dd45c0fb3a71f2d7f7691fd301d2c33a6'

tk_checksum='76fb852b2f167592fe8b41aa6549ce4e486dbf3b259a269646600e3894517c76'
tk_checksum='bf344efadb618babb7933f69275620f72454d1c8220130da93e3f7feb0efbf9b'
tk_patches = []


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix interaction of the Tachyon profiler and :mod:`ctypes` and other modules
that load the Python shared library (if present) in an independent map as
this was causing the mechanism that loads the binary information to be
confused. Patch by Pablo Galindo
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Update macOS installer to use OpenSSL 3.5.5.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Update macOS installer to use Tcl/Tk 9.0.3.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Allowed _remote_debugging to build on more OS versions by using proc_listpids() rather than proc_listallpids().
12 changes: 8 additions & 4 deletions Modules/_remote_debugging/asyncio.c
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle)

#ifdef MS_WINDOWS
// On Windows, search for asyncio debug in executable or DLL
address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio");
address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio",
NULL);
if (address == 0) {
// Error out: 'python' substring covers both executable and DLL
PyObject *exc = PyErr_GetRaisedException();
Expand All @@ -27,7 +28,8 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle)
}
#elif defined(__linux__) && HAVE_PROCESS_VM_READV
// On Linux, search for asyncio debug in executable or DLL
address = search_linux_map_for_section(handle, "AsyncioDebug", "python");
address = search_linux_map_for_section(handle, "AsyncioDebug", "python",
NULL);
if (address == 0) {
// Error out: 'python' substring covers both executable and DLL
PyObject *exc = PyErr_GetRaisedException();
Expand All @@ -36,10 +38,12 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle)
}
#elif defined(__APPLE__) && TARGET_OS_OSX
// On macOS, try libpython first, then fall back to python
address = search_map_for_section(handle, "AsyncioDebug", "libpython");
address = search_map_for_section(handle, "AsyncioDebug", "libpython",
NULL);
if (address == 0) {
PyErr_Clear();
address = search_map_for_section(handle, "AsyncioDebug", "python");
address = search_map_for_section(handle, "AsyncioDebug", "python",
NULL);
}
if (address == 0) {
// Error out: 'python' substring covers both executable and DLL
Expand Down
5 changes: 3 additions & 2 deletions Modules/_remote_debugging/subprocess.c
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result)

#if defined(__APPLE__) && TARGET_OS_OSX

#include <libproc.h>
#include <sys/proc_info.h>

static int
Expand All @@ -283,7 +284,7 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result)
pid_t *ppids = NULL;

/* Get count of all PIDs */
int n_pids = proc_listallpids(NULL, 0);
int n_pids = proc_listpids(PROC_ALL_PIDS, 0, NULL, 0);
if (n_pids <= 0) {
PyErr_SetString(PyExc_OSError, "Failed to get process count");
goto done;
Expand All @@ -298,7 +299,7 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result)
}

/* Get actual PIDs */
int actual = proc_listallpids(pid_list, buffer_size * sizeof(pid_t));
int actual = proc_listpids(PROC_ALL_PIDS, 0, pid_list, buffer_size * sizeof(pid_t));
if (actual <= 0) {
PyErr_SetString(PyExc_OSError, "Failed to list PIDs");
goto done;
Expand Down
59 changes: 49 additions & 10 deletions Python/remote_debug.h
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,31 @@ typedef struct {
Py_ssize_t page_size;
} proc_handle_t;

// Forward declaration for use in validation function
static int
_Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst);

// Optional callback to validate a candidate section address found during
// memory map searches. Returns 1 if the address is valid, 0 to skip it.
// This allows callers to filter out duplicate/stale mappings (e.g. from
// ctypes dlopen) whose sections were never initialized.
typedef int (*section_validator_t)(proc_handle_t *handle, uintptr_t address);

// Validate that a candidate address starts with _Py_Debug_Cookie.
static int
_Py_RemoteDebug_ValidatePyRuntimeCookie(proc_handle_t *handle, uintptr_t address)
{
if (address == 0) {
return 0;
}
char buf[sizeof(_Py_Debug_Cookie) - 1];
if (_Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(buf), buf) != 0) {
PyErr_Clear();
return 0;
}
return memcmp(buf, _Py_Debug_Cookie, sizeof(buf)) == 0;
}

static void
_Py_RemoteDebug_FreePageCache(proc_handle_t *handle)
{
Expand Down Expand Up @@ -509,7 +534,8 @@ pid_to_task(pid_t pid)
}

static uintptr_t
search_map_for_section(proc_handle_t *handle, const char* secname, const char* substr) {
search_map_for_section(proc_handle_t *handle, const char* secname, const char* substr,
section_validator_t validator) {
mach_vm_address_t address = 0;
mach_vm_size_t size = 0;
mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t);
Expand Down Expand Up @@ -561,7 +587,9 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s
if (strncmp(filename, substr, strlen(substr)) == 0) {
uintptr_t result = search_section_in_file(
secname, map_filename, address, size, proc_ref);
if (result != 0) {
if (result != 0
&& (validator == NULL || validator(handle, result)))
{
return result;
}
}
Expand Down Expand Up @@ -678,7 +706,8 @@ search_elf_file_for_section(
}

static uintptr_t
search_linux_map_for_section(proc_handle_t *handle, const char* secname, const char* substr)
search_linux_map_for_section(proc_handle_t *handle, const char* secname, const char* substr,
section_validator_t validator)
{
char maps_file_path[64];
sprintf(maps_file_path, "/proc/%d/maps", handle->pid);
Expand Down Expand Up @@ -753,9 +782,12 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c

if (strstr(filename, substr)) {
retval = search_elf_file_for_section(handle, secname, start, path);
if (retval) {
if (retval
&& (validator == NULL || validator(handle, retval)))
{
break;
}
retval = 0;
}
}

Expand Down Expand Up @@ -859,7 +891,8 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char*


static uintptr_t
search_windows_map_for_section(proc_handle_t* handle, const char* secname, const wchar_t* substr) {
search_windows_map_for_section(proc_handle_t* handle, const char* secname, const wchar_t* substr,
section_validator_t validator) {
HANDLE hProcSnap;
do {
hProcSnap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, handle->pid);
Expand All @@ -882,8 +915,11 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const
for (BOOL hasModule = Module32FirstW(hProcSnap, &moduleEntry); hasModule; hasModule = Module32NextW(hProcSnap, &moduleEntry)) {
// Look for either python executable or DLL
if (wcsstr(moduleEntry.szModule, substr)) {
runtime_addr = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname);
if (runtime_addr != NULL) {
void *candidate = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname);
if (candidate != NULL
&& (validator == NULL || validator(handle, (uintptr_t)candidate)))
{
runtime_addr = candidate;
break;
}
}
Expand All @@ -904,7 +940,8 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle)

#ifdef MS_WINDOWS
// On Windows, search for 'python' in executable or DLL
address = search_windows_map_for_section(handle, "PyRuntime", L"python");
address = search_windows_map_for_section(handle, "PyRuntime", L"python",
_Py_RemoteDebug_ValidatePyRuntimeCookie);
if (address == 0) {
// Error out: 'python' substring covers both executable and DLL
PyObject *exc = PyErr_GetRaisedException();
Expand All @@ -915,7 +952,8 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle)
}
#elif defined(__linux__) && HAVE_PROCESS_VM_READV
// On Linux, search for 'python' in executable or DLL
address = search_linux_map_for_section(handle, "PyRuntime", "python");
address = search_linux_map_for_section(handle, "PyRuntime", "python",
_Py_RemoteDebug_ValidatePyRuntimeCookie);
if (address == 0) {
// Error out: 'python' substring covers both executable and DLL
PyObject *exc = PyErr_GetRaisedException();
Expand All @@ -929,7 +967,8 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle)
const char* candidates[] = {"libpython", "python", "Python", NULL};
for (const char** candidate = candidates; *candidate; candidate++) {
PyErr_Clear();
address = search_map_for_section(handle, "PyRuntime", *candidate);
address = search_map_for_section(handle, "PyRuntime", *candidate,
_Py_RemoteDebug_ValidatePyRuntimeCookie);
if (address != 0) {
break;
}
Expand Down
Loading