Merge branch 'main' into refactor-dict-tuple-opt

rashes2006 · web-flow · commit 7dbd1632ffdc · 2026-02-06T04:09:54.000+05:30
diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst
@@ -78,6 +78,14 @@ although there is currently no date scheduled for their removal.
 
 * :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process.
 
+* :mod:`os.path`: :func:`os.path.commonprefix` is deprecated, use
+  :func:`os.path.commonpath` for path prefixes. The :func:`os.path.commonprefix`
+  function is being deprecated due to having a misleading name and module.
+  The function is not safe to use for path prefixes despite being included in a
+  module about path manipulation, meaning it is easy to accidentally
+  introduce path traversal vulnerabilities into Python programs by using this
+  function.
+
 * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is
   deprecated, use an exception instance.
 
diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst
@@ -120,6 +120,14 @@ the :mod:`glob` module.)
    .. versionchanged:: 3.6
       Accepts a :term:`path-like object`.
 
+   .. deprecated:: next
+      Deprecated in favor of :func:`os.path.commonpath` for path prefixes.
+      The :func:`os.path.commonprefix` function is being deprecated due to
+      having a misleading name and module. The function is not safe to use for
+      path prefixes despite being included in a module about path manipulation,
+      meaning it is easy to accidentally introduce path traversal
+      vulnerabilities into Python programs by using this function.
+
 
 .. function:: dirname(path, /)
 
diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst
@@ -704,6 +704,13 @@ os.path
   (Contributed by Petr Viktorin for :cve:`2025-4517`.)
 
 
+pickle
+------
+
+* Add support for pickling private methods and nested classes.
+  (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.)
+
+
 resource
 --------
 
diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h
@@ -151,7 +151,12 @@ extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject
 extern void _PySymtable_Free(struct symtable *);
 
 extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name);
-extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
+
+// Export for '_pickle' shared extension
+PyAPI_FUNC(PyObject *)
+_Py_Mangle(PyObject *, PyObject *);
+PyAPI_FUNC(int)
+_Py_IsPrivateName(PyObject *);
 
 /* Flags for def-use information */
 
diff --git a/Lib/genericpath.py b/Lib/genericpath.py
@@ -105,6 +105,15 @@ def getctime(filename, /):
 # Return the longest prefix of all list elements.
 def commonprefix(m, /):
     "Given a list of pathnames, returns the longest common leading component"
+    import warnings
+    warnings.warn('os.path.commonprefix() is deprecated. Use '
+                  'os.path.commonpath() for longest path prefix.',
+                  category=DeprecationWarning,
+                  stacklevel=2)
+    return _commonprefix(m)
+
+def _commonprefix(m, /):
+    "Internal implementation of commonprefix()"
     if not m: return ''
     # Some people pass in a list of pathname parts to operate in an OS-agnostic
     # fashion; don't try to translate in that case as that's an abuse of the
diff --git a/Lib/pickle.py b/Lib/pickle.py
@@ -1175,6 +1175,17 @@ def save_global(self, obj, name=None):
             if name is None:
                 name = obj.__name__
 
+        if '.__' in name:
+            # Mangle names of private attributes.
+            dotted_path = name.split('.')
+            for i, subpath in enumerate(dotted_path):
+                if i and subpath.startswith('__') and not subpath.endswith('__'):
+                    prev = prev.lstrip('_')
+                    if prev:
+                        dotted_path[i] = f"_{prev.lstrip('_')}{subpath}"
+                prev = subpath
+            name = '.'.join(dotted_path)
+
         module_name = whichmodule(obj, name)
         if self.proto >= 2:
             code = _extension_registry.get((module_name, name), _NoValue)
diff --git a/Lib/posixpath.py b/Lib/posixpath.py
@@ -542,7 +542,7 @@ def relpath(path, start=None):
         start_list = start_tail.split(sep) if start_tail else []
         path_list = path_tail.split(sep) if path_tail else []
         # Work out how much of the filepath is shared by start and path.
-        i = len(commonprefix([start_list, path_list]))
+        i = len(genericpath._commonprefix([start_list, path_list]))
 
         rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
         if not rel_list:
diff --git a/Lib/test/picklecommon.py b/Lib/test/picklecommon.py
@@ -388,3 +388,48 @@ def pie(self):
 class Subclass(tuple):
     class Nested(str):
         pass
+
+# For test_private_methods
+class PrivateMethods:
+    def __init__(self, value):
+        self.value = value
+
+    def __private_method(self):
+        return self.value
+
+    def get_method(self):
+        return self.__private_method
+
+    @classmethod
+    def get_unbound_method(cls):
+        return cls.__private_method
+
+    @classmethod
+    def __private_classmethod(cls):
+        return 43
+
+    @classmethod
+    def get_classmethod(cls):
+        return cls.__private_classmethod
+
+    @staticmethod
+    def __private_staticmethod():
+        return 44
+
+    @classmethod
+    def get_staticmethod(cls):
+        return cls.__private_staticmethod
+
+# For test_private_nested_classes
+class PrivateNestedClasses:
+    @classmethod
+    def get_nested(cls):
+        return cls.__Nested
+
+    class __Nested:
+        @classmethod
+        def get_nested2(cls):
+            return cls.__Nested2
+
+        class __Nested2:
+            pass
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
@@ -4118,6 +4118,33 @@ def test_c_methods(self):
                 with self.subTest(proto=proto, descr=descr):
                     self.assertRaises(TypeError, self.dumps, descr, proto)
 
+    def test_private_methods(self):
+        if self.py_version < (3, 15):
+            self.skipTest('not supported in Python < 3.15')
+        obj = PrivateMethods(42)
+        for proto in protocols:
+            with self.subTest(proto=proto):
+                unpickled = self.loads(self.dumps(obj.get_method(), proto))
+                self.assertEqual(unpickled(), 42)
+                unpickled = self.loads(self.dumps(obj.get_unbound_method(), proto))
+                self.assertEqual(unpickled(obj), 42)
+                unpickled = self.loads(self.dumps(obj.get_classmethod(), proto))
+                self.assertEqual(unpickled(), 43)
+                unpickled = self.loads(self.dumps(obj.get_staticmethod(), proto))
+                self.assertEqual(unpickled(), 44)
+
+    def test_private_nested_classes(self):
+        if self.py_version < (3, 15):
+            self.skipTest('not supported in Python < 3.15')
+        cls1 = PrivateNestedClasses.get_nested()
+        cls2 = cls1.get_nested2()
+        for proto in protocols:
+            with self.subTest(proto=proto):
+                unpickled = self.loads(self.dumps(cls1, proto))
+                self.assertIs(unpickled, cls1)
+                unpickled = self.loads(self.dumps(cls2, proto))
+                self.assertIs(unpickled, cls2)
+
     def test_object_with_attrs(self):
         obj = Object()
         obj.a = 1
diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
@@ -34,6 +34,10 @@ def test_no_argument(self):
                                 .format(self.pathmodule.__name__, attr))
 
     def test_commonprefix(self):
+        with warnings_helper.check_warnings((".*commonpath().*", DeprecationWarning)):
+            self.do_test_commonprefix()
+
+    def do_test_commonprefix(self):
         commonprefix = self.pathmodule.commonprefix
         self.assertEqual(
             commonprefix([]),
@@ -606,8 +610,9 @@ def test_path_isdir(self):
         self.assertPathEqual(os.path.isdir)
 
     def test_path_commonprefix(self):
-        self.assertEqual(os.path.commonprefix([self.file_path, self.file_name]),
-                         self.file_name)
+        with warnings_helper.check_warnings((".*commonpath().*", DeprecationWarning)):
+            self.assertEqual(os.path.commonprefix([self.file_path, self.file_name]),
+                             self.file_name)
 
     def test_path_getsize(self):
         self.assertPathEqual(os.path.getsize)
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
@@ -10,6 +10,7 @@
 from ntpath import ALL_BUT_LAST, ALLOW_MISSING
 from test import support
 from test.support import os_helper
+from test.support import warnings_helper
 from test.support.os_helper import FakePath
 from test import test_genericpath
 from tempfile import TemporaryFile
@@ -298,6 +299,10 @@ def test_isabs(self):
         tester('ntpath.isabs("\\\\.\\C:")', 1)
 
     def test_commonprefix(self):
+        with warnings_helper.check_warnings((".*commonpath().*", DeprecationWarning)):
+            self.do_test_commonprefix()
+
+    def do_test_commonprefix(self):
         tester('ntpath.commonprefix(["/home/swenson/spam", "/home/swen/spam"])',
                "/home/swen")
         tester('ntpath.commonprefix(["\\home\\swen\\spam", "\\home\\swen\\eggs"])',
diff --git a/Lib/unittest/util.py b/Lib/unittest/util.py
@@ -1,7 +1,6 @@
 """Various utility functions."""
 
 from collections import namedtuple, Counter
-from os.path import commonprefix
 
 __unittest = True
 
@@ -21,13 +20,23 @@ def _shorten(s, prefixlen, suffixlen):
         s = '%s[%d chars]%s' % (s[:prefixlen], skip, s[len(s) - suffixlen:])
     return s
 
+def _common_prefix(m):
+    if not m:
+        return ""
+    s1 = min(m)
+    s2 = max(m)
+    for i, c in enumerate(s1):
+        if c != s2[i]:
+            return s1[:i]
+    return s1
+
 def _common_shorten_repr(*args):
     args = tuple(map(safe_repr, args))
     maxlen = max(map(len, args))
     if maxlen <= _MAX_LENGTH:
         return args
 
-    prefix = commonprefix(args)
+    prefix = _common_prefix(args)
     prefixlen = len(prefix)
 
     common_len = _MAX_LENGTH - \
diff --git a/Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst b/Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst
@@ -0,0 +1 @@
+The :mod:`pickle` module now properly handles name-mangled private methods.
diff --git a/Misc/NEWS.d/next/Library/2026-02-02-12-09-38.gh-issue-74453.19h4Z5.rst b/Misc/NEWS.d/next/Library/2026-02-02-12-09-38.gh-issue-74453.19h4Z5.rst
@@ -0,0 +1,8 @@
+Deprecate :func:`os.path.commonprefix` in favor of
+:func:`os.path.commonpath` for path segment prefixes.
+
+The :func:`os.path.commonprefix` function is being deprecated due to
+having a misleading name and module. The function is not safe to use for
+path prefixes despite being included in a module about path manipulation,
+meaning it is easy to accidentally introduce path traversal
+vulnerabilities into Python programs by using this function.
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
@@ -19,6 +19,7 @@
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_runtime.h"       // _Py_ID()
 #include "pycore_setobject.h"     // _PySet_NextEntry()
+#include "pycore_symtable.h"      // _Py_Mangle()
 #include "pycore_sysmodule.h"     // _PySys_GetSizeOf()
 #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString()
 
@@ -1928,6 +1929,37 @@ get_dotted_path(PyObject *name)
     return PyUnicode_Split(name, _Py_LATIN1_CHR('.'), -1);
 }
 
+static PyObject *
+join_dotted_path(PyObject *dotted_path)
+{
+    return PyUnicode_Join(_Py_LATIN1_CHR('.'), dotted_path);
+}
+
+/* Returns -1 (with an exception set) on error, 0 if there were no changes,
+ * 1 if some names were mangled. */
+static int
+mangle_dotted_path(PyObject *dotted_path)
+{
+    int rc = 0;
+    Py_ssize_t n = PyList_GET_SIZE(dotted_path);
+    for (Py_ssize_t i = n-1; i > 0; i--) {
+        PyObject *subpath = PyList_GET_ITEM(dotted_path, i);
+        if (_Py_IsPrivateName(subpath)) {
+            PyObject *parent = PyList_GET_ITEM(dotted_path, i-1);
+            PyObject *mangled = _Py_Mangle(parent, subpath);
+            if (mangled == NULL) {
+                return -1;
+            }
+            if (mangled != subpath) {
+                rc = 1;
+            }
+            PyList_SET_ITEM(dotted_path, i, mangled);
+            Py_DECREF(subpath);
+        }
+    }
+    return rc;
+}
+
 static int
 check_dotted_path(PickleState *st, PyObject *obj, PyObject *dotted_path)
 {
@@ -3809,6 +3841,15 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
     dotted_path = get_dotted_path(global_name);
     if (dotted_path == NULL)
         goto error;
+    switch (mangle_dotted_path(dotted_path)) {
+        case -1:
+            goto error;
+        case 1:
+            Py_SETREF(global_name, join_dotted_path(dotted_path));
+            if (global_name == NULL) {
+                goto error;
+            }
+    }
     module_name = whichmodule(st, obj, global_name, dotted_path);
     if (module_name == NULL)
         goto error;
diff --git a/Objects/classobject.c b/Objects/classobject.c
@@ -7,6 +7,7 @@
 #include "pycore_object.h"
 #include "pycore_pyerrors.h"
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_symtable.h"      // _Py_Mangle()
 #include "pycore_weakref.h"       // FT_CLEAR_WEAKREFS()
 
 
@@ -143,6 +144,20 @@ method___reduce___impl(PyMethodObject *self)
     if (funcname == NULL) {
         return NULL;
     }
+    if (_Py_IsPrivateName(funcname)) {
+        PyObject *classname = PyType_Check(funcself)
+            ? PyType_GetName((PyTypeObject *)funcself)
+            : PyType_GetName(Py_TYPE(funcself));
+        if (classname == NULL) {
+            Py_DECREF(funcname);
+            return NULL;
+        }
+        Py_SETREF(funcname, _Py_Mangle(classname, funcname));
+        Py_DECREF(classname);
+        if (funcname == NULL) {
+            return NULL;
+        }
+    }
     return Py_BuildValue(
             "N(ON)", _PyEval_GetBuiltin(&_Py_ID(getattr)), funcself, funcname);
 }
diff --git a/Python/symtable.c b/Python/symtable.c
@@ -3183,6 +3183,27 @@ _Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name)
     return _Py_Mangle(privateobj, name);
 }
 
+int
+_Py_IsPrivateName(PyObject *ident)
+{
+    if (!PyUnicode_Check(ident)) {
+        return 0;
+    }
+    Py_ssize_t nlen = PyUnicode_GET_LENGTH(ident);
+    if (nlen < 3 ||
+        PyUnicode_READ_CHAR(ident, 0) != '_' ||
+        PyUnicode_READ_CHAR(ident, 1) != '_')
+    {
+        return 0;
+    }
+    if (PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&
+        PyUnicode_READ_CHAR(ident, nlen-2) == '_')
+    {
+        return 0; /* Don't mangle __whatever__ */
+    }
+    return 1;
+}
+
 PyObject *
 _Py_Mangle(PyObject *privateobj, PyObject *ident)
 {
