Fix: support CPython free-threaded builds (3.14t)

.github/workflows/tests.yml

@@ -9,6 +9,7 @@ on:
     - main
     tags:
     - '*'
+  workflow_dispatch:
 
 jobs:
   julia:
@@ -66,6 +67,43 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  julia-ft:
+    name: Julia (1, ubuntu-latest, python3.14t)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Julia 1
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+          arch: x64
+
+      - uses: julia-actions/cache@v2
+
+      - name: Set up micromamba (python3.14t)
+        id: micromamba
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-name: py314t
+          create-args: >-
+            -c conda-forge python=3.14.*=*cp314t*
+          cache-environment: true
+
+      - name: Build package
+        uses: julia-actions/julia-buildpkg@v1
+        env:
+          PYTHON: ${{ steps.micromamba.outputs.environment-path }}/bin/python
+
+      - name: Run tests
+        uses: julia-actions/julia-runtest@v1
+        env:
+          JULIA_DEBUG: PythonCall
+          JULIA_NUM_THREADS: '2'
+          PYTHON: ${{ steps.micromamba.outputs.environment-path }}/bin/python
+          JULIA_PYTHONCALL_EXE: ${{ steps.micromamba.outputs.environment-path }}/bin/python
+
   python:
     name: Python (${{ matrix.pyversion }}, ${{ matrix.os }}, ${{ matrix.juliaexe }})
     runs-on: ${{ matrix.os }}

src/C/consts.jl

@@ -124,6 +124,20 @@ end
     type::Ptr{Cvoid} = C_NULL # really is Ptr{PyObject} or Ptr{PyTypeObject} but Julia 1.3 and below get the layout incorrect when circular types are involved
 end
 
+@kwdef struct PyMutex
+    bits::Cuchar = 0
+end
+
+@kwdef struct PyObjectFT
+    tid::Csize_t = 0
+    flags::Cushort = 0
+    mutex::PyMutex = PyMutex()
+    gc_bits::Cuchar = 0
+    ref_local::Cuint = 0
+    ref_shared::Py_ssize_t = 0
+    type::Ptr{Cvoid} = C_NULL # really is Ptr{PyObject} or Ptr{PyTypeObject} but Julia 1.3 and below get the layout incorrect when circular types are involved
+end
+
 const PyPtr = Ptr{PyObject}
 const PyNULL = PyPtr(0)
 
@@ -139,6 +153,11 @@ Base.unsafe_convert(::Type{PyPtr}, o::PyObjectRef) = o.ptr
     size::Py_ssize_t = 0
 end
 
+@kwdef struct PyVarObjectFT
+    ob_base::PyObjectFT = PyObjectFT()
+    size::Py_ssize_t = 0
+end
+
 @kwdef struct PyMethodDef
     name::Cstring = C_NULL
     meth::Ptr{Cvoid} = C_NULL
@@ -249,6 +268,16 @@ end
     weakreflist::PyPtr = PyNULL
 end
 
+@kwdef struct PyMemoryViewObjectFT
+    ob_base::PyVarObjectFT = PyVarObjectFT()
+    mbuf::PyPtr = PyNULL
+    hash::Py_hash_t = 0
+    flags::Cint = 0
+    exports::Py_ssize_t = 0
+    view::Py_buffer = Py_buffer()
+    weakreflist::PyPtr = PyNULL
+end
+
 @kwdef struct PyTypeObject
     ob_base::PyVarObject = PyVarObject()
     name::Cstring = C_NULL
@@ -327,6 +356,11 @@ end
     value::T
 end
 
+@kwdef struct PySimpleObjectFT{T}
+    ob_base::PyObjectFT = PyObjectFT()
+    value::T
+end
+
 @kwdef struct PyArrayInterface
     two::Cint = 0
     nd::Cint = 0

src/C/context.jl

@@ -17,6 +17,7 @@ A handle to a loaded instance of libpython, its interpreter, function pointers,
     pyhome_w::Any = missing
     which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown
     version::Union{VersionNumber,Missing} = missing
+    is_free_threaded::Bool = false
 end
 
 const CTX = Context()
@@ -312,10 +313,11 @@ function init_context()
     v"3.10" ≤ CTX.version < v"4" || error(
         "Only Python 3.10+ is supported, this is Python $(CTX.version) at $(CTX.exe_path===missing ? "unknown location" : CTX.exe_path).",
     )
+    CTX.is_free_threaded = occursin("free-threading build", verstr)
 
     launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable
 
-    @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version
+    @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded
 
     return
 end

src/C/extras.jl

@@ -1,44 +1,97 @@
 asptr(x) = Base.unsafe_convert(PyPtr, x)
 
-Py_Type(x) = Base.GC.@preserve x PyPtr(UnsafePtr(asptr(x)).type[!])
+# Free-threaded CPython builds ("3.14t") currently have different C struct layouts,
+# but there is no stable ABI yet. To keep the code manageable, we centralize the
+# branching in a single macro that rewrites type names in the expression.
+const _FT_TYPE_REPLACEMENTS = Dict{Symbol,Symbol}(
+    :PyObject => :PyObjectFT,
+    :PyVarObject => :PyVarObjectFT,
+    :PyMemoryViewObject => :PyMemoryViewObjectFT,
+    :PySimpleObject => :PySimpleObjectFT,
+    # Used from JlWrap/C.jl via `C.@ft`.
+    :PyJuliaValueObject => :PyJuliaValueObjectFT,
+)
+
+_ft_replace(sym::Symbol) = get(_FT_TYPE_REPLACEMENTS, sym, sym)
+
+function _ft_transform(ex)
+    if ex isa Symbol
+        return _ft_replace(ex)
+    elseif ex isa QuoteNode
+        v = ex.value
+        return v isa Symbol ? QuoteNode(_ft_replace(v)) : ex
+    elseif ex isa Expr
+        # Handle dotted refs like `C.PyObject` (Expr(:., ...)).
+        if ex.head === :. && length(ex.args) == 2 && ex.args[2] isa QuoteNode && ex.args[2].value isa Symbol
+            return Expr(:., _ft_transform(ex.args[1]), QuoteNode(_ft_replace(ex.args[2].value)))
+        end
+        return Expr(ex.head, map(_ft_transform, ex.args)...)
+    else
+        return ex
+    end
+end
+
+"""
+    @ft expr
+
+Evaluate `expr`, but when `CTX.is_free_threaded` is true (CPython "free-threaded"
+builds), rewrite internal type names like `PyObject` → `PyObjectFT` inside the
+expression.
+
+This keeps free-threaded branching centralized, so we don't scatter `if
+CTX.is_free_threaded` throughout the code.
+"""
+macro ft(ex)
+    ex_ft = _ft_transform(ex)
+    ctx = GlobalRef(@__MODULE__, :CTX)
+    return esc(:(if $ctx.is_free_threaded
+        $ex_ft
+    else
+        $ex
+    end))
+end
+
+Py_Type(x) = Base.GC.@preserve x @ft PyPtr(UnsafePtr{PyObject}(asptr(x)).type[!])
 
 PyObject_Type(x) = Base.GC.@preserve x (t = Py_Type(asptr(x)); Py_IncRef(t); t)
 
 Py_TypeCheck(o, t) = Base.GC.@preserve o t PyType_IsSubtype(Py_Type(asptr(o)), asptr(t))
 Py_TypeCheckFast(o, f::Integer) = Base.GC.@preserve o PyType_IsSubtypeFast(Py_Type(asptr(o)), f)
 
 PyType_IsSubtypeFast(t, f::Integer) =
-    Base.GC.@preserve t Cint(!iszero(UnsafePtr{PyTypeObject}(asptr(t)).flags[] & f))
+    Base.GC.@preserve t Cint(!iszero(PyType_GetFlags(asptr(t)) & f))
 
-PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view)
+PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m @ft Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view)
 
 PyType_CheckBuffer(t) = Base.GC.@preserve t begin
-    p = UnsafePtr{PyTypeObject}(asptr(t)).as_buffer[]
-    return p != C_NULL && p.get[!] != C_NULL
+    getbuf = PyType_GetSlot(asptr(t), Py_bf_getbuffer)
+    return getbuf != C_NULL
 end
 
 PyObject_CheckBuffer(o) = Base.GC.@preserve o PyType_CheckBuffer(Py_Type(asptr(o)))
 
 PyObject_GetBuffer(_o, b, flags) = Base.GC.@preserve _o begin
     o = asptr(_o)
-    p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
-    if p == C_NULL || p.get[!] == C_NULL
-        PyErr_SetString(
-            POINTERS.PyExc_TypeError,
-            "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'",
-        )
+    getbuf = PyType_GetSlot(Py_Type(o), Py_bf_getbuffer)
+    if getbuf == C_NULL
+        msg = if CTX.is_free_threaded
+            "a bytes-like object is required"
+        else
+            "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'"
+        end
+        PyErr_SetString(POINTERS.PyExc_TypeError, msg)
         return Cint(-1)
     end
-    return ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
+    return ccall(getbuf, Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
 end
 
 PyBuffer_Release(_b) = begin
     b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b))
     o = b.obj[]
     o == C_NULL && return
-    p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
-    if (p != C_NULL && p.release[!] != C_NULL)
-        ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b)
+    releasebuf = PyType_GetSlot(Py_Type(o), Py_bf_releasebuffer)
+    if releasebuf != C_NULL
+        ccall(releasebuf, Cvoid, (PyPtr, Ptr{Py_buffer}), o, b)
     end
     b.obj[] = C_NULL
     Py_DecRef(o)
@@ -65,7 +118,7 @@ function PyOS_RunInputHook()
 end
 
 function PySimpleObject_GetValue(::Type{T}, o) where {T}
-    Base.GC.@preserve o UnsafePtr{PySimpleObject{T}}(asptr(o)).value[!]
+    Base.GC.@preserve o @ft UnsafePtr{PySimpleObject{T}}(asptr(o)).value[!]
 end
 
 # FAST REFCOUNTING

src/C/pointers.jl

@@ -77,6 +77,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     :PyType_Ready => (PyPtr,) => Cint,
     :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr,
     :PyType_FromSpec => (Ptr{Cvoid},) => PyPtr,
+    :PyType_GetFlags => (PyPtr,) => Culong,
+    :PyType_GetSlot => (PyPtr, Cint) => Ptr{Cvoid},
     # MAPPING
     :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint,
     :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint,

src/JlWrap/C.jl

@@ -12,6 +12,12 @@ using Serialization: serialize, deserialize
     weaklist::C.PyPtr = C_NULL
 end
 
+@kwdef struct PyJuliaValueObjectFT
+    ob_base::C.PyObjectFT = C.PyObjectFT()
+    value::Int = 0
+    weaklist::C.PyPtr = C_NULL
+end
+
 const PyJuliaBase_Type = Ref(C.PyNULL)
 
 # we store the actual julia values here
@@ -21,21 +27,24 @@ const PYJLVALUES = []
 const PYJLFREEVALUES = Int[]
 
 function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr)
-    o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0)
+    alloc = C.PyType_GetSlot(t, C.Py_tp_alloc)
+    alloc == C_NULL && return C.PyNULL
+    o = ccall(alloc, C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0)
     o == C.PyNULL && return C.PyNULL
-    UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL
-    UnsafePtr{PyJuliaValueObject}(o).value[] = 0
+    C.@ft UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL
+    C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = 0
     return o
 end
 
 function _pyjl_dealloc(o::C.PyPtr)
-    idx = UnsafePtr{PyJuliaValueObject}(o).value[]
+    idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[]
     if idx != 0
         PYJLVALUES[idx] = nothing
         push!(PYJLFREEVALUES, idx)
     end
-    UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o)
-    ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (C.PyPtr,), o)
+    (C.@ft UnsafePtr{PyJuliaValueObject}(o).weaklist[!]) == C.PyNULL || C.PyObject_ClearWeakRefs(o)
+    freeptr = C.PyType_GetSlot(C.Py_Type(o), C.Py_tp_free)
+    freeptr == C_NULL || ccall(freeptr, Cvoid, (C.PyPtr,), o)
     nothing
 end
 
@@ -319,7 +328,7 @@ function init_c()
         C.PyMemberDef(
             name = pointer(_pyjlbase_weaklistoffset_name),
             typ = C.Py_T_PYSSIZET,
-            offset = fieldoffset(PyJuliaValueObject, 3),
+            offset = (C.@ft fieldoffset(PyJuliaValueObject, 3)),
             flags = C.Py_READONLY,
         ),
         C.PyMemberDef(), # NULL terminator
@@ -341,7 +350,7 @@ function init_c()
     # Create PyType_Spec
     _pyjlbase_spec[] = C.PyType_Spec(
         name = pointer(_pyjlbase_name),
-        basicsize = sizeof(PyJuliaValueObject),
+        basicsize = (C.@ft sizeof(PyJuliaValueObject)),
         flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG,
         slots = pointer(_pyjlbase_slots),
     )
@@ -358,13 +367,13 @@ function __init__()
     init_c()
 end
 
-PyJuliaValue_IsNull(o) = Base.GC.@preserve o UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] == 0
+PyJuliaValue_IsNull(o) = Base.GC.@preserve o (C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]) == 0
 
-PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]]
+PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[(C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[])]
 
 PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin
     o = C.asptr(_o)
-    idx = UnsafePtr{PyJuliaValueObject}(o).value[]
+    idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[]
     if idx == 0
         if isempty(PYJLFREEVALUES)
             push!(PYJLVALUES, v)
@@ -373,7 +382,7 @@ PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin
             idx = pop!(PYJLFREEVALUES)
             PYJLVALUES[idx] = v
         end
-        UnsafePtr{PyJuliaValueObject}(o).value[] = idx
+        C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = idx
     else
         PYJLVALUES[idx] = v
     end