diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cc021ac2..0073b4f8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,7 @@ on: - main tags: - '*' + workflow_dispatch: jobs: julia: @@ -66,6 +67,43 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + julia-ft: + name: Julia (1, ubuntu-latest, python3.14t) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Julia 1 + uses: julia-actions/setup-julia@v2 + with: + version: '1' + arch: x64 + + - uses: julia-actions/cache@v2 + + - name: Set up micromamba (python3.14t) + id: micromamba + uses: mamba-org/setup-micromamba@v2 + with: + environment-name: py314t + create-args: >- + -c conda-forge python=3.14.*=*cp314t* + cache-environment: true + + - name: Build package + uses: julia-actions/julia-buildpkg@v1 + env: + PYTHON: ${{ steps.micromamba.outputs.environment-path }}/bin/python + + - name: Run tests + uses: julia-actions/julia-runtest@v1 + env: + JULIA_DEBUG: PythonCall + JULIA_NUM_THREADS: '2' + PYTHON: ${{ steps.micromamba.outputs.environment-path }}/bin/python + JULIA_PYTHONCALL_EXE: ${{ steps.micromamba.outputs.environment-path }}/bin/python + python: name: Python (${{ matrix.pyversion }}, ${{ matrix.os }}, ${{ matrix.juliaexe }}) runs-on: ${{ matrix.os }} diff --git a/src/C/consts.jl b/src/C/consts.jl index 4560e928..7d174968 100644 --- a/src/C/consts.jl +++ b/src/C/consts.jl @@ -124,6 +124,20 @@ end type::Ptr{Cvoid} = C_NULL # really is Ptr{PyObject} or Ptr{PyTypeObject} but Julia 1.3 and below get the layout incorrect when circular types are involved end +@kwdef struct PyMutex + bits::Cuchar = 0 +end + +@kwdef struct PyObjectFT + tid::Csize_t = 0 + flags::Cushort = 0 + mutex::PyMutex = PyMutex() + gc_bits::Cuchar = 0 + ref_local::Cuint = 0 + ref_shared::Py_ssize_t = 0 + type::Ptr{Cvoid} = C_NULL # really is Ptr{PyObject} or Ptr{PyTypeObject} but Julia 1.3 and below get the layout incorrect when circular types are involved +end + const PyPtr = Ptr{PyObject} const PyNULL = PyPtr(0) @@ -139,6 +153,11 @@ Base.unsafe_convert(::Type{PyPtr}, o::PyObjectRef) = o.ptr size::Py_ssize_t = 0 end +@kwdef struct PyVarObjectFT + ob_base::PyObjectFT = PyObjectFT() + size::Py_ssize_t = 0 +end + @kwdef struct PyMethodDef name::Cstring = C_NULL meth::Ptr{Cvoid} = C_NULL @@ -249,6 +268,16 @@ end weakreflist::PyPtr = PyNULL end +@kwdef struct PyMemoryViewObjectFT + ob_base::PyVarObjectFT = PyVarObjectFT() + mbuf::PyPtr = PyNULL + hash::Py_hash_t = 0 + flags::Cint = 0 + exports::Py_ssize_t = 0 + view::Py_buffer = Py_buffer() + weakreflist::PyPtr = PyNULL +end + @kwdef struct PyTypeObject ob_base::PyVarObject = PyVarObject() name::Cstring = C_NULL @@ -327,6 +356,11 @@ end value::T end +@kwdef struct PySimpleObjectFT{T} + ob_base::PyObjectFT = PyObjectFT() + value::T +end + @kwdef struct PyArrayInterface two::Cint = 0 nd::Cint = 0 diff --git a/src/C/context.jl b/src/C/context.jl index 9d1f5d30..4d83cf2a 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -17,6 +17,7 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, pyhome_w::Any = missing which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown version::Union{VersionNumber,Missing} = missing + is_free_threaded::Bool = false end const CTX = Context() @@ -312,10 +313,11 @@ function init_context() v"3.10" ≤ CTX.version < v"4" || error( "Only Python 3.10+ is supported, this is Python $(CTX.version) at $(CTX.exe_path===missing ? "unknown location" : CTX.exe_path).", ) + CTX.is_free_threaded = occursin("free-threading build", verstr) launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable - @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version + @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version CTX.is_free_threaded return end diff --git a/src/C/extras.jl b/src/C/extras.jl index cf46b6b8..086fbbcf 100644 --- a/src/C/extras.jl +++ b/src/C/extras.jl @@ -1,6 +1,57 @@ asptr(x) = Base.unsafe_convert(PyPtr, x) -Py_Type(x) = Base.GC.@preserve x PyPtr(UnsafePtr(asptr(x)).type[!]) +# Free-threaded CPython builds ("3.14t") currently have different C struct layouts, +# but there is no stable ABI yet. To keep the code manageable, we centralize the +# branching in a single macro that rewrites type names in the expression. +const _FT_TYPE_REPLACEMENTS = Dict{Symbol,Symbol}( + :PyObject => :PyObjectFT, + :PyVarObject => :PyVarObjectFT, + :PyMemoryViewObject => :PyMemoryViewObjectFT, + :PySimpleObject => :PySimpleObjectFT, + # Used from JlWrap/C.jl via `C.@ft`. + :PyJuliaValueObject => :PyJuliaValueObjectFT, +) + +_ft_replace(sym::Symbol) = get(_FT_TYPE_REPLACEMENTS, sym, sym) + +function _ft_transform(ex) + if ex isa Symbol + return _ft_replace(ex) + elseif ex isa QuoteNode + v = ex.value + return v isa Symbol ? QuoteNode(_ft_replace(v)) : ex + elseif ex isa Expr + # Handle dotted refs like `C.PyObject` (Expr(:., ...)). + if ex.head === :. && length(ex.args) == 2 && ex.args[2] isa QuoteNode && ex.args[2].value isa Symbol + return Expr(:., _ft_transform(ex.args[1]), QuoteNode(_ft_replace(ex.args[2].value))) + end + return Expr(ex.head, map(_ft_transform, ex.args)...) + else + return ex + end +end + +""" + @ft expr + +Evaluate `expr`, but when `CTX.is_free_threaded` is true (CPython "free-threaded" +builds), rewrite internal type names like `PyObject` → `PyObjectFT` inside the +expression. + +This keeps free-threaded branching centralized, so we don't scatter `if +CTX.is_free_threaded` throughout the code. +""" +macro ft(ex) + ex_ft = _ft_transform(ex) + ctx = GlobalRef(@__MODULE__, :CTX) + return esc(:(if $ctx.is_free_threaded + $ex_ft + else + $ex + end)) +end + +Py_Type(x) = Base.GC.@preserve x @ft PyPtr(UnsafePtr{PyObject}(asptr(x)).type[!]) PyObject_Type(x) = Base.GC.@preserve x (t = Py_Type(asptr(x)); Py_IncRef(t); t) @@ -8,37 +59,39 @@ Py_TypeCheck(o, t) = Base.GC.@preserve o t PyType_IsSubtype(Py_Type(asptr(o)), a Py_TypeCheckFast(o, f::Integer) = Base.GC.@preserve o PyType_IsSubtypeFast(Py_Type(asptr(o)), f) PyType_IsSubtypeFast(t, f::Integer) = - Base.GC.@preserve t Cint(!iszero(UnsafePtr{PyTypeObject}(asptr(t)).flags[] & f)) + Base.GC.@preserve t Cint(!iszero(PyType_GetFlags(asptr(t)) & f)) -PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view) +PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m @ft Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view) PyType_CheckBuffer(t) = Base.GC.@preserve t begin - p = UnsafePtr{PyTypeObject}(asptr(t)).as_buffer[] - return p != C_NULL && p.get[!] != C_NULL + getbuf = PyType_GetSlot(asptr(t), Py_bf_getbuffer) + return getbuf != C_NULL end PyObject_CheckBuffer(o) = Base.GC.@preserve o PyType_CheckBuffer(Py_Type(asptr(o))) PyObject_GetBuffer(_o, b, flags) = Base.GC.@preserve _o begin o = asptr(_o) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if p == C_NULL || p.get[!] == C_NULL - PyErr_SetString( - POINTERS.PyExc_TypeError, - "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'", - ) + getbuf = PyType_GetSlot(Py_Type(o), Py_bf_getbuffer) + if getbuf == C_NULL + msg = if CTX.is_free_threaded + "a bytes-like object is required" + else + "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'" + end + PyErr_SetString(POINTERS.PyExc_TypeError, msg) return Cint(-1) end - return ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) + return ccall(getbuf, Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) end PyBuffer_Release(_b) = begin b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) o = b.obj[] o == C_NULL && return - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if (p != C_NULL && p.release[!] != C_NULL) - ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) + releasebuf = PyType_GetSlot(Py_Type(o), Py_bf_releasebuffer) + if releasebuf != C_NULL + ccall(releasebuf, Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) end b.obj[] = C_NULL Py_DecRef(o) @@ -65,7 +118,7 @@ function PyOS_RunInputHook() end function PySimpleObject_GetValue(::Type{T}, o) where {T} - Base.GC.@preserve o UnsafePtr{PySimpleObject{T}}(asptr(o)).value[!] + Base.GC.@preserve o @ft UnsafePtr{PySimpleObject{T}}(asptr(o)).value[!] end # FAST REFCOUNTING diff --git a/src/C/pointers.jl b/src/C/pointers.jl index d7441960..9644329f 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -77,6 +77,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( :PyType_Ready => (PyPtr,) => Cint, :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr, :PyType_FromSpec => (Ptr{Cvoid},) => PyPtr, + :PyType_GetFlags => (PyPtr,) => Culong, + :PyType_GetSlot => (PyPtr, Cint) => Ptr{Cvoid}, # MAPPING :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint, :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint, diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 50c17987..19ff2904 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -12,6 +12,12 @@ using Serialization: serialize, deserialize weaklist::C.PyPtr = C_NULL end +@kwdef struct PyJuliaValueObjectFT + ob_base::C.PyObjectFT = C.PyObjectFT() + value::Int = 0 + weaklist::C.PyPtr = C_NULL +end + const PyJuliaBase_Type = Ref(C.PyNULL) # we store the actual julia values here @@ -21,21 +27,24 @@ const PYJLVALUES = [] const PYJLFREEVALUES = Int[] function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr) - o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) + alloc = C.PyType_GetSlot(t, C.Py_tp_alloc) + alloc == C_NULL && return C.PyNULL + o = ccall(alloc, C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0) o == C.PyNULL && return C.PyNULL - UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL - UnsafePtr{PyJuliaValueObject}(o).value[] = 0 + C.@ft UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL + C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = 0 return o end function _pyjl_dealloc(o::C.PyPtr) - idx = UnsafePtr{PyJuliaValueObject}(o).value[] + idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] if idx != 0 PYJLVALUES[idx] = nothing push!(PYJLFREEVALUES, idx) end - UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o) - ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (C.PyPtr,), o) + (C.@ft UnsafePtr{PyJuliaValueObject}(o).weaklist[!]) == C.PyNULL || C.PyObject_ClearWeakRefs(o) + freeptr = C.PyType_GetSlot(C.Py_Type(o), C.Py_tp_free) + freeptr == C_NULL || ccall(freeptr, Cvoid, (C.PyPtr,), o) nothing end @@ -319,7 +328,7 @@ function init_c() C.PyMemberDef( name = pointer(_pyjlbase_weaklistoffset_name), typ = C.Py_T_PYSSIZET, - offset = fieldoffset(PyJuliaValueObject, 3), + offset = (C.@ft fieldoffset(PyJuliaValueObject, 3)), flags = C.Py_READONLY, ), C.PyMemberDef(), # NULL terminator @@ -341,7 +350,7 @@ function init_c() # Create PyType_Spec _pyjlbase_spec[] = C.PyType_Spec( name = pointer(_pyjlbase_name), - basicsize = sizeof(PyJuliaValueObject), + basicsize = (C.@ft sizeof(PyJuliaValueObject)), flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG, slots = pointer(_pyjlbase_slots), ) @@ -358,13 +367,13 @@ function __init__() init_c() end -PyJuliaValue_IsNull(o) = Base.GC.@preserve o UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[] == 0 +PyJuliaValue_IsNull(o) = Base.GC.@preserve o (C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]) == 0 -PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[]] +PyJuliaValue_GetValue(o) = Base.GC.@preserve o PYJLVALUES[(C.@ft UnsafePtr{PyJuliaValueObject}(C.asptr(o)).value[])] PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin o = C.asptr(_o) - idx = UnsafePtr{PyJuliaValueObject}(o).value[] + idx = C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] if idx == 0 if isempty(PYJLFREEVALUES) push!(PYJLVALUES, v) @@ -373,7 +382,7 @@ PyJuliaValue_SetValue(_o, @nospecialize(v)) = Base.GC.@preserve _o begin idx = pop!(PYJLFREEVALUES) PYJLVALUES[idx] = v end - UnsafePtr{PyJuliaValueObject}(o).value[] = idx + C.@ft UnsafePtr{PyJuliaValueObject}(o).value[] = idx else PYJLVALUES[idx] = v end