feat: JSON handling with JSON.jl v1.0 and StructTypes

.github/workflows/ci.yml

@@ -1,7 +1,9 @@
 name: CI
 on:
   push:
-    branches: [main]
+    branches:
+      - main
+      - 'feat/**'
     tags: ["*"]
   pull_request:
 jobs:

Project.toml

@@ -3,8 +3,8 @@ uuid = "d5e62ea6-ddf3-4d43-8e4c-ad5e6c8bfd7d"
 keywords = ["Swagger", "OpenAPI", "REST"]
 license = "MIT"
 desc = "OpenAPI server and client helper for Julia"
+version = "0.2.1"
 authors = ["JuliaHub Inc."]
-version = "0.2.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -15,17 +15,19 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibCURL = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
 MIMEs = "6c6e2e6c-3030-632d-7369-2d6c69616d65"
 MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
+StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 
 [compat]
 Downloads = "1"
 HTTP = "1"
-JSON = "0.20, 0.21"
+JSON = "1.1.0"
 LibCURL = "0.6"
 MIMEs = "0.1, 1"
 MbedTLS = "0.6.8, 0.7, 1"
+StructTypes = "1.11.0"
 TimeZones = "1"
 URIs = "1.3"
 julia = "1.6"

src/OpenAPI.jl

@@ -3,7 +3,7 @@ module OpenAPI
 using HTTP, JSON, URIs, Dates, TimeZones, Base64
 using Downloads
 using p7zip_jll
-
+using StructTypes
 import Base: getindex, keys, length, iterate, hasproperty
 import JSON: lower
 

src/client.jl

@@ -9,6 +9,7 @@ using TimeZones
 using LibCURL
 using HTTP
 using MIMEs
+using StructTypes
 
 import Base: convert, show, summary, getproperty, setproperty!, iterate
 import ..OpenAPI: APIModel, UnionAPIModel, OneOfAPIModel, AnyOfAPIModel, APIClientImpl, OpenAPIException, InvocationException, to_json, from_json, validate_property, property_type
@@ -424,30 +425,21 @@ function header(resp::Downloads.Response, name::AbstractString, defaultval::Abst
     return defaultval
 end
 
+
 response(::Type{Nothing}, resp::Downloads.Response, body) = nothing::Nothing
-response(::Type{T}, resp::Downloads.Response, body) where {T <: Real} = response(T, body)::T
-response(::Type{T}, resp::Downloads.Response, body) where {T <: String} = response(T, body)::T
+response(::Type{T}, resp::Downloads.Response, body) where {T <: Real} = Base.parse(T, String(body))::T
+response(::Type{T}, resp::Downloads.Response, body) where {T <: String} = String(body)::T
 function response(::Type{T}, resp::Downloads.Response, body) where {T}
     ctype = header(resp, "Content-Type", "application/json")
-    response(T, is_json_mime(ctype), body)::T
-end
-response(::Type{T}, ::Nothing, body) where {T} = response(T, true, body)
-function response(::Type{T}, is_json::Bool, body) where {T}
-    (length(body) == 0) && return T()
-    response(T, is_json ? JSON.parse(String(body)) : body)::T
+    if is_json_mime(ctype)
+        (length(body) == 0) && return T() # Handle empty body for model types
+        # Use JSON.read for direct deserialization
+        return JSON.parse(body, T)::T
+    else
+        # Fallback for non-JSON content
+        return convert(T, body)
+    end
 end
-response(::Type{String}, data::Vector{UInt8}) = String(data)
-response(::Type{T}, data::Vector{UInt8}) where {T<:Real} = parse(T, String(data))
-response(::Type{T}, data::T) where {T} = data
-
-response(::Type{ZonedDateTime}, data) = str2zoneddatetime(data)
-response(::Type{DateTime}, data) = str2datetime(data)
-response(::Type{Date}, data) = str2date(data)
-
-response(::Type{T}, data) where {T} = convert(T, data)
-response(::Type{T}, data::Dict{String,Any}) where {T} = from_json(T, data)::T
-response(::Type{T}, data::Dict{String,Any}) where {T<:Dict} = convert(T, data)
-response(::Type{Vector{T}}, data::Vector{V}) where {T,V} = T[response(T, v) for v in data]
 
 struct LineChunkReader <: AbstractChunkReader
     buffered_input::Base.BufferStream
@@ -471,23 +463,72 @@ struct JSONChunkReader <: AbstractChunkReader
     buffered_input::Base.BufferStream
 end
 
-function Base.iterate(iter::JSONChunkReader, _state=nothing)
-    if eof(iter.buffered_input)
-        return nothing
-    else
-        # read all whitespaces
-        while !eof(iter.buffered_input)
-            byte = peek(iter.buffered_input, UInt8)
-            if isspace(Char(byte))
-                read(iter.buffered_input, UInt8)
+function Base.iterate(iter::JSONChunkReader, buffer::IOBuffer=IOBuffer())
+    # This function now uses an IOBuffer as its state to hold
+    # data that has been read from the stream but not yet parsed.
+
+    while true
+        # First, try to parse the data we already have in our buffer.
+        seekstart(buffer)
+        data = read(buffer)
+
+        if !isempty(data)
+            # Skip leading whitespace in our buffer
+            # This is important if the previous chunk had trailing spaces.
+            start_pos = 1
+            while start_pos <= length(data) && isspace(Char(data[start_pos]))
+                start_pos += 1
+            end
+
+            if start_pos > length(data)
+                # Buffer only contained whitespace, clear it and read more.
+                take!(buffer)
             else
-                break
+                local lazy_val
+                local end_pos
+                try
+                    # Check if the buffer contains at least one complete JSON object.
+                    # We parse from the first non-whitespace character.
+                    lazy_val = JSON.lazy(view(data, start_pos:length(data)))
+
+                    # If successful, find where this object ends.
+                    end_pos = JSON.skip(lazy_val)
+
+                    # The bytes for the complete JSON chunk.
+                    json_chunk = data[start_pos:(start_pos + end_pos - 2)]
+
+                    # The rest of the data is carried over to the next iteration.
+                    remaining_data = data[(start_pos + end_pos - 1):end]
+                    next_buffer = IOBuffer(remaining_data)
+
+                    return (json_chunk, next_buffer)
+                catch e
+                    # An UnexpectedEOF error means our buffer contains an incomplete object.
+                    # We'll loop again to read more data from the main stream.
+                    # Any other error indicates a real JSON syntax problem.
+                    if !(e isa ArgumentError && occursin("UnexpectedEOF", e.msg))
+                        rethrow()
+                    end
+                end
             end
         end
-        eof(iter.buffered_input) && return nothing
-        valid_json = JSON.parse(iter.buffered_input)
-        bytes = convert(Vector{UInt8}, codeunits(JSON.json(valid_json)))
-        return (bytes, iter)
+
+        # If we are here, our buffer is empty or has an incomplete object.
+        # We need to read more data from the source input stream.
+        if eof(iter.buffered_input)
+            # The source stream is finished. If the buffer still contains non-whitespace
+            # data, it means the stream ended with an incomplete JSON object.
+            if bytesavailable(buffer) > 0
+                seekstart(buffer)
+                if !eof(skipchars(isspace, buffer))
+                    error("Incomplete JSON data at end of stream.")
+                end
+            end
+            return nothing # Correctly end the iteration.
+        end
+
+        # Block and read new data from the input, then loop to retry parsing.
+        write(buffer, readavailable(iter.buffered_input))
     end
 end
 

src/json.jl

@@ -1,32 +1,24 @@
-# JSONWrapper for OpenAPI models handles
-# - null fields
-# - field names that are Julia keywords
-struct JSONWrapper{T<:APIModel} <: AbstractDict{Symbol, Any}
-    wrapped::T
-    flds::Tuple
-end
+# Declare how StructTypes should handle APIModels
+StructTypes.StructType(::Type{<:APIModel}) = StructTypes.Struct()
 
-JSONWrapper(o::T) where {T<:APIModel} = JSONWrapper(o, filter(n->hasproperty(o,n) && (getproperty(o,n) !== nothing), propertynames(o)))
+# This single line replaces the entire JSONWrapper implementation.
+# It tells JSON.jl to automatically omit fields whose values are `nothing`.
+StructTypes.omitempties(::Type{<:APIModel}) = true
 
-getindex(w::JSONWrapper, s::Symbol) = getproperty(w.wrapped, s)
-keys(w::JSONWrapper) = w.flds
-length(w::JSONWrapper) = length(w.flds)
+# This hook tells JSON.read to use our custom `from_json` logic
+# for constructing APIModel types. This preserves our handling of dates,
+# discriminated unions, and other special cases.
+StructTypes.construct(::Type{T}, dict::Dict) where {T <: APIModel} = from_json(T, dict)
 
-function iterate(w::JSONWrapper, state...)
-    result = iterate(w.flds, state...)
-    if result === nothing
-        return result
-    else
-        name,nextstate = result
-        val = getproperty(w.wrapped, name)
-        return (name=>val, nextstate)
-    end
-end
 
-lower(o::T) where {T<:APIModel} = JSONWrapper(o)
+# The `lower` method for UnionAPIModel is still useful because it allows us to
+# serialize the inner .value of a oneOf/anyOf type, not the wrapper itself.
+# JSON.jl v1.0 still respects `lower`.
+
 function lower(o::T) where {T<:UnionAPIModel}
     if typeof(o.value) <: APIModel
-        return JSONWrapper(o.value)
+        # Use JSON.lower on the wrapped value to apply its own rules
+        return JSON.lower(o.value)
     elseif typeof(o.value) <: Union{String,Real}
         return o.value
     else
@@ -59,10 +51,20 @@ end
 to_json(o) = JSON.json(o)
 
 from_json(::Type{Union{Nothing,T}}, json::Dict{String,Any}; stylectx=nothing) where {T} = from_json(T, json; stylectx)
+from_json(::Type{Union{Nothing,T}}, json::JSON.Object{String, Any}; stylectx=nothing) where {T} = from_json(T, json; stylectx)
+
 from_json(::Type{T}, json::Dict{String,Any}; stylectx=nothing) where {T} = from_json(T(), json; stylectx)
+from_json(::Type{T}, json::JSON.Object{String, Any}; stylectx=nothing) where {T} = from_json(T(), json; stylectx)
+
 from_json(::Type{T}, json::Dict{String,Any}; stylectx=nothing) where {T <: Dict} = convert(T, json)
+from_json(::Type{T}, json::JSON.Object{String, Any}; stylectx=nothing) where {T <: Dict} = convert(T, json)
+
 from_json(::Type{T}, j::Dict{String,Any}; stylectx=nothing) where {T <: String} = to_json(j)
+from_json(::Type{T}, j::JSON.Object{String, Any}; stylectx=nothing) where {T <: String} = to_json(j)
+
 from_json(::Type{Any}, j::Dict{String,Any}; stylectx=nothing) = j
+from_json(::Type{Any}, j::JSON.Object{String, Any}; stylectx=nothing) = j
+
 from_json(::Type{Vector{T}}, j::Vector{Any}; stylectx=nothing) where {T} = j
 
 function from_json(::Type{Vector{T}}, json::Dict{String, Any}; stylectx=nothing) where {T}
@@ -78,10 +80,27 @@ function from_json(::Type{Vector{T}}, json::Dict{String, Any}; stylectx=nothing)
     end
 end
 
+function from_json(::Type{Vector{T}}, json::JSON.Object{String, Any}; stylectx=nothing) where {T}
+    if !isnothing(stylectx) && is_deep_explode(stylectx)
+        cvt = deep_object_to_array(json)
+        if isa(cvt, Vector)
+            return from_json(Vector{T}, cvt; stylectx)
+        else
+            return from_json(T, json; stylectx)
+        end
+    else
+        return from_json(T, json; stylectx)
+    end
+end
+
 function from_json(o::T, json::Dict{String,Any};stylectx=nothing) where {T <: UnionAPIModel}
     return from_json(o, :value, json;stylectx)
 end
 
+function from_json(o::T, json::JSON.Object{String, Any};stylectx=nothing) where {T <: UnionAPIModel}
+    return from_json(o, :value, json;stylectx)
+end
+
 from_json(::Type{T}, val::Union{String,Real};stylectx=nothing) where {T <: UnionAPIModel} = T(val)
 function from_json(o::T, val::Union{String,Real};stylectx=nothing) where {T <: UnionAPIModel}
     o.value = val
@@ -96,13 +115,28 @@ function from_json(o::T, json::Dict{String,Any};stylectx=nothing) where {T <: AP
     return o
 end
 
+function from_json(o::T, json::JSON.Object{String, Any};stylectx=nothing) where {T <: APIModel}
+    jsonkeys = [Symbol(k) for k in collect(keys(json))]
+    for name in intersect(propertynames(o), jsonkeys)
+        from_json(o, name, json[String(name)];stylectx)
+    end
+    return o
+end
+
 function from_json(o::T, name::Symbol, json::Dict{String,Any};stylectx=nothing) where {T <: APIModel}
     ftype = (T <: UnionAPIModel) ? property_type(T, name, json) : property_type(T, name)
     fval = from_json(ftype, json; stylectx)
     setfield!(o, name, convert(ftype, fval))
     return o
 end
 
+function from_json(o::T, name::Symbol, json::JSON.Object{String, Any};stylectx=nothing) where {T <: APIModel}
+    ftype = (T <: UnionAPIModel) ? property_type(T, name, json) : property_type(T, name)
+    fval = from_json(ftype, json; stylectx)
+    setfield!(o, name, convert(ftype, fval))
+    return o
+end
+
 function from_json(o::T, name::Symbol, v; stylectx=nothing) where {T <: APIModel}
     ftype = (T <: UnionAPIModel) ? property_type(T, name, Dict{String,Any}()) : property_type(T, name)
     atype = isa(ftype, Union) ? ((ftype.a === Nothing) ? ftype.b : ftype.a) : ftype

src/server.jl

@@ -2,6 +2,7 @@ module Servers
 
 using JSON
 using HTTP
+using StructTypes
 
 import ..OpenAPI: APIModel, ValidationException, from_json, to_json, deep_object_to_array, StyleCtx, is_deep_explode
 
@@ -82,28 +83,16 @@ function to_param_type(::Type{T}, strval::String; stylectx=nothing) where {T <:
     parse(T, strval)
 end
 
-to_param_type(::Type{T}, val::T; stylectx=nothing) where {T} = val
-to_param_type(::Type{T}, ::Nothing; stylectx=nothing) where {T} = nothing
-to_param_type(::Type{String}, val::Vector{UInt8}; stylectx=nothing) = String(copy(val))
-to_param_type(::Type{Vector{UInt8}}, val::String; stylectx=nothing) = convert(Vector{UInt8}, copy(codeunits(val)))
-to_param_type(::Type{Vector{T}}, val::Vector{T}, _collection_format::Union{String,Nothing}; stylectx=nothing) where {T} = val
-to_param_type(::Type{Vector{T}}, json::Vector{Any}; stylectx=nothing) where {T} = [to_param_type(T, x; stylectx) for x in json]
-
-function to_param_type(::Type{Vector{T}}, json::Dict{String, Any}; stylectx=nothing) where {T}
-    if !isnothing(stylectx) && is_deep_explode(stylectx)
-        cvt = deep_object_to_array(json)
-        if isa(cvt, Vector)
-            return to_param_type(Vector{T}, cvt; stylectx)
-        end
-    end
-    error("Unable to convert $json to $(Vector{T})")
+# UPDATED: Use JSON.read for direct deserialization from a string.
+function to_param_type(::Type{T}, strval::String; stylectx=nothing) where {T <: APIModel}
+    return JSON.read(strval, T)
 end
 
-function to_param_type(::Type{T}, strval::String; stylectx=nothing) where {T <: APIModel}
-    from_json(T, JSON.parse(strval); stylectx)
+function to_param_type(::Type{T}, json::Dict{String,Any}; stylectx=nothing) where {T <: APIModel}
+    from_json(T, json; stylectx)
 end
 
-function  to_param_type(::Type{T}, json::Dict{String,Any}; stylectx=nothing) where {T <: APIModel}
+function to_param_type(::Type{T}, json::JSON.Object{String, Any}; stylectx=nothing) where {T <: APIModel}
     from_json(T, json; stylectx)
 end
 
@@ -112,9 +101,9 @@ function to_param_type(::Type{Vector{T}}, strval::String, delim::String; stylect
     return map(x->to_param_type(T, x; stylectx), elems)
 end
 
+#  Use JSON.read for direct deserialization from a string.
 function to_param_type(::Type{Vector{T}}, strval::String; stylectx=nothing) where {T}
-    elems = JSON.parse(strval)
-    return map(x->to_param_type(T, x; stylectx), elems)
+    return JSON.read(strval, Vector{T})
 end
 
 function to_param(T, source::Dict, name::String; required::Bool=false, collection_format::Union{String,Nothing}=",", multipart::Bool=false, isfile::Bool=false, style::String="form", is_explode::Bool=true, location=:query)