commit 2f40f4b6d4b6bf56459e910fdd52f3ff10b2be66
parent a4c039a806b2aefc4d5f0229d018f10122580f6a
Author: Erik Loualiche <[email protected]>
Date: Fri, 6 Feb 2026 10:56:11 -0600
Implement FFI binary protocol for native Julia types
Add nickel_eval_native() that parses Nickel directly to Julia types:
- Int64 for whole numbers
- Float64 for decimals
- Bool, String, Nothing for primitives
- Vector{Any} for arrays
- Dict{String, Any} for records
Binary protocol preserves Nickel type semantics without JSON round-trip.
41 new FFI tests, 94 total tests passing.
Co-Authored-By: Claude Opus 4.5 <[email protected]>
Diffstat:
4 files changed, 242 insertions(+), 26 deletions(-)
diff --git a/src/NickelEval.jl b/src/NickelEval.jl
@@ -4,7 +4,7 @@ using JSON
export nickel_eval, nickel_eval_file, nickel_export, nickel_read, @ncl_str, NickelError
export nickel_to_json, nickel_to_toml, nickel_to_yaml
-export check_ffi_available, nickel_eval_ffi
+export check_ffi_available, nickel_eval_ffi, nickel_eval_native
export find_nickel_executable
# Custom exception for Nickel errors
diff --git a/src/ffi.jl b/src/ffi.jl
@@ -3,10 +3,9 @@
# Native FFI bindings to a Rust wrapper around Nickel for high-performance evaluation
# without subprocess overhead.
#
-# API:
-# - nickel_eval_ffi(code::String) -> Any
-# - Direct ccall to libnickel_jl
-# - Memory management via nickel_free_string
+# Two modes:
+# - nickel_eval_ffi: Uses JSON serialization (supports typed parsing)
+# - nickel_eval_native: Uses binary protocol (preserves Nickel types directly)
#
# Benefits over subprocess:
# - No process spawn overhead
@@ -28,6 +27,21 @@ const LIB_PATH = joinpath(@__DIR__, "..", "deps", LIB_NAME)
# Check if FFI library is available
const FFI_AVAILABLE = isfile(LIB_PATH)
+# Binary protocol type tags (must match Rust definitions)
+const TYPE_NULL = 0x00
+const TYPE_BOOL = 0x01
+const TYPE_INT = 0x02
+const TYPE_FLOAT = 0x03
+const TYPE_STRING = 0x04
+const TYPE_ARRAY = 0x05
+const TYPE_RECORD = 0x06
+
+# C struct for native buffer (must match Rust NativeBuffer)
+struct NativeBuffer
+ data::Ptr{UInt8}
+ len::Csize_t
+end
+
"""
check_ffi_available() -> Bool
@@ -39,11 +53,11 @@ function check_ffi_available()
end
"""
- nickel_eval_ffi(code::String) -> JSON.Object
+ nickel_eval_ffi(code::String) -> Any
nickel_eval_ffi(code::String, ::Type{T}) -> T
-Evaluate Nickel code using native FFI bindings (faster than subprocess).
-Returns the parsed JSON result, optionally typed.
+Evaluate Nickel code using native FFI bindings via JSON serialization.
+Returns the parsed result, optionally typed.
Throws `NickelError` if FFI is not available or if evaluation fails.
@@ -71,30 +85,128 @@ function nickel_eval_ffi(code::String, ::Type{T}) where T
end
function _eval_ffi_to_json(code::String)
- if !FFI_AVAILABLE
- error("FFI not available. Build the Rust library with: cd rust/nickel-jl && cargo build --release && cp target/release/libnickel_jl.dylib ../../deps/")
- end
+ _check_ffi_available()
- # Call the Rust function
result_ptr = ccall((:nickel_eval_string, LIB_PATH),
Ptr{Cchar}, (Cstring,), code)
if result_ptr == C_NULL
- # Get error message
- error_ptr = ccall((:nickel_get_error, LIB_PATH), Ptr{Cchar}, ())
- if error_ptr != C_NULL
- error_msg = unsafe_string(error_ptr)
- throw(NickelError(error_msg))
- else
- throw(NickelError("Nickel evaluation failed with unknown error"))
- end
+ _throw_ffi_error()
end
- # Convert result to Julia string
result_json = unsafe_string(result_ptr)
-
- # Free the allocated memory
ccall((:nickel_free_string, LIB_PATH), Cvoid, (Ptr{Cchar},), result_ptr)
return result_json
end
+
+"""
+ nickel_eval_native(code::String) -> Any
+
+Evaluate Nickel code using native FFI with binary protocol.
+Returns Julia native types directly from Nickel's type system:
+
+- Nickel `Number` (integer) → `Int64`
+- Nickel `Number` (decimal) → `Float64`
+- Nickel `String` → `String`
+- Nickel `Bool` → `Bool`
+- Nickel `null` → `nothing`
+- Nickel `Array` → `Vector{Any}`
+- Nickel `Record` → `Dict{String, Any}`
+
+This preserves type information that would be lost through JSON serialization.
+
+# Examples
+```julia
+julia> nickel_eval_native("42")
+42
+
+julia> typeof(nickel_eval_native("42"))
+Int64
+
+julia> typeof(nickel_eval_native("42.0"))
+Float64
+
+julia> nickel_eval_native("{ name = \"test\", count = 5 }")
+Dict{String, Any}("name" => "test", "count" => 5)
+```
+"""
+function nickel_eval_native(code::String)
+ _check_ffi_available()
+
+ buffer = ccall((:nickel_eval_native, LIB_PATH),
+ NativeBuffer, (Cstring,), code)
+
+ if buffer.data == C_NULL
+ _throw_ffi_error()
+ end
+
+ # Copy data before freeing (Rust owns the memory)
+ data = Vector{UInt8}(undef, buffer.len)
+ unsafe_copyto!(pointer(data), buffer.data, buffer.len)
+
+ # Free the Rust buffer
+ ccall((:nickel_free_buffer, LIB_PATH), Cvoid, (NativeBuffer,), buffer)
+
+ # Decode the binary protocol
+ return _decode_native(data)
+end
+
+"""
+ _decode_native(data::Vector{UInt8}) -> Any
+
+Decode binary-encoded Nickel value to Julia native types.
+"""
+function _decode_native(data::Vector{UInt8})
+ io = IOBuffer(data)
+ return _decode_value(io)
+end
+
+function _decode_value(io::IOBuffer)
+ tag = read(io, UInt8)
+
+ if tag == TYPE_NULL
+ return nothing
+ elseif tag == TYPE_BOOL
+ return read(io, UInt8) != 0x00
+ elseif tag == TYPE_INT
+ return ltoh(read(io, Int64)) # little-endian to host
+ elseif tag == TYPE_FLOAT
+ return ltoh(read(io, Float64))
+ elseif tag == TYPE_STRING
+ len = ltoh(read(io, UInt32))
+ bytes = read(io, len)
+ return String(bytes)
+ elseif tag == TYPE_ARRAY
+ len = ltoh(read(io, UInt32))
+ return Any[_decode_value(io) for _ in 1:len]
+ elseif tag == TYPE_RECORD
+ len = ltoh(read(io, UInt32))
+ dict = Dict{String, Any}()
+ for _ in 1:len
+ key_len = ltoh(read(io, UInt32))
+ key = String(read(io, key_len))
+ dict[key] = _decode_value(io)
+ end
+ return dict
+ else
+ error("Unknown type tag in binary protocol: $tag")
+ end
+end
+
+function _check_ffi_available()
+ if !FFI_AVAILABLE
+ error("FFI not available. Build the Rust library with:\n" *
+ " cd rust/nickel-jl && cargo build --release\n" *
+ " mkdir -p deps && cp target/release/$LIB_NAME ../../deps/")
+ end
+end
+
+function _throw_ffi_error()
+ error_ptr = ccall((:nickel_get_error, LIB_PATH), Ptr{Cchar}, ())
+ if error_ptr != C_NULL
+ throw(NickelError(unsafe_string(error_ptr)))
+ else
+ throw(NickelError("Nickel evaluation failed with unknown error"))
+ end
+end
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -1,7 +1,7 @@
using NickelEval
using Test
-# Check if Nickel is available
+# Check if Nickel CLI is available
function nickel_available()
try
Sys.which("nickel") !== nothing
@@ -14,7 +14,14 @@ end
if nickel_available()
include("test_subprocess.jl")
else
- @warn "Nickel executable not found in PATH, skipping tests. Install from: https://nickel-lang.org/"
- @test_skip "Nickel not available"
+ @warn "Nickel executable not found in PATH, skipping subprocess tests. Install from: https://nickel-lang.org/"
+ @test_skip "Nickel CLI not available"
+ end
+
+ if check_ffi_available()
+ include("test_ffi.jl")
+ else
+ @warn "FFI library not available, skipping FFI tests. Build with: cd rust/nickel-jl && cargo build --release"
+ @test_skip "FFI not available"
end
end
diff --git a/test/test_ffi.jl b/test/test_ffi.jl
@@ -0,0 +1,97 @@
+@testset "FFI Native Evaluation" begin
+ @testset "Primitive types" begin
+ # Integers
+ @test nickel_eval_native("42") === Int64(42)
+ @test nickel_eval_native("-42") === Int64(-42)
+ @test nickel_eval_native("0") === Int64(0)
+ @test nickel_eval_native("1000000000000") === Int64(1000000000000)
+
+ # Floats (only true decimals)
+ @test nickel_eval_native("3.14") ≈ 3.14
+ @test nickel_eval_native("-2.718") ≈ -2.718
+ @test nickel_eval_native("0.5") ≈ 0.5
+ @test typeof(nickel_eval_native("3.14")) == Float64
+
+ # Booleans
+ @test nickel_eval_native("true") === true
+ @test nickel_eval_native("false") === false
+
+ # Null
+ @test nickel_eval_native("null") === nothing
+
+ # Strings
+ @test nickel_eval_native("\"hello\"") == "hello"
+ @test nickel_eval_native("\"\"") == ""
+ @test nickel_eval_native("\"hello 世界\"") == "hello 世界"
+ end
+
+ @testset "Arrays" begin
+ @test nickel_eval_native("[]") == Any[]
+ @test nickel_eval_native("[1, 2, 3]") == Any[1, 2, 3]
+ @test nickel_eval_native("[true, false]") == Any[true, false]
+ @test nickel_eval_native("[\"a\", \"b\"]") == Any["a", "b"]
+
+ # Nested arrays
+ result = nickel_eval_native("[[1, 2], [3, 4]]")
+ @test result == Any[Any[1, 2], Any[3, 4]]
+
+ # Mixed types
+ result = nickel_eval_native("[1, \"two\", true, null]")
+ @test result == Any[1, "two", true, nothing]
+ end
+
+ @testset "Records" begin
+ result = nickel_eval_native("{ x = 1 }")
+ @test result isa Dict{String, Any}
+ @test result["x"] === Int64(1)
+
+ result = nickel_eval_native("{ name = \"test\", count = 42 }")
+ @test result["name"] == "test"
+ @test result["count"] === Int64(42)
+
+ # Empty record
+ @test nickel_eval_native("{}") == Dict{String, Any}()
+
+ # Nested records
+ result = nickel_eval_native("{ outer = { inner = 42 } }")
+ @test result["outer"]["inner"] === Int64(42)
+ end
+
+ @testset "Type preservation" begin
+ # The key feature: integers stay Int64, floats stay Float64
+ @test typeof(nickel_eval_native("42")) == Int64
+ @test typeof(nickel_eval_native("42.5")) == Float64
+ @test typeof(nickel_eval_native("42.0")) == Int64 # whole numbers → Int64
+ end
+
+ @testset "Computed values" begin
+ @test nickel_eval_native("1 + 2") === Int64(3)
+ @test nickel_eval_native("10 - 3") === Int64(7)
+ @test nickel_eval_native("let x = 10 in x * 2") === Int64(20)
+ @test nickel_eval_native("let add = fun x y => x + y in add 3 4") === Int64(7)
+ end
+
+ @testset "Record operations" begin
+ # Merge
+ result = nickel_eval_native("{ a = 1 } & { b = 2 }")
+ @test result["a"] === Int64(1)
+ @test result["b"] === Int64(2)
+ end
+
+ @testset "Array operations" begin
+ result = nickel_eval_native("[1, 2, 3] |> std.array.map (fun x => x * 2)")
+ @test result == Any[2, 4, 6]
+ end
+end
+
+@testset "FFI JSON Evaluation" begin
+ # Test the JSON path still works
+ @test nickel_eval_ffi("42") == 42
+ @test nickel_eval_ffi("\"hello\"") == "hello"
+ @test nickel_eval_ffi("{ x = 1 }").x == 1
+
+ # Typed evaluation
+ result = nickel_eval_ffi("{ x = 1, y = 2 }", Dict{String, Int})
+ @test result isa Dict{String, Int}
+ @test result["x"] == 1
+end
