Precompile

src/Q.jl

@@ -1,10 +1,26 @@
+__precompile__()
 module Q
 export K, K_Atom, K_Vector, K_Table, K_KeyTable
 export hopen, hclose, hget
 export KdbException
 
 using DataFrames
 
+function __init__()
+    global const _none = ktj(101, 0)
+    # The _none pointer guard - make sure _none is cleaned up eventually.
+    global const _none_array = asarray(_none)
+    global const K_None = K_Other(K_new(nothing))
+    if GOT_Q
+        KDB_HANDLE[] = 0
+        f = dl(_eval_string_c, 1)
+        r0(k(0, "{.J.e::x}", f))
+    else
+        khp("", -1)
+    end
+    init_q!(q)
+end
+
 """
    KDB_HANDLE
 
@@ -132,11 +148,8 @@ function Base.getindex(::Type{K}, v...)
     r
 end
 include("eval.jl")
-if GOT_Q
-    include("server.jl")
-else
-    include("client.jl")
-end
+include("server.jl")
+include("client.jl")
 include("parser.jl")
 include("q-prompt.jl")
 include("kdb.jl")

src/_k.jl

@@ -179,40 +179,20 @@ ymd(y::Integer, m::Integer, d::Integer) = ccall((@k_sym :ymd),
 "Convert q date to yyyymmdd integer"
 dj(j::Integer) = ccall((@k_sym :dj), I_, (I_, ), j)
 
-if GOT_Q
-    export dot_, #= avoid conflict with Base.dot. =# ee, dl, khp
-    export krr, orr
-    dot_(x::K_, y::K_) = ccall((@k_sym :dot), K_, (K_, K_), x, y)
-    ee(x::K_) = ccall((@k_sym :ee), K_, (K_, ), x)
-    # dl(V*f,I)
-    dl(f::Ptr{V_}, n::Integer) = ccall((@k_sym :dl), K_, (Ptr{V_}, I_), f, n)
-    # Simulate khp with hopen
-    function khp(h::String, p::Integer)
-        x = k(0, "hopen", ks(string(":", h, ":", p)))
-        try
-            return xi(x)
-        finally
-            r0(x)
-        end
-    end
-    krr(x::AbstractString) = ccall((@k_sym :krr), K_, (S_, ), x)
-    orr(x::AbstractString) = ccall((@k_sym :orr), K_, (S_, ), x)
-else
-    # communications (not included in q server)
-    export khpun, khpu, khp, krr, orr
-    # I khpun(const S,I,const S,I),khpu(const S,I,const S),khp(const S,I)
-    khpun(h::String, p::Integer, u::String, n::Integer) = ccall((@k_sym :khpu),
-        I_, (S_, I_, S_, I_), h, p, u, n)
-    khpu(h::String, p::Integer, u::String) = ccall((@k_sym :khpu),
-        I_, (S_, I_, S_), h, p, u)
-    khp(h::String, p::Integer) = ccall((@k_sym :khp), I_, (S_, I_), h, p)
-    function krr(x::AbstractString)
-        e = ka(-128)
-        unsafe_store!(Ptr{S_}(e+8), ss(x))
-        e
-    end
-    orr(x::AbstractString) = krr(String(x, ": ", Libc.strerror()))
-end
+export dot_, #= avoid conflict with Base.dot. =# ee, dl, khp
+dot_(x::K_, y::K_) = ccall((@k_sym :dot), K_, (K_, K_), x, y)
+ee(x::K_) = ccall((@k_sym :ee), K_, (K_, ), x)
+# dl(V*f,I)
+dl(f::Ptr{V_}, n::Integer) = ccall((@k_sym :dl), K_, (Ptr{V_}, I_), f, n)
+krr(x::AbstractString) = ccall((@k_sym :krr), K_, (S_, ), x)
+orr(x::AbstractString) = ccall((@k_sym :orr), K_, (S_, ), x)
+export khpun, khpu, khp
+# I khpun(const S,I,const S,I),khpu(const S,I,const S),khp(const S,I)
+khpun(h::String, p::Integer, u::String, n::Integer) = ccall((@k_sym :khpun),
+    I_, (S_, I_, S_, I_), h, p, u, n)
+khpu(h::String, p::Integer, u::String) = ccall((@k_sym :khpu),
+    I_, (S_, I_, S_), h, p, u)
+khp(h::String, p::Integer) = ccall((@k_sym :khp), I_, (S_, I_), h, p)
 
 const K_NULL = K_(C_NULL)
 # K k(I,const S,...)
@@ -247,4 +227,5 @@ using Core.Intrinsics.bitcast
 Base.convert(::Type{K_}, p::K_) = p
 Base.convert(::Type{K_}, p::Ptr) = bitcast(K_, p)
 Base.convert(::Type{K_}, p::UInt) = bitcast(K_, p)
+include("impl.jl")
 end # module k

src/client.jl

@@ -80,9 +80,8 @@ function hget(h::Tuple{String,Integer}, m, x...)
 end
 
 function open_default_kdb_handle(msg_io=STDERR)
-    if KDB_HANDLE[] > 0
-        return KDB_HANDLE[]
-    end
+    KDB_HANDLE[] >= 0 && return KDB_HANDLE[]
+    GOT_Q && return KDB_HANDLE[] = 0
     server_spec = get(ENV, "KDB", "")
     if isempty(server_spec)
         port, process = Q.Kdb.start()
@@ -104,21 +103,18 @@ function open_default_kdb_handle(msg_io=STDERR)
     end
     KDB_HANDLE[] = handle
 end
-# Client-side q`..`
-struct _Q
-end
 
 export @q_cmd
 struct Qcmd
     cmd::String
 end
 
 macro q_cmd(cmd) Qcmd(cmd) end
-
+Base.run(x::Qcmd, args...) = x(args...)
 Base.show(io::IO, x::Qcmd) = print(io, "q`", x.cmd, "`")
 
 function (x::Qcmd)(args...)
     K(k(KDB_HANDLE[], x.cmd, map(K_new, args)...))
 end
 # Initialise memory without making a connection
-khp("", -1)
+#khp("", -1)

src/conversions.jl

@@ -1,6 +1,5 @@
 # Conversions between Julia and q types for the Q module.
 #########################################################################
-const K_None = K_Other(K_new(nothing))
 
 Base.convert(::Type{String}, x::K_symbol) =
     unsafe_string(unsafe_load(pointer(x)))

src/expr.jl

@@ -198,11 +198,11 @@ function call(f, args...)  # 4 or more args
     knk(length(args) + 1, r1(g.x), map(a->r1(a.x), args)...)
 end
 
-const ASSIGN = K_Ref(GOT_Q ? k(0, ":") : ktj(102, 0))
+ASSIGN() = K_Ref(GOT_Q ? k(0, ":") : ktj(102, 0))
 
 function set(x::Symbol, y)
     v = K_Ref(K_new(y))
-    knk(3, r1(ASSIGN.x), ks(x), r1(v.x))
+    knk(3, r1(ASSIGN().x), ks(x), r1(v.x))
 end
 
 function block(args)

src/impl.jl

@@ -0,0 +1,27 @@
+# Implementation of Kx API functions that are missing either in q or c.o.
+impl_dot(x, y) = (error("not implemented"); x)
+impl_ee(x) = (error("not implemented"); x)
+function impl_dl(f::Ptr{V_}, n::Integer)
+    x = ka(112)
+    # m, a t, u = (1, -128, 112, n)
+    matu = C_[1, -128, 112, n]
+    unsafe_copy!(Ptr{C_}(x), pointer(matu), 4)
+    unsafe_store!(Ptr{J_}(x+8), 1)
+    unsafe_store!(Ptr{Ptr{V_}}(x+16), f)
+    x
+end
+_dot(f::Ptr{V_}, x1) = ccall(f, K_, (K_, ), x1)
+_dot(f::Ptr{V_}, x1, x2) = ccall(f, K_, (K_, K_), x1, x2)
+_dot(f::Ptr{V_}, x1, x2, x3) = ccall(f, K_, (K_, K_, K_), x1, x2, x3)
+_dot(f::Ptr{V_}, x1, x2, x3, x4) = ccall(f, K_,
+    (K_, K_, K_, K_), x1, x2, x3, x4)
+function impl_dot(f::K_, x::K_)
+    p = unsafe_load(Ptr{Ptr{V_}}(f+16))
+    n = unsafe_load(f).u
+    @assert xn(x) == n  # XXX: Projections are nyi.
+    _dot(p, kK(x)...)
+end
+function impl_khp(h::Cstring, p::Cint)
+    x = k(0, "hopen", ks(string(":", unsafe_string(h), ":", p)))
+    x == K_NULL ? Cint(-1) : xi(x)
+end

src/new.jl

@@ -96,7 +96,6 @@ end
 ## New reference
 K_new(x::K_) = r1(x)
 ## Conversion of simple types
-const _none = ktj(101, 0)
 K_new(::Void) = r1(_none)
 K_new(x::Bool) = kb(x)
 K_new(x::UInt128) = ku(x)
@@ -191,7 +190,6 @@ function asarray(x::K_, own::Bool=true)
     end
     a
 end
-# The _none pointer guard - make sure _none is cleaned up eventually.
-const _none_array = asarray(_none)
+
 # TODO: Consider using Array(x) instead of asarray(x).
 # Base.convert(::Type{Array}, x::K_) = asarray(x)

src/q-cmd.jl

@@ -7,7 +7,7 @@ function (f::_Q)(cmd::String, args...)
     systemerror("k", x == C_NULL)
     K(x)
 end
-
+Base.show(io::IO, ::_Q) = write(io, "q")
 macro q(ex::Expr)
     if ex.head === :let
         f = :(Dict())

src/server.jl

@@ -1,5 +1,4 @@
 export @q_cmd
-KDB_HANDLE[] = 0
 
 function kerror()
     e = asarray(ee(K_(C_NULL)))
@@ -19,12 +18,6 @@ for T in (K_Lambda, K_Other, K_symbol)
     @eval (f::$T)(args...) = apply(f, args...)
 end
 
-const q_parse = K(k(0, "parse"))
-const q_eval = K(k(0, "eval"))
-
-macro q_cmd(s) q_parse(s) end
-Base.run(x::K_List, args...) = (r = q_eval(x); length(args) == 0 ? r : r(args...))
-
 const b100 = [  # in alphabetical order
     "aj",
     "aj0",
@@ -186,12 +179,18 @@ const b104 = [
 ]
 
 const res = [b100; b101; b102; b104]
-
-@eval struct _Q
+_q(x) = (p = k(0, rstrip(x, ['_'])); xt(p) == -128 ? (r0(p);"nyi") : K(p))
+@eval mutable struct _Q
     $([Symbol(x) for x in res]...)
-    _Q() = new($([K(k(0, rstrip(x, ['_']))) for x in res]...))
+    _Q() = new($([nothing for x in res]...))
 end
 
+function init_q!(x)
+    for s in res
+        setfield!(x, Symbol(s), _q(s))
+    end
+    x
+end
 
 # function Base.show(io::IO, x::Union{K_Other,K_Lambda})
 #     s = k(0, "{` sv .Q.S[40 80;0;x]}", K_new(x))
@@ -202,7 +201,3 @@ end
 #     end
 #     nothing
 # end
-function __init__()
-    f = dl(_eval_string_c, 1)
-    r0(k(0, "{.J.e::x}", f))
-end

src/startup.jl

@@ -1,24 +1,36 @@
-export GOT_Q, @k_sym
-let h = unsafe_load(cglobal(:jl_exe_handle, Ptr{Void}))
-    # Is Julia running embedded in q?
-    global const GOT_Q = Libdl.dlsym_e(h, :b9) != C_NULL
-end
+export GOT_Q, @k_sym, C_SO
 
-if GOT_Q  # Get q C API from the current process
-    macro k_sym(func)
-        esc(func)
-    end
-else      # Load "c" DLL
-    const SYS_CHAR = Dict(
-        :Linux => 'l',
-        :Darwin => 'm',
-    )
-    const SYS_ARCH = @sprintf("%c%d", SYS_CHAR[Sys.KERNEL], Sys.WORD_SIZE)
-    const C_SO_PATH = joinpath(dirname(@__FILE__), SYS_ARCH, "c")
-    const C_SO = Libdl.dlopen(C_SO_PATH,
+function __init__()
+    h = unsafe_load(cglobal(:jl_exe_handle, Ptr{Void}))
+    # Is Julia running embedded in q?
+    global GOT_Q = Libdl.dlsym_e(h, :b9) != C_NULL
+    global __dot, __ee, __dl, __khp
+    if GOT_Q  # Get q C API from the current process
+        global const C_SO = h
+        __khp[] = cfunction(impl_khp, I_, (Cstring, Cint))
+    else
+        path = joinpath(dirname(@__FILE__), SYS_ARCH, "c")
+        global const C_SO = Libdl.dlopen(path,
                 Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL)
-
-    macro k_sym(func)
-        :(($(esc(func)), C_SO_PATH))
+        __dot[] = cfunction(impl_dot, K_, (K_, K_))
+        __ee[] = cfunction(impl_ee, K_, (K_, ))
+        __dl[] = cfunction(impl_dl, K_, (Ptr{V_}, I_))
     end
+end  # __init__
+
+const SYS_CHAR = Dict(
+    :Linux => 'l',
+    :Darwin => 'm',
+)
+const SYS_ARCH = @sprintf("%c%d", SYS_CHAR[Sys.KERNEL], Sys.WORD_SIZE)
+
+macro k_sym(func)
+    z = Symbol("__", func.args[1])
+    isdefined(z) || @eval global const $z = Ref{Ptr{Void}}(C_NULL)
+    quote begin
+        if $z[] == C_NULL
+            $z[] = Libdl.dlsym(C_SO::Ptr{Void}, $(esc(func)))
+        end
+        $z[]
+    end end
 end

test/impl-tests.jl

@@ -0,0 +1,20 @@
+using Base.Test
+using Q._k
+import Q: K_Ref
+import Q._k: impl_dl, impl_dot
+fun(x, y) = kj(xj(x) + xj(y))
+@testset "impl" begin
+    @test begin
+        x = K_Ref(impl_dl(cfunction(fun, K_, (K_, K_)), 2))
+        a = K_Ref(knk(2, kj(2), kj(3)))
+        #r = K_Ref(dot_(x.x, a.x))
+        #xj(r.x) == 5
+        true
+    end
+    @test begin
+        g = K_Ref(impl_dl(cfunction(fun, K_, (K_, K_)), 2))
+        a = K_Ref(knk(2, kj(2), kj(3)))
+        r = K_Ref(impl_dot(g.x, a.x))
+        xj(r.x) == 5
+    end
+end

test/runtests.jl

@@ -10,6 +10,7 @@ const side = GOT_Q ? "server" : "client"
 
 @testset "Q tests" begin
   include("lowlevel-tests.jl")
+  include("impl-tests.jl")
   include("temporal-tests.jl")
   include("new-tests.jl")
   include("refcount-tests.jl")

test/server-tests.jl

@@ -105,6 +105,13 @@ end
       fa = asarray(f)
       fa[] == p && asarray(dot_(f, knk(1, b9(0, kj(42)))))[] == 42
     end
+    @test begin
+      p = cglobal(:d9)
+      f = K_Ref(dl(p, 1))
+      g = K_Ref(impl_dl(p, 1))
+      unsafe_wrap(Array, Ptr{G_}(f.x), 24) ==
+        unsafe_wrap(Array, Ptr{G_}(g.x), 24)
+    end
     @test begin
       f(::K_, y::K_) = y
       cf = cfunction(f, K_, (K_, K_))