From 2636515435e77a10ec3ec2c980fd945169c776dc Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 3 Dec 2024 14:41:06 -0800 Subject: [PATCH] Integrate symbol server --- Project.toml | 22 +- src/JuliaWorkspaces.jl | 4 + src/SymbolServer/SymbolServer.jl | 14 + src/SymbolServer/faketypes.jl | 172 ++++++++ src/SymbolServer/serialize.jl | 290 +++++++++++++ src/SymbolServer/symbols.jl | 659 +++++++++++++++++++++++++++++ src/SymbolServer/utils.jl | 688 +++++++++++++++++++++++++++++++ src/fileio.jl | 4 +- src/inputs.jl | 2 + src/layer_symbols.jl | 7 + src/public.jl | 28 ++ src/types.jl | 63 ++- 12 files changed, 1939 insertions(+), 14 deletions(-) create mode 100644 src/SymbolServer/SymbolServer.jl create mode 100644 src/SymbolServer/faketypes.jl create mode 100644 src/SymbolServer/serialize.jl create mode 100644 src/SymbolServer/symbols.jl create mode 100644 src/SymbolServer/utils.jl create mode 100644 src/layer_symbols.jl diff --git a/Project.toml b/Project.toml index 3bd583c..85bdec5 100644 --- a/Project.toml +++ b/Project.toml @@ -4,25 +4,27 @@ authors = ["David Anthoff "] version = "4.6.1-DEV" [deps] +AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f" +CancellationTokens = "2e8d271d-f2e2-407b-a864-17eb2156783e" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f" Salsa = "1fbf2c77-44e2-4d5d-8131-0fa618a5c278" -UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" TestItemDetection = "76b0de8b-5c4b-48ef-a724-914b33ca988d" -CancellationTokens = "2e8d271d-f2e2-407b-a864-17eb2156783e" - -[extras] -TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -JuliaSyntax = "0.4" -julia = "1.6" AutoHashEquals = "1,2" +CancellationTokens = "1" +InteractiveUtils = "1.11.0" +JuliaSyntax = "0.4" Salsa = "2.2.0" TestItemDetection = "1.1" -CancellationTokens = "1" +julia = "1.6" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" [targets] test = ["Test", "TestItemRunner"] diff --git a/src/JuliaWorkspaces.jl b/src/JuliaWorkspaces.jl index b3fce9e..dd9f2f4 100644 --- a/src/JuliaWorkspaces.jl +++ b/src/JuliaWorkspaces.jl @@ -17,6 +17,9 @@ using .URIs2: filepath2uri, uri2filepath using .URIs2: URI, @uri_str +include("SymbolServer/SymbolServer.jl") +import .SymbolServer + include("exception_types.jl") include("types.jl") include("sourcetext.jl") @@ -27,6 +30,7 @@ include("layer_projects.jl") include("layer_testitems.jl") include("layer_diagnostics.jl") include("fileio.jl") +include("layer_symbols.jl") include("public.jl") end diff --git a/src/SymbolServer/SymbolServer.jl b/src/SymbolServer/SymbolServer.jl new file mode 100644 index 0000000..ab6c309 --- /dev/null +++ b/src/SymbolServer/SymbolServer.jl @@ -0,0 +1,14 @@ +module SymbolServer + +import Pkg, InteractiveUtils, UUIDs + +using UUIDs: UUID + +include("faketypes.jl") +include("symbols.jl") +include("utils.jl") +include("serialize.jl") + +using .CacheStore + +end diff --git a/src/SymbolServer/faketypes.jl b/src/SymbolServer/faketypes.jl new file mode 100644 index 0000000..94e1123 --- /dev/null +++ b/src/SymbolServer/faketypes.jl @@ -0,0 +1,172 @@ +########## Fake type-system + + +# Used to label all objects +@auto_hash_equals struct VarRef + parent::Union{VarRef,Nothing} + name::Symbol +end +VarRef(m::Module) = VarRef((parentmodule(m) == Main || parentmodule(m) == m) ? nothing : VarRef(parentmodule(m)), nameof(m)) + +# These mirror Julia types (w/o the Fake prefix) +@auto_hash_equals struct FakeTypeName + name::VarRef + parameters::Vector{Any} +end + +function FakeTypeName(@nospecialize(x); justname=false) + @static if !(Vararg isa Type) + x isa typeof(Vararg) && return FakeTypeofVararg(x) + end + if x isa DataType + xname = x.name + xnamename = xname.name # necessary but unclear why. + if justname + FakeTypeName(VarRef(VarRef(x.name.module), x.name.name), []) + else + # FakeTypeName(VarRef(VarRef(x.name.module), x.name.name), _parameter.(x.parameters)) + ft = FakeTypeName(VarRef(VarRef(x.name.module), x.name.name), []) + for p in x.parameters + push!(ft.parameters, _parameter(p)) + end + ft + end + elseif x isa Union + FakeUnion(x) + elseif x isa UnionAll + FakeUnionAll(x) + elseif x isa TypeVar + FakeTypeVar(x) + elseif x isa Core.TypeofBottom + FakeTypeofBottom() + elseif x isa Module + VarRef(x) + else + error((x, typeof(x))) + end +end + +@auto_hash_equals struct FakeTypeofBottom end +@auto_hash_equals struct FakeUnion + a + b +end +FakeUnion(u::Union) = FakeUnion(FakeTypeName(u.a, justname=true), FakeTypeName(u.b, justname=true)) +@auto_hash_equals struct FakeTypeVar + name::Symbol + lb + ub +end +FakeTypeVar(tv::TypeVar) = FakeTypeVar(tv.name, FakeTypeName(tv.lb, justname=true), FakeTypeName(tv.ub, justname=true)) +@auto_hash_equals struct FakeUnionAll + var::FakeTypeVar + body::Any +end +FakeUnionAll(ua::UnionAll) = FakeUnionAll(FakeTypeVar(ua.var), FakeTypeName(ua.body, justname=true)) + +function _parameter(@nospecialize(p)) + if p isa Union{Int,Symbol,Bool,Char} + p + elseif !(p isa Type) && isbitstype(typeof(p)) + 0 + elseif p isa Tuple + _parameter.(p) + else + FakeTypeName(p, justname=true) + end +end + +Base.print(io::IO, vr::VarRef) = vr.parent === nothing ? print(io, vr.name) : print(io, vr.parent, ".", vr.name) +function Base.print(io::IO, tn::FakeTypeName) + print(io, tn.name) + if !isempty(tn.parameters) + print(io, "{") + for i = 1:length(tn.parameters) + print(io, tn.parameters[i]) + i != length(tn.parameters) && print(io, ",") + end + print(io, "}") + end +end +Base.print(io::IO, x::FakeUnionAll) = print(io, x.body, " where ", x.var) +function Base.print(io::IO, x::FakeUnion; inunion=false) + !inunion && print(io, "Union{") + print(io, x.a, ",") + if x.b isa FakeUnion + print(io, x.b, inunion=true) + else + print(io, x.b, "}") + end +end +function Base.print(io::IO, x::FakeTypeVar) + if isfakebottom(x.lb) + if isfakeany(x.ub) + print(io, x.name) + else + print(io, x.name, "<:", x.ub) + end + elseif isfakeany(x.ub) + print(io, x.lb, "<:", x.name) + else + print(io, x.lb, "<:", x.name, "<:", x.ub) + end +end + +isfakeany(t) = false +isfakeany(t::FakeTypeName) = isfakeany(t.name) +isfakeany(vr::VarRef) = vr.name === :Any && vr.parent isa VarRef && vr.parent.name === :Core && vr.parent.parent === nothing + +isfakebottom(t) = false +isfakebottom(t::FakeTypeofBottom) = true + +Base.:(==)(a::FakeTypeName, b::FakeTypeName) = a.name == b.name && a.parameters == b.parameters +Base.:(==)(a::VarRef, b::VarRef) = a.parent == b.parent && a.name == b.name +Base.:(==)(a::FakeTypeVar, b::FakeTypeVar) = a.lb == b.lb && a.name == b.name && a.ub == b.ub +Base.:(==)(a::FakeUnionAll, b::FakeUnionAll) = a.var == b.var && a.body == b.body +Base.:(==)(a::FakeUnion, b::FakeUnion) = a.a == b.a && a.b == b.b +Base.:(==)(a::FakeTypeofBottom, b::FakeTypeofBottom) = true + +@static if !(Vararg isa Type) + @auto_hash_equals struct FakeTypeofVararg + T + N + FakeTypeofVararg() = new() + FakeTypeofVararg(T) = (new(T)) + FakeTypeofVararg(T, N) = new(T, N) + end + function FakeTypeofVararg(va::typeof(Vararg)) + if isdefined(va, :N) + vaN = va.N isa TypeVar ? FakeTypeVar(va.N) : va.N + FakeTypeofVararg(FakeTypeName(va.T; justname=true), vaN) # This should be FakeTypeName(va.N) but seems to crash inference. + elseif isdefined(va, :T) + FakeTypeofVararg(FakeTypeName(va.T; justname=true)) + else + FakeTypeofVararg() + end + end + function Base.print(io::IO, va::FakeTypeofVararg) + print(io, "Vararg") + if isdefined(va, :T) + print(io, "{", va.T) + if isdefined(va, :N) + print(io, ",", va.N) + end + print(io, "}") + end + end + function Base.:(==)(a::FakeTypeofVararg, b::FakeTypeofVararg) + if isdefined(a, :T) + if isdefined(b, :T) && a.T == b.T + if isdefined(a, :N) + isdefined(b, :N) && a.N == b.N + else + !isdefined(b, :N) + end + else + false + end + else + !isdefined(b, :T) + end + end +end diff --git a/src/SymbolServer/serialize.jl b/src/SymbolServer/serialize.jl new file mode 100644 index 0000000..d19344d --- /dev/null +++ b/src/SymbolServer/serialize.jl @@ -0,0 +1,290 @@ +module CacheStore +using ..SymbolServer: VarRef, FakeTypeName, FakeTypeofBottom, FakeTypeVar, FakeUnion, FakeUnionAll +using ..SymbolServer: ModuleStore, Package, FunctionStore, MethodStore, DataTypeStore, GenericStore +@static if !(Vararg isa Type) + using ..SymbolServer: FakeTypeofVararg +end + +const NothingHeader = 0x01 +const SymbolHeader = 0x02 +const CharHeader = 0x03 +const IntegerHeader = 0x04 +const StringHeader = 0x05 +const VarRefHeader = 0x06 +const FakeTypeNameHeader = 0x07 +const FakeTypeofBottomHeader = 0x08 +const FakeTypeVarHeader = 0x09 +const FakeUnionHeader = 0x0a +const FakeUnionAllHeader = 0xb +const ModuleStoreHeader = 0x0c +const MethodStoreHeader = 0x0d +const FunctionStoreHeader = 0x0e +const DataTypeStoreHeader = 0x0f +const GenericStoreHeader = 0x10 +const PackageHeader = 0x11 +const TrueHeader = 0x12 +const FalseHeader = 0x13 +const TupleHeader = 0x14 +const FakeTypeofVarargHeader = 0x15 +const UndefHeader = 0x16 + + +function write(io, x::VarRef) + Base.write(io, VarRefHeader) + write(io, x.parent) + write(io, x.name) +end +function write(io, x::Nothing) + Base.write(io, NothingHeader) +end +function write(io, x::Char) + Base.write(io, CharHeader) + Base.write(io, UInt32(x)) +end +function write(io, x::Bool) + x ? Base.write(io, TrueHeader) : Base.write(io, FalseHeader) +end +function write(io, x::Int) + Base.write(io, IntegerHeader) + Base.write(io, x) +end +function write(io, x::Symbol) + Base.write(io, SymbolHeader) + Base.write(io, sizeof(x)) + Base.write(io, String(x)) +end +function write(io, x::NTuple{N,Any}) where N + Base.write(io, TupleHeader) + Base.write(io, N) + for i = 1:N + write(io, x[i]) + end +end +function write(io, x::String) + Base.write(io, StringHeader) + Base.write(io, sizeof(x)) + Base.write(io, x) +end +function write(io, x::FakeTypeName) + Base.write(io, FakeTypeNameHeader) + write(io, x.name) + write_vector(io, x.parameters) +end +write(io, x::FakeTypeofBottom) = Base.write(io, FakeTypeofBottomHeader) +function write(io, x::FakeTypeVar) + Base.write(io, FakeTypeVarHeader) + write(io, x.name) + write(io, x.lb) + write(io, x.ub) +end +function write(io, x::FakeUnion) + Base.write(io, FakeUnionHeader) + write(io, x.a) + write(io, x.b) +end +function write(io, x::FakeUnionAll) + Base.write(io, FakeUnionAllHeader) + write(io, x.var) + write(io, x.body) +end + +@static if !(Vararg isa Type) + function write(io, x::FakeTypeofVararg) + Base.write(io, FakeTypeofVarargHeader) + isdefined(x, :T) ? write(io, x.T) : Base.write(io, UndefHeader) + isdefined(x, :N) ? write(io, x.N) : Base.write(io, UndefHeader) + end +end + +function write(io, x::MethodStore) + Base.write(io, MethodStoreHeader) + write(io, x.name) + write(io, x.mod) + write(io, x.file) + Base.write(io, x.line) + Base.write(io, length(x.sig)) + for p in x.sig + write(io, p[1]) + write(io, p[2]) + end + write_vector(io, x.kws) + write(io, x.rt) +end + +function write(io, x::FunctionStore) + Base.write(io, FunctionStoreHeader) + write(io, x.name) + write_vector(io, x.methods) + write(io, x.doc) + write(io, x.extends) + write(io, x.exported) +end + +function write(io, x::DataTypeStore) + Base.write(io, DataTypeStoreHeader) + write(io, x.name) + write(io, x.super) + write_vector(io, x.parameters) + write_vector(io, x.types) + write_vector(io, x.fieldnames) + write_vector(io, x.methods) + write(io, x.doc) + write(io, x.exported) +end + +function write(io, x::GenericStore) + Base.write(io, GenericStoreHeader) + write(io, x.name) + write(io, x.typ) + write(io, x.doc) + write(io, x.exported) +end + +function write(io, x::ModuleStore) + Base.write(io, ModuleStoreHeader) + write(io, x.name) + Base.write(io, length(x.vals)) + for p in x.vals + write(io, p[1]) + write(io, p[2]) + end + write(io, x.doc) + write(io, x.exported) + write_vector(io, x.exportednames) + write_vector(io, x.used_modules) +end + +function write(io, x::Package) + Base.write(io, PackageHeader) + write(io, x.name) + write(io, x.val) + Base.write(io, UInt128(x.uuid)) + Base.write(io, x.sha === nothing ? zeros(UInt8, 32) : x.sha) +end + +function write_vector(io, x) + Base.write(io, length(x)) + for p in x + write(io, p) + end +end + +function read(io, t = Base.read(io, UInt8)) + # There are a bunch of `yield`s in potentially expensive code paths. + # One top-level `yield` would probably increase responsiveness in the + # LS, but increases runtime by 3x. This seems like a good compromise. + + if t === VarRefHeader + VarRef(read(io), read(io)) + elseif t === NothingHeader + nothing + elseif t === SymbolHeader + n = Base.read(io, Int) + out = Vector{UInt8}(undef, n) + readbytes!(io, out, n) + Symbol(String(out)) + elseif t === StringHeader + # yield() + n = Base.read(io, Int) + out = Vector{UInt8}(undef, n) + readbytes!(io, out, n) + String(out) + elseif t === CharHeader + Char(Base.read(io, UInt32)) + elseif t === IntegerHeader + Base.read(io, Int) + elseif t === FakeTypeNameHeader + FakeTypeName(read(io), read_vector(io, Any)) + elseif t === FakeTypeofBottomHeader + FakeTypeofBottom() + elseif t === FakeTypeVarHeader + FakeTypeVar(read(io), read(io), read(io)) + elseif t === FakeUnionHeader + FakeUnion(read(io), read(io)) + elseif t === FakeUnionAllHeader + FakeUnionAll(read(io), read(io)) + elseif t === FakeTypeofVarargHeader + T, N = read(io), read(io) + if T === nothing + FakeTypeofVararg() + elseif N === nothing + FakeTypeofVararg(T) + else + FakeTypeofVararg(T, N) + end + elseif t === UndefHeader + nothing + elseif t === MethodStoreHeader + # yield() + name = read(io) + mod = read(io) + file = read(io) + line = Base.read(io, UInt32) + nsig = Base.read(io, Int) + sig = Vector{Pair{Any, Any}}(undef, nsig) + for i in 1:nsig + sig[i] = read(io) => read(io) + end + kws = read_vector(io, Symbol) + rt = read(io) + MethodStore(name, mod, file, line, sig, kws, rt) + elseif t === FunctionStoreHeader + # yield() + FunctionStore(read(io), read_vector(io, MethodStore), read(io), read(io), read(io)) + elseif t === DataTypeStoreHeader + # yield() + DataTypeStore(read(io), read(io), read_vector(io, Any), read_vector(io, Any), read_vector(io, Any), read_vector(io, MethodStore), read(io), read(io)) + elseif t === GenericStoreHeader + # yield() + GenericStore(read(io), read(io), read(io), read(io)) + elseif t === ModuleStoreHeader + # yield() + name = read(io) + n = Base.read(io, Int) + vals = Dict{Symbol,Any}() + sizehint!(vals, n) + for _ = 1:n + k = read(io) + v = read(io) + vals[k] = v + end + doc = read(io) + exported = read(io) + exportednames = read_vector(io, Symbol) + used_modules = read_vector(io, Symbol) + ModuleStore(name, vals, doc, exported, exportednames, used_modules) + elseif t === TrueHeader + true + elseif t === FalseHeader + false + elseif t === TupleHeader + N = Base.read(io, Int) + ntuple(i->read(io), N) + elseif t === PackageHeader + # yield() + name = read(io) + val = read(io) + uuid = Base.UUID(Base.read(io, UInt128)) + sha = Base.read(io, 32) + Package(name, val, uuid, all(x == 0x00 for x in sha) ? nothing : sha) + else + error("Unknown type: $t") + end +end + +function read_vector(io, T) + n = Base.read(io, Int) + v = Vector{T}(undef, n) + for i in 1:n + v[i] = read(io) + end + v +end + +function storeunstore(x) + io = IOBuffer() + write(io, x) + bs = take!(io) + read(IOBuffer(bs)) +end +end diff --git a/src/SymbolServer/symbols.jl b/src/SymbolServer/symbols.jl new file mode 100644 index 0000000..fc870e1 --- /dev/null +++ b/src/SymbolServer/symbols.jl @@ -0,0 +1,659 @@ +# using LibGit2, InteractiveUtils + +mutable struct Server + storedir::String + context::Pkg.Types.Context + depot::Dict +end + +abstract type SymStore end +@auto_hash_equals struct ModuleStore <: SymStore + name::VarRef + vals::Dict{Symbol,Any} + doc::String + exported::Bool + exportednames::Vector{Symbol} + used_modules::Vector{Symbol} +end + +ModuleStore(m) = ModuleStore(VarRef(m), Dict{Symbol,Any}(), _doc(Base.Docs.Binding(m, nameof(m))), true, unsorted_names(m), Symbol[]) +Base.getindex(m::ModuleStore, k) = m.vals[k] +Base.setindex!(m::ModuleStore, v, k) = (m.vals[k] = v) +Base.haskey(m::ModuleStore, k) = haskey(m.vals, k) + +const EnvStore = Dict{Symbol,ModuleStore} + +@auto_hash_equals struct Package + name::String + val::ModuleStore + uuid::Base.UUID + sha::Union{Vector{UInt8},Nothing} +end +Package(name::String, val::ModuleStore, uuid::String, sha) = Package(name, val, Base.UUID(uuid), sha) + +@auto_hash_equals struct MethodStore + name::Symbol + mod::Symbol + file::String + line::Int32 + sig::Vector{Pair{Any,Any}} + kws::Vector{Symbol} + rt::Any +end + +@auto_hash_equals struct DataTypeStore <: SymStore + name::FakeTypeName + super::FakeTypeName + parameters::Vector{Any} + types::Vector{Any} + fieldnames::Vector{Any} + methods::Vector{MethodStore} + doc::String + exported::Bool + function DataTypeStore(names, super, parameters, fieldtypes, fieldnames, methods, doc, exported) + if length(fieldtypes) < length(fieldnames) + append!(fieldtypes, [Any for _ in 1:(length(fieldnames)-length(fieldtypes))]) + end + new(names, super, parameters, fieldtypes, fieldnames, methods, doc, exported) + end +end + +function DataTypeStore(@nospecialize(t), symbol, parent_mod, exported) + ur_t = Base.unwrap_unionall(t) + parameters = if isdefined(ur_t, :parameters) + map(ur_t.parameters) do p + _parameter(p) + end + else + [] + end + types = if isdefined(ur_t, :types) + map(ur_t.types) do p + FakeTypeName(p) + end + else + [] + end + DataTypeStore(FakeTypeName(ur_t), FakeTypeName(ur_t.super), parameters, types, isconcretetype(ur_t) && fieldcount(ur_t) > 0 ? collect(fieldnames(ur_t)) : Symbol[], MethodStore[], _doc(Base.Docs.Binding(parent_mod, symbol)), exported) +end + +@auto_hash_equals struct FunctionStore <: SymStore + name::VarRef + methods::Vector{MethodStore} + doc::String + extends::VarRef + exported::Bool +end + +function FunctionStore(@nospecialize(f), symbol, parent_mod, exported) + if f isa Core.IntrinsicFunction + FunctionStore(VarRef(VarRef(Core.Intrinsics), nameof(f)), MethodStore[], _doc(Base.Docs.Binding(parent_mod, symbol)), VarRef(VarRef(parentmodule(f)), nameof(f)), exported) + else + FunctionStore(VarRef(VarRef(parent_mod), nameof(f)), MethodStore[], _doc(Base.Docs.Binding(parent_mod, symbol)), VarRef(VarRef(parentmodule(f)), nameof(f)), exported) + end +end + +@auto_hash_equals struct GenericStore <: SymStore + name::VarRef + typ::Any + doc::String + exported::Bool +end + +# adapted from https://github.com/timholy/CodeTracking.jl/blob/afc73a957f5034cc7f02e084a91283c47882f92b/src/utils.jl#L87-L122 + +""" + path = maybe_fix_path(path) + +Return a normalized, absolute path for a source file `path`. +""" +function maybe_fix_path(file) + if !isabspath(file) + # This may be a Base or Core method + newfile = Base.find_source_file(file) + if isa(newfile, AbstractString) + file = normpath(newfile) + end + end + return maybe_fixup_stdlib_path(file) +end + +safe_isfile(x) = try isfile(x); catch; false end +const BUILDBOT_STDLIB_PATH = dirname(abspath(joinpath(String((InteractiveUtils.@which InteractiveUtils.versioninfo()).file), "..", "..", ".."))) +replace_buildbot_stdlibpath(str::String) = replace(str, BUILDBOT_STDLIB_PATH => Sys.STDLIB) +""" + path = maybe_fixup_stdlib_path(path::String) + +Return `path` corrected for julia issue [#26314](https://github.com/JuliaLang/julia/issues/26314) if applicable. +Otherwise, return the input `path` unchanged. + +Due to the issue mentioned above, location info for methods defined one of Julia's standard libraries +are, for non source Julia builds, given as absolute paths on the worker that built the `julia` executable. +This function corrects such a path to instead refer to the local path on the users drive. +""" +function maybe_fixup_stdlib_path(path) + if !safe_isfile(path) + maybe_stdlib_path = replace_buildbot_stdlibpath(path) + safe_isfile(maybe_stdlib_path) && return maybe_stdlib_path + end + return path +end + +_default_world_age() = + if isdefined(Base, :get_world_counter) + Base.get_world_counter() + else + typemax(UInt) + end + +const _global_method_cache = IdDict{Any,Vector{Any}}() +function methodinfo(@nospecialize(f); types = Tuple, world = _default_world_age()) + key = (f, types, world) + if haskey(_global_method_cache, key) + return _global_method_cache[key] + else + ms = Base._methods(f, types, -1, world) + ms isa Vector || (ms = []) + _global_method_cache[key] = ms + return ms + end +end + +function methodlist(@nospecialize(f)) + ms = methodinfo(f) + Method[x[3]::Method for x in ms] +end + +function sparam_syms(meth::Method) + s = Symbol[] + sig = meth.sig + while sig isa UnionAll + push!(s, Symbol(sig.var.name)) + sig = sig.body + end + return s +end + +function cache_methods(@nospecialize(f), name, env, get_return_type) + if isa(f, Core.Builtin) + return MethodStore[] + end + types = Tuple + world = _default_world_age() + ms = Tuple{Module,MethodStore}[] + methods0 = try + methodinfo(f; types = types, world = world) + catch err + @debug "Error in method lookup for $f" ex=(err, catch_backtrace()) + return ms + end + ind_of_method_w_kws = Int[] # stores the index of methods with kws. + i = 1 + for m in methods0 + # Get inferred method return type + if get_return_type + sparams = Core.svec(sparam_syms(m[3])...) + rt = try + @static if isdefined(Core.Compiler, :NativeInterpreter) + Core.Compiler.typeinf_type(Core.Compiler.NativeInterpreter(), m[3], m[3].sig, sparams) + else + Core.Compiler.typeinf_type(m[3], m[3].sig, sparams, Core.Compiler.Params(world)) + end + catch e + Any + end + else + rt = Any + end + file = maybe_fix_path(String(m[3].file)) + MS = MethodStore(m[3].name, nameof(m[3].module), file, m[3].line, [], Symbol[], FakeTypeName(rt)) + # Get signature + sig = Base.unwrap_unionall(m[1]) + argnames = getargnames(m[3]) + for i = 2:m[3].nargs + push!(MS.sig, argnames[i] => FakeTypeName(sig.parameters[i])) + end + kws = getkws(m[3]) + if !isempty(kws) + push!(ind_of_method_w_kws, i) + end + for kw in kws + push!(MS.kws, kw) + end + push!(ms, (m[3].module, MS)) + i += 1 + end + + # Go back and add kws to methods defined in the same place as others with kws. + for i in ind_of_method_w_kws + for mj in ms + if mj[2].file == ms[i][2].file && mj[2].line == ms[i][2].line && isempty(mj[2].kws) + for kw in ms[i][2].kws + push!(mj[2].kws, kw) + end + end + end + end + + func_vr = VarRef(VarRef(parentmodule(f)), name) + for m in ms + mvr = VarRef(m[1]) + modstore = _lookup(mvr, env) + modstore === nothing && continue + + if !haskey(modstore, name) + modstore[name] = FunctionStore(VarRef(mvr, name), MethodStore[m[2]], "", func_vr, false) + elseif !(modstore[name] isa DataTypeStore || modstore[name] isa FunctionStore) + modstore[name] = FunctionStore(VarRef(mvr, name), MethodStore[m[2]], "", func_vr, false) + else + push!(modstore[name].methods, m[2]) + end + end + return ms +end + +getargnames(m::Method) = Base.method_argnames(m) +@static if length(first(methods(Base.kwarg_decl)).sig.parameters) == 2 + getkws = Base.kwarg_decl +else + function getkws(m::Method) + sig = Base.unwrap_unionall(m.sig) + length(sig.parameters) == 0 && return [] + sig.parameters[1] isa Union && return [] + !isdefined(Base.unwrap_unionall(sig.parameters[1]), :name) && return [] + fname = Base.unwrap_unionall(sig.parameters[1]).name + if isdefined(fname.mt, :kwsorter) + Base.kwarg_decl(m, typeof(fname.mt.kwsorter)) + else + [] + end + end +end + +function apply_to_everything(f, m = nothing, visited = Base.IdSet{Module}()) + if m isa Module + push!(visited, m) + for s in unsorted_names(m, all = true, imported = true) + (!isdefined(m, s) || s == nameof(m)) && continue + x = getfield(m, s) + f(x) + if x isa Module && !in(x, visited) + apply_to_everything(f, x, visited) + end + end + else + for m in Base.loaded_modules_array() + in(m, visited) || apply_to_everything(f, m, visited) + end + end +end + + + +function oneverything(f, m = nothing, visited = Base.IdSet{Module}()) + if m isa Module + push!(visited, m) + state = nothing + for s in unsorted_names(m, all = true, imported = true) + !isdefined(m, s) && continue + x = getfield(m, s) + state = f(m, s, x, state) + if x isa Module && !in(x, visited) + oneverything(f, x, visited) + end + end + else + for m in Base.loaded_modules_array() + in(m, visited) || oneverything(f, m, visited) + end + end +end + +const _global_symbol_cache_by_mod = IdDict{Module,Base.IdSet{Symbol}}() +function build_namecache(m, s, @nospecialize(x), state::Union{Base.IdSet{Symbol},Nothing} = nothing) + if state === nothing + state = get(_global_symbol_cache_by_mod, m, nothing) + if state === nothing + state = _global_symbol_cache_by_mod[m] = Base.IdSet{Symbol}() + end + end + push!(state, s) +end + +function getnames(m::Module) + cache = get(_global_symbol_cache_by_mod, m, nothing) + if cache === nothing + oneverything(build_namecache, m) + cache = _global_symbol_cache_by_mod[m] + end + return cache +end + +function allmodulenames() + symbols = Base.IdSet{Symbol}() + oneverything((m, s, x, state) -> (x isa Module && push!(symbols, s); return state)) + return symbols +end + +function allthingswithmethods() + symbols = Base.IdSet{Any}() + oneverything(function (m, s, x, state) + if !Base.isvarargtype(x) && !isempty(methodlist(x)) + push!(symbols, x) + end + return state + end) + return symbols +end + +function allmethods() + ms = Method[] + oneverything(function (m, s, x, state) + if !Base.isvarargtype(x) && !isempty(methodlist(x)) + append!(ms, methodlist(x)) + end + return state + end) + return ms +end + +usedby(outer, inner) = outer !== inner && isdefined(outer, nameof(inner)) && getproperty(outer, nameof(inner)) === inner && all(isdefined(outer, name) || !isdefined(inner, name) for name in unsorted_names(inner)) +istoplevelmodule(m) = parentmodule(m) === m || parentmodule(m) === Main + +function getmoduletree(m::Module, amn, visited = Base.IdSet{Module}()) + push!(visited, m) + cache = ModuleStore(m) + for s in unsorted_names(m, all = true, imported = true) + !isdefined(m, s) && continue + x = getfield(m, s) + if x isa Module + if istoplevelmodule(x) + cache[s] = VarRef(x) + elseif m === parentmodule(x) + cache[s] = getmoduletree(x, amn, visited) + else + cache[s] = VarRef(x) + end + end + end + for n in amn + if n !== nameof(m) && isdefined(m, n) + x = getfield(m, n) + if x isa Module + if !haskey(cache, n) + cache[n] = VarRef(x) + end + if x !== Main && usedby(m, x) + push!(cache.used_modules, n) + end + end + end + end + cache +end + +function getenvtree(names = nothing) + amn = allmodulenames() + EnvStore(nameof(m) => getmoduletree(m, amn) for m in Base.loaded_modules_array() if names === nothing || nameof(m) in names) +end + +# faster and more correct split_module_names +all_names(m) = all_names(m, x -> isdefined(m, x)) +function all_names(m, pred, symbols = Set(Symbol[]), seen = Set(Module[])) + push!(seen, m) + ns = unsorted_names(m; all = true, imported = false) + for n in ns + isdefined(m, n) || continue + Base.isdeprecated(m, n) && continue + val = getfield(m, n) + if val isa Module && !(val in seen) + all_names(val, pred, symbols, seen) + end + if pred(n) + push!(symbols, n) + end + end + symbols +end + +function symbols(env::EnvStore, m::Union{Module,Nothing} = nothing, allnames::Base.IdSet{Symbol} = getallns(), visited = Base.IdSet{Module}(); get_return_type = false) + if m isa Module + cache = _lookup(VarRef(m), env, true) + cache === nothing && return + push!(visited, m) + ns = all_names(m) + for s in ns + !isdefined(m, s) && continue + x = getfield(m, s) + if Base.unwrap_unionall(x) isa DataType # Unions aren't handled here. + if parentmodule((x)) === m + cache[s] = DataTypeStore(x, s, m, s in getnames(m)) + cache_methods(x, s, env, get_return_type) + elseif nameof(x) !== s + # This needs some finessing. + cache[s] = DataTypeStore(x, s, m, s in getnames(m)) + ms = cache_methods(x, s, env, get_return_type) + # A slightly difficult case. `s` is probably a shadow binding of `x` but we should store the methods nonetheless. + # Example: DataFrames.Not points to InvertedIndices.InvertedIndex + for m in ms + push!(cache[s].methods, m[2]) + end + else + # These are imported variables that are reexported. + cache[s] = VarRef(VarRef(parentmodule(x)), nameof(x)) + end + elseif x isa Function + if parentmodule(x) === m || (x isa Core.IntrinsicFunction && m === Core.Intrinsics) + cache[s] = FunctionStore(x, s, m, s in getnames(m)) + cache_methods(x, s, env, get_return_type) + elseif !haskey(cache, s) + # This will be replaced at a later point by a FunctionStore if methods for `x` are defined within `m`. + if x isa Core.IntrinsicFunction + cache[s] = VarRef(VarRef(Core.Intrinsics), nameof(x)) + else + cache[s] = VarRef(VarRef(parentmodule(x)), nameof(x)) + end + elseif !((cache[s] isa FunctionStore || cache[s] isa DataTypeStore) && !isempty(cache[s].methods)) + # These are imported variables that are reexported. + # We don't want to remove Func/DT stores that have methods (these will be specific to the module) + if x isa Core.IntrinsicFunction + cache[s] = VarRef(VarRef(Core.Intrinsics), nameof(x)) + else + cache[s] = VarRef(VarRef(parentmodule(x)), nameof(x)) + end + end + elseif x isa Module + if x === m + cache[s] = VarRef(x) + elseif parentmodule(x) === m + symbols(env, x, allnames, visited, get_return_type = get_return_type) + else + cache[s] = VarRef(x) + end + else + cache[s] = GenericStore(VarRef(VarRef(m), s), FakeTypeName(typeof(x)), _doc(Base.Docs.Binding(m, s)), s in getnames(m)) + end + end + else + for m in Base.loaded_modules_array() + in(m, visited) || symbols(env, m, allnames, visited, get_return_type = get_return_type) + end + end +end + + +function load_core(; get_return_type = false) + c = Pkg.Types.Context() + cache = getenvtree([:Core,:Base]) + symbols(cache, get_return_type = get_return_type) + cache[:Main] = ModuleStore(VarRef(nothing, :Main), Dict(), "", true, [], []) + + # This is wrong. Every module contains it's own include function. + push!(cache[:Base].exportednames, :include) + let f = cache[:Base][:include] + if haskey(cache[:Base][:MainInclude], :include) + cache[:Base][:include] = FunctionStore(f.name, cache[:Base][:MainInclude][:include].methods, f.doc, f.extends, true) + else + m1 = first(f.methods) + push!(f.methods, MethodStore( + m1.name, + m1.mod, + m1.file, + m1.line, + Pair{Any,Any}[ + :x => SymbolServer.FakeTypeName(SymbolServer.VarRef(SymbolServer.VarRef(nothing, :Core), :AbstractString), Any[]) + ], + [], + m1.rt + )) + end + end + + cache[:Base][Symbol("@.")] = cache[:Base][Symbol("@__dot__")] + cache[:Core][:Main] = GenericStore(VarRef(nothing, :Main), FakeTypeName(Module), _doc(Base.Docs.Binding(Main, :Main)), true) + # Add built-ins + builtins = Symbol[nameof(getfield(Core, n).instance) for n in unsorted_names(Core, all = true) if isdefined(Core, n) && getfield(Core, n) isa DataType && isdefined(getfield(Core, n), :instance) && getfield(Core, n).instance isa Core.Builtin] + cnames = unsorted_names(Core) + for f in builtins + if !haskey(cache[:Core], f) + cache[:Core][f] = FunctionStore(getfield(Core, Symbol(f)), Symbol(f), Core, Symbol(f) in cnames) + end + end + haskey(cache[:Core], :_typevar) && push!(cache[:Core][:_typevar].methods, MethodStore(:_typevar, :Core, "built-in", 0, [:n => FakeTypeName(Symbol), :lb => FakeTypeName(Any), :ub => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:_apply].methods, MethodStore(:_apply, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + haskey(cache[:Core].vals, :_apply_iterate) && push!(cache[:Core][:_apply_iterate].methods, MethodStore(:_apply_iterate, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + if isdefined(Core, :_call_latest) + push!(cache[:Core][:_call_latest].methods, MethodStore(:_call_latest, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:_call_in_world].methods, MethodStore(:_call_in_world, :Core, "built-in", 0, [:world => FakeTypeName(UInt), :f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + else + if isdefined(Core, :_apply_in_world) + push!(cache[:Core][:_apply_in_world].methods, MethodStore(:_apply_in_world, :Core, "built-in", 0, [:world => FakeTypeName(UInt), :f => FakeTypeName(Function), :args => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + end + push!(cache[:Core][:_apply_latest].methods, MethodStore(:_apply_latest, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + end + push!(cache[:Core][:_apply_pure].methods, MethodStore(:_apply_pure, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:_expr].methods, MethodStore(:_expr, :Core, "built-in", 0, [:head => FakeTypeName(Symbol), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Expr))) + haskey(cache[:Core].vals, :_typevar) && push!(cache[:Core][:_typevar].methods, MethodStore(:_typevar, :Core, "built-in", 0, [:name => FakeTypeName(Symbol), :lb => FakeTypeName(Any), :ub => FakeTypeName(Any)], Symbol[], FakeTypeName(TypeVar))) + push!(cache[:Core][:applicable].methods, MethodStore(:applicable, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Bool))) + push!(cache[:Core][:apply_type].methods, MethodStore(:apply_type, :Core, "built-in", 0, [:T => FakeTypeName(UnionAll), :types => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(UnionAll))) + push!(cache[:Core][:arrayref].methods, MethodStore(:arrayref, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any), :c => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:arrayset].methods, MethodStore(:arrayset, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any), :c => FakeTypeName(Any), :d => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:arraysize].methods, MethodStore(:arraysize, :Core, "built-in", 0, [:a => FakeTypeName(Array), :i => FakeTypeName(Int)], Symbol[], FakeTypeName(Int))) + haskey(cache[:Core], :const_arrayref) && push!(cache[:Core][:const_arrayref].methods, MethodStore(:const_arrayref, :Core, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:fieldtype].methods, MethodStore(:fieldtype, :Core, "built-in", 0, [:t => FakeTypeName(DataType), :field => FakeTypeName(Symbol)], Symbol[], FakeTypeName(Type{T} where T))) + push!(cache[:Core][:getfield].methods, MethodStore(:setfield, :Core, "built-in", 0, [:object => FakeTypeName(Any), :item => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:ifelse].methods, MethodStore(:ifelse, :Core, "built-in", 0, [:condition => FakeTypeName(Bool), :x => FakeTypeName(Any), :y => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:invoke].methods, MethodStore(:invoke, :Core, "built-in", 0, [:f => FakeTypeName(Function), :x => FakeTypeName(Any), :argtypes => FakeTypeName(Type{T} where T) , :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:isa].methods, MethodStore(:isa, :Core, "built-in", 0, [:a => FakeTypeName(Any), :T => FakeTypeName(Type{T} where T)], Symbol[], FakeTypeName(Bool))) + push!(cache[:Core][:isdefined].methods, MethodStore(:getproperty, :Core, "built-in", 0, [:value => FakeTypeName(Any), :field => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:nfields].methods, MethodStore(:nfields, :Core, "built-in", 0, [:x => FakeTypeName(Any)], Symbol[], FakeTypeName(Int))) + push!(cache[:Core][:setfield!].methods, MethodStore(:setfield!, :Core, "built-in", 0, [:value => FakeTypeName(Any), :name => FakeTypeName(Symbol), :x => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:sizeof].methods, MethodStore(:sizeof, :Core, "built-in", 0, [:obj => FakeTypeName(Any)], Symbol[], FakeTypeName(Int))) + push!(cache[:Core][:svec].methods, MethodStore(:svec, :Core, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:throw].methods, MethodStore(:throw, :Core, "built-in", 0, [:e => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:tuple].methods, MethodStore(:tuple, :Core, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:typeassert].methods, MethodStore(:typeassert, :Core, "built-in", 0, [:x => FakeTypeName(Any), :T => FakeTypeName(Type{T} where T)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:typeof].methods, MethodStore(:typeof, :Core, "built-in", 0, [:x => FakeTypeName(Any)], Symbol[], FakeTypeName(Type{T} where T))) + + push!(cache[:Core][:getproperty].methods, MethodStore(:getproperty, :Core, "built-in", 0, [:value => FakeTypeName(Any), :name => FakeTypeName(Symbol)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:setproperty!].methods, MethodStore(:setproperty!, :Core, "built-in", 0, [:value => FakeTypeName(Any), :name => FakeTypeName(Symbol), :x => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:setproperty!].methods, MethodStore(:setproperty!, :Core, "built-in", 0, [:value => FakeTypeName(Any), :name => FakeTypeName(Symbol), :x => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + haskey(cache[:Core], :_abstracttype) && push!(cache[:Core][:_abstracttype].methods, MethodStore(:_abstracttype, :Core, "built-in", 0, [:m => FakeTypeName(Module), :x => FakeTypeName(Symbol), :p => FakeTypeName(Core.SimpleVector)], Symbol[], FakeTypeName(Any))) + haskey(cache[:Core], :_primitivetype) && push!(cache[:Core][:_primitivetype].methods, MethodStore(:_primitivetype, :Core, "built-in", 0, [:m => FakeTypeName(Module), :x => FakeTypeName(Symbol), :p => FakeTypeName(Core.SimpleVector), :n => FakeTypeName(Core.Int)], Symbol[], FakeTypeName(Any))) + haskey(cache[:Core], :_equiv_typedef) && push!(cache[:Core][:_equiv_typedef].methods, MethodStore(:_equiv_typedef, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + haskey(cache[:Core], :_setsuper!) && push!(cache[:Core][:_setsuper!].methods, MethodStore(:_setsuper!, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + haskey(cache[:Core], :_structtype) && push!(cache[:Core][:_structtype].methods, MethodStore(:_structtype, :Core, "built-in", 0, [:m => FakeTypeName(Module), :x => FakeTypeName(Symbol), :p => FakeTypeName(Core.SimpleVector), :fields => FakeTypeName(Core.SimpleVector), :mut => FakeTypeName(Bool), :z => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + haskey(cache[:Core], :_typebody) && push!(cache[:Core][:_typebody!].methods, MethodStore(:_typebody!, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:(===)].methods, MethodStore(:(===), :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any)], Symbol[], FakeTypeName(Any))) + push!(cache[:Core][:(<:)].methods, MethodStore(:(<:), :Core, "built-in", 0, [:a => FakeTypeName(Type{T} where T), :b => FakeTypeName(Type{T} where T)], Symbol[], FakeTypeName(Any))) + # Add unspecified methods for Intrinsics, working out the actual methods will need to be done by hand? + for n in names(Core.Intrinsics) + if getfield(Core.Intrinsics, n) isa Core.IntrinsicFunction + push!(cache[:Core][:Intrinsics][n].methods, MethodStore(n, :Intrinsics, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + :args => FakeTypeName(Vararg{Any}) + end + end + + for bi in builtins + if haskey(cache[:Core], bi) && isempty(cache[:Core][bi].methods) + # Add at least one arbitrary method for anything left over + push!(cache[:Core][bi].methods, MethodStore(bi, :none, "built-in", 0, [:x => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any))) + end + end + + cache[:Core][:ccall] = FunctionStore(VarRef(VarRef(Core), :ccall), + MethodStore[ + MethodStore(:ccall, :Core, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)) # General method - should be fixed + ], + "`ccall((function_name, library), returntype, (argtype1, ...), argvalue1, ...)`\n`ccall(function_name, returntype, (argtype1, ...), argvalue1, ...)`\n`ccall(function_pointer, returntype, (argtype1, ...), argvalue1, ...)`\n\nCall a function in a C-exported shared library, specified by the tuple (`function_name`, `library`), where each component is either a string or symbol. Instead of specifying a library, one\ncan also use a `function_name` symbol or string, which is resolved in the current process. Alternatively, `ccall` may also be used to call a function pointer `function_pointer`, such as one\nreturned by `dlsym`.\n\nNote that the argument type tuple must be a literal tuple, and not a tuple-valued variable or expression.\n\nEach `argvalue` to the `ccall` will be converted to the corresponding `argtype`, by automatic insertion of calls to `unsafe_convert(argtype, cconvert(argtype, argvalue))`. (See also the documentation for `unsafe_convert` and `cconvert` for further details.) In most cases, this simply results in a call to `convert(argtype, argvalue)`.", + VarRef(VarRef(Core), :ccall), + true) + push!(cache[:Core].exportednames, :ccall) + cache[:Core][Symbol("@__doc__")] = FunctionStore(VarRef(VarRef(Core), Symbol("@__doc__")), [], "", VarRef(VarRef(Core), Symbol("@__doc__")), true) + cache_methods(getfield(Core, Symbol("@__doc__")), Symbol("@__doc__"), cache, false) + # Accounts for the dd situation where Base.rand only has methods from Random which doesn't appear to be explicitly used. + # append!(cache[:Base][:rand].methods, cache_methods(Base.rand, cache)) + for m in cache_methods(Base.rand, :rand, cache, get_return_type) + push!(cache[:Base][:rand].methods, m[2]) + end + for m in cache_methods(Base.randn, :randn, cache, get_return_type) + push!(cache[:Base][:randn].methods, m[2]) + end + + # Intrinsics + cache[:Core][:add_int] = VarRef(VarRef(VarRef(nothing, :Core), :Intrinsics), :add_int) + cache[:Core][:sle_int] = VarRef(VarRef(VarRef(nothing, :Core), :Intrinsics), :sle_int) + return cache +end + + +function collect_extended_methods(depot::EnvStore, extendeds = Dict{VarRef,Vector{VarRef}}()) + for m in depot + collect_extended_methods(m[2], extendeds, m[2].name) + end + extendeds +end + +function collect_extended_methods(mod::ModuleStore, extendeds, mname) + for (n, v) in mod.vals + if (v isa FunctionStore) && v.extends != v.name + haskey(extendeds, v.extends) ? push!(extendeds[v.extends], mname) : (extendeds[v.extends] = VarRef[v.extends.parent, mname]) + elseif v isa ModuleStore + collect_extended_methods(v, extendeds, v.name) + end + end +end + +getallns() = let allns = Base.IdSet{Symbol}(); oneverything((m, s, x, state) -> push!(allns, s)); allns end + +""" + split_module_names(m::Module, allns) + +Return two lists of names accessible from calling `getfield(m, somename)`. The first +contains those symbols returned by `Base.names(m, all = true)`. The second contains +all others, including imported symbols and those introduced by the `using` of modules. +""" +function split_module_names(m::Module, allns) + internal_names = getnames(m) + availablenames = Set{Symbol}([s for s in allns if isdefined(m, s)]) + # usinged_names = Set{Symbol}() + + for n in availablenames + if (n in internal_names) + pop!(availablenames, n) + end + end + allms = get_all_modules() + for u in get_used_modules(m, allms) + for n in unsorted_names(u) + if n in availablenames + pop!(availablenames, n) + # push!(usinged_names, pop!(availablenames, n)) + end + end + end + internal_names, availablenames +end + +get_all_modules() = let allms = Base.IdSet{Module}(); apply_to_everything(x -> if x isa Module push!(allms, x) end); allms end +get_used_modules(M, allms = get_all_modules()) = [m for m in allms if usedby(M, m)] diff --git a/src/SymbolServer/utils.jl b/src/SymbolServer/utils.jl new file mode 100644 index 0000000..4efdb7e --- /dev/null +++ b/src/SymbolServer/utils.jl @@ -0,0 +1,688 @@ +using Pkg + +@static if VERSION < v"1.1" + const PackageEntry = Vector{Dict{String,Any}} +else + using Pkg.Types: PackageEntry +end + +@static if isdefined(Base, :parsed_toml) + parsed_toml(args...) = Base.parsed_toml(args...) +else + parsed_toml(file) = Pkg.TOML.parsefile(file) +end + +""" + manifest(c::Pkg.Types.Context) +Retrieves the UUID -> PackageEntry map from the manifest of a Context. +""" +function manifest(c::Pkg.Types.Context) + m = c.env.manifest + if m isa Dict + return m + else + return m.deps + end +end + +""" + read_manifest(manifest_filename) + +Read the manifest from the path and return the UUID -> PackageEntry map. +If the file can't be read, return `nothing`. +""" +function read_manifest(manifest_filename) + try + m = Pkg.API.read_manifest(manifest_filename) + if m isa Dict + return m + else + return m.deps + end + catch err + @warn "Could not load manifest." exception=(err, catch_backtrace()) + return nothing + end +end + +""" + project(c::Pkg.Types.Context) +Retrieves the project of a Context. +""" +project(c::Pkg.Types.Context) = c.env.project + +""" + isinproject(context, package::Union{String,UUID}) +Checks whether a package is in the dependencies of a given context, e.g. is directly loadable. +""" +function isinproject end + +""" + isinmanifest(context, package::Union{String,UUID}) +Checks whether a package is in the manifest of a given context, e.g. is either directly loadable or is a dependency of an loadable package. +""" +function isinmanifest end + +@static if VERSION < v"1.1" + isinmanifest(context::Pkg.Types.Context, module_name::String) = module_name in keys(manifest(context)) + isinmanifest(context::Pkg.Types.Context, uuid::UUID) = any(get(p[1], "uuid", "") == string(uuid) for (u, p) in manifest(context)) + isinmanifest(manifest::Dict{String,Any}, uuid::AbstractString) = any(get(p[1], "uuid", "") == uuid for (u, p) in manifest) + isinmanifest(manifest::Dict{String,Any}, uuid::UUID) = isinmanifest(manifest, string(uuid)) + + isinproject(context::Pkg.Types.Context, package_name::String) = haskey(deps(project(context)), package_name) + isinproject(context::Pkg.Types.Context, package_uuid::UUID) = any(u == package_uuid for (n, u) in deps(project(context))) + + function packageuuid(c::Pkg.Types.Context, name::String) + for pkg in manifest(c) + if first(pkg) == name + return UUID(last(pkg)[1]["uuid"]) + end + end + end + packageuuid(pkg::Pair{Any,Any}) = last(pkg) isa String ? UUID(last(pkg)) : UUID(first(last(pkg))["uuid"]) + packageuuid(pkg::Pair{String,Any}) = last(pkg) isa String ? UUID(last(pkg)) : UUID(first(last(pkg))["uuid"]) + + packagename(pkg::Pair{String,Any})::String = first(pkg) + function packagename(c::Pkg.Types.Context, uuid) + for (n, p) in manifest(c) + if get(first(p), "uuid", "") == string(uuid) + return n + end + end + return nothing + end + function packagename(manifest::Dict{String,Any}, uuid::String) + for (n, p) in manifest + if get(first(p), "uuid", "") == string(uuid) + return n + end + end + return nothing + end + packagename(manifest::Dict{String,Any}, uuid::UUID) = packagename(manifest, string(uuid)) + + function deps(uuid::UUID, c::Pkg.Types.Context) + if any(p[1]["uuid"] == string(uuid) for (n, p) in manifest(c)) + return manifest(c)[string(uuid)][1].deps + else + return Dict{Any,Any}() + end + end + deps(d::Dict{String,Any}) = get(d, "deps", Dict{String,Any}()) + deps(pe::PackageEntry) = get(pe[1], "deps", Dict{String,Any}()) + path(pe::PackageEntry) = get(pe[1], "path", nothing) + version(pe::PackageEntry) = get(pe[1], "version", nothing) + tree_hash(pe) = get(pe[1], "git-tree-sha1", nothing) + + frommanifest(c::Pkg.Types.Context, uuid) = frommanifest(manifest(c), uuid) + + function frommanifest(manifest::Dict{String,Any}, uuid) + for p in values(manifest) + if get(first(p), "uuid", "") == string(uuid) + return (p) + end + end + return nothing + end + is_package_deved(manifest, uuid) = get(first([p[2][1] for p in manifest if get(p[2][1], "uuid", "") == string(uuid)]), "path", "") != "" +else + isinmanifest(context::Pkg.Types.Context, module_name::String) = any(p.name == module_name for (u, p) in manifest(context)) + isinmanifest(context::Pkg.Types.Context, uuid::UUID) = haskey(manifest(context), uuid) + isinmanifest(manifest::Dict{UUID,PackageEntry}, uuid::UUID) = haskey(manifest, uuid) + + isinproject(context::Pkg.Types.Context, package_name::String) = haskey(deps(project(context)), package_name) + isinproject(context::Pkg.Types.Context, package_uuid::UUID) = any(u == package_uuid for (n, u) in deps(project(context))) + + function packageuuid(c::Pkg.Types.Context, name::String) + for pkg in manifest(c) + if last(pkg).name == name + return first(pkg) + end + end + end + packageuuid(pkg::Pair{String,UUID}) = last(pkg) + packageuuid(pkg::Pair{UUID,PackageEntry}) = first(pkg) + + packagename(pkg::Pair{UUID,PackageEntry})::Union{Nothing,String} = last(pkg).name + packagename(c::Pkg.Types.Context, uuid::UUID) = manifest(c)[uuid].name + packagename(manifest::Dict{UUID,PackageEntry}, uuid::UUID) = manifest[uuid].name + + function deps(uuid::UUID, c::Pkg.Types.Context) + if haskey(manifest(c), uuid) + return deps(manifest(c)[uuid]) + else + return Dict{String,Base.UUID}() + end + end + deps(pe::PackageEntry) = pe.deps + deps(proj::Pkg.Types.Project) = proj.deps + deps(pkg::Pair{String,UUID}, c::Pkg.Types.Context) = deps(packageuuid(pkg), c) + path(pe::PackageEntry) = pe.path + version(pe::PackageEntry) = pe.version + version(pe::Pair{UUID,PackageEntry}) = last(pe).version + frommanifest(c::Pkg.Types.Context, uuid) = manifest(c)[uuid] + frommanifest(manifest::Dict{UUID,PackageEntry}, uuid) = manifest[uuid] + tree_hash(pkg::Pair{UUID,PackageEntry}) = tree_hash(last(pkg)) + + @static if VERSION >= v"1.3" + tree_hash(pe::PackageEntry) = pe.tree_hash + else + tree_hash(pe::PackageEntry) = (pe.other === nothing ? nothing : get(pe.other, "git-tree-sha1", nothing)) + end + + is_package_deved(manifest, uuid) = manifest[uuid].path !== nothing +end + +function sha2_256_dir(path, sha=zeros(UInt8, 32)) + (uperm(path) & 0x04) != 0x04 && return + startswith(path, ".") && return + if isfile(path) && endswith(path, ".jl") + s1 = open(path) do f + sha2_256(f) + end + sha .+= s1 + elseif isdir(path) + for f in readdir(path) + sha = sha2_256_dir(joinpath(path, f), sha) + end + end + return sha +end + +function sha_pkg(manifest_dir::AbstractString, pe::PackageEntry) + relpath = path(pe) + isa(relpath, String) || return nothing + src_path = normpath(joinpath(manifest_dir, relpath, "src")) + return isdir(src_path) ? sha2_256_dir(src_path) : nothing +end + +function _doc(binding::Base.Docs.Binding) + try + sig = Union{} + if Base.Docs.defined(binding) + result = Base.Docs.getdoc(Base.Docs.resolve(binding), sig) + result === nothing || return string(result) + end + results, groups = Base.Docs.DocStr[], Base.Docs.MultiDoc[] + # Lookup `binding` and `sig` for matches in all modules of the docsystem. + for mod in Base.Docs.modules + dict = Base.Docs.meta(mod)::IdDict{Any,Any} + if haskey(dict, binding) + multidoc = dict[binding] + push!(groups, multidoc) + for msig in multidoc.order + sig <: msig && push!(results, multidoc.docs[msig]) + end + end + end + if isempty(results) + for group in groups, each in group.order + push!(results, group.docs[each]) + end + end + md = try + Base.Docs.catdoc(map(Base.Docs.parsedoc, results)...) + catch + nothing + end + return md === nothing ? "" : string(md) + catch + return "" + end +end + +_lookup(vr::FakeUnion, depot::EnvStore, cont=false) = nothing +_lookup(vr::FakeTypeName, depot::EnvStore, cont=false) = _lookup(vr.name, depot, cont) +_lookup(vr::FakeUnionAll, depot::EnvStore, cont=false) = _lookup(vr.body, depot, cont) +function _lookup(vr::VarRef, depot::EnvStore, cont=false) + if vr.parent === nothing + if haskey(depot, vr.name) + val = depot[vr.name] + if cont && val isa VarRef + return _lookup(val, depot, cont) + else + return val + end + else + return nothing + end + else + par = _lookup(vr.parent, depot, cont) + if par !== nothing && par isa ModuleStore && haskey(par, vr.name) + val = par[vr.name] + if cont && val isa VarRef + return _lookup(val, depot, cont) + else + return val + end + else + return nothing + end + end +end + +maybe_lookup(x, env) = x isa VarRef ? _lookup(x, env, true) : x + +""" + maybe_getfield(k::Symbol , m::ModuleStore, server) + +Try to get `k` from `m`. This includes: unexported variables, and variables +exported by modules used within `m`. +""" +function maybe_getfield(k::Symbol, m::ModuleStore, envstore) + if haskey(m.vals, k) + return m.vals[k] + else + for v in m.used_modules + !haskey(m.vals, v) && continue + submod = m.vals[v] + if submod isa ModuleStore && k in submod.exportednames && haskey(submod.vals, k) + return submod.vals[k] + elseif submod isa VarRef + submod = _lookup(submod, envstore, true) + if submod isa ModuleStore && k in submod.exportednames && haskey(submod.vals, k) + return submod.vals[k] + end + end + end + end +end + +function issubmodof(m::Module, M::Module) + if m == M + return true + elseif parentmodule(m) === m + return false + elseif parentmodule(m) == M + return true + else + return issubmodof(parentmodule(m), M) + end +end + +function Base.print(io::IO, f::FunctionStore) + println(io, f.name, " is a Function.") + nm = length(f.methods) + println(io, "# $nm method", nm == 1 ? "" : "s", " for function ", f.name) + for i = 1:nm + print(io, "[$i] ") + println(io, f.methods[i]) + end +end + +const JULIA_DIR = normpath(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia")) + +function Base.print(io::IO, m::MethodStore) + print(io, m.name, "(") + for i = 1:length(m.sig) + if m.sig[i][1] != Symbol("#unused#") + print(io, m.sig[i][1]) + end + print(io, "::", m.sig[i][2]) + i != length(m.sig) && print(io, ", ") + end + print(io, ")") + path = replace(m.file, JULIA_DIR => "") + print(io, " in ", m.mod, " at ", path, ':', m.line) +end + +function Base.print(io::IO, t::DataTypeStore) + print(io, t.name, " <: ", t.super) + for i = 1:length(t.fieldnames) + print(io, "\n ", t.fieldnames[i], "::", t.types[i]) + end +end + +Base.print(io::IO, m::ModuleStore) = print(io, m.name) +Base.print(io::IO, x::GenericStore) = print(io, x.name, "::", x.typ) + +extends_methods(f) = false +extends_methods(f::FunctionStore) = f.name != f.extends +get_top_module(vr::VarRef) = vr.parent === nothing ? vr.name : get_top_module(vr.parent) + +# Sorting is the main performance of calling `names` +unsorted_names(m::Module; all::Bool=false, imported::Bool=false) = + ccall(:jl_module_names, Array{Symbol,1}, (Any, Cint, Cint), m, all, imported) + +## recursive_copy +# +# `deepcopy` is reliable but incredibly slow. Its slowness comes from two factors: +# - generically iterating over, e.g., `fieldnames(typeof(x))` rather than having a method +# optimized for each struct type +# - its care to protect against circular depenency graphs +# When you don't need to worry about cycles, you can do much better by defining your own function. + +recursive_copy(x) = deepcopy(x) + +recursive_copy(::Nothing) = nothing + +recursive_copy(s::Symbol) = s + +recursive_copy(c::Char) = c + +recursive_copy(str::String) = str + +recursive_copy(x::Number) = x + +recursive_copy(p::Pair) = typeof(p)(recursive_copy(p.first), recursive_copy(p.second)) + +recursive_copy(A::Array) = eltype(A)[recursive_copy(a) for a in A] + +recursive_copy(d::Dict) = typeof(d)(recursive_copy(p) for p in d) + + +recursive_copy(ref::VarRef) = VarRef(recursive_copy(ref.parent), ref.name) + +recursive_copy(tn::FakeTypeName) = FakeTypeName(recursive_copy(tn.name), recursive_copy(tn.parameters)) + +recursive_copy(tb::FakeTypeofBottom) = tb + +recursive_copy(u::FakeUnion) = FakeUnion(recursive_copy(u.a), recursive_copy(u.b)) + +recursive_copy(tv::FakeTypeVar) = FakeTypeVar(tv.name, recursive_copy(tv.lb), recursive_copy(tv.ub)) + +recursive_copy(ua::FakeUnionAll) = FakeUnionAll(recursive_copy(ua.var), recursive_copy(ua.body)) + +@static if !(Vararg isa Type) + function recursive_copy(va::FakeTypeofVararg) + if isdefined(va, :N) + FakeTypeofVararg(recursive_copy(va.T), va.N) + elseif isdefined(va, :T) + FakeTypeofVararg(recursive_copy(va.T)) + else + FakeTypeofVararg() + end + end +end + +recursive_copy(m::ModuleStore) = ModuleStore(recursive_copy(m.name), recursive_copy(m.vals), m.doc, + m.exported, copy(m.exportednames), copy(m.used_modules)) + +recursive_copy(p::Package) = Package(p.name, + recursive_copy(p.val), + p.uuid, + recursive_copy(p.sha)) + +recursive_copy(ms::MethodStore) = MethodStore(ms.name, + ms.mod, + ms.file, + ms.line, + recursive_copy(ms.sig), + copy(ms.kws), + recursive_copy(ms.rt)) + +recursive_copy(dts::DataTypeStore) = DataTypeStore(recursive_copy(dts.name), + recursive_copy(dts.super), + recursive_copy(dts.parameters), + recursive_copy(dts.types), + recursive_copy(dts.fieldnames), + recursive_copy(dts.methods), + dts.doc, + dts.exported) + +recursive_copy(fs::FunctionStore) = FunctionStore(recursive_copy(fs.name), + recursive_copy(fs.methods), + fs.doc, + recursive_copy(fs.extends), + fs.exported) + +recursive_copy(gs::GenericStore) = GenericStore(recursive_copy(gs.name), + recursive_copy(gs.typ), + gs.doc, + gs.exported) + + +# Tools for modifying source location +# env = getenvtree([:somepackage]) +# symbols(env, somepackage) +# m = env[:somepackage] +# To strip actual src path: +# modify_dirs(m, f -> modify_dir(f, pkg_src_dir(somepackage), "PLACEHOLDER")) +# To replace the placeholder: +# modify_dirs(m, f -> modify_dir(f, "PLACEHOLDER", new_src_dir)) +function modify_dirs(m::ModuleStore, f) + for (k, v) in m.vals + if v isa FunctionStore + m.vals[k] = FunctionStore(v.name, MethodStore[MethodStore(m.name, m.mod, f(m.file), m.line, m.sig, m.kws, m.rt) for m in v.methods], v.doc, v.extends, v.exported) + elseif v isa DataTypeStore + m.vals[k] = DataTypeStore(v.name, v.super, v.parameters, v.types, v.fieldnames, MethodStore[MethodStore(m.name, m.mod, f(m.file), m.line, m.sig, m.kws, m.rt) for m in v.methods], v.doc, v.exported) + elseif v isa ModuleStore + modify_dirs(v, f) + end + end +end + +pkg_src_dir(m::Module) = dirname(pathof(m)) + +# replace s1 with s2 at the start of a string +function modify_dir(f, s1, s2) + # @assert startswith(f, s1) + # Removed assertion because of Enums issue + replace(f, s1 => s2) +end + + +# tools to retrieve cache from the cloud + +function get_file_from_cloud(manifest, uuid, environment_path, depot_dir, cache_dir="../cache", download_dir="../downloads/", symbolcache_upstream="https://www.julia-vscode.org/symbolcache") + paths = get_cache_path(manifest, uuid) + name = packagename(manifest, uuid) + link = string(first(splitext(join([symbolcache_upstream, "store/v1/packages", paths...], '/'))), ".tar.gz") + + dest_filepath = joinpath(cache_dir, paths...) + dest_filepath_unavailable = string(first(splitext(dest_filepath)), ".unavailable") + + download_dir = joinpath(download_dir, first(splitext(last(paths)))) + download_filepath = joinpath(download_dir, last(paths)) + download_filepath_unavailable = string(first(splitext(download_filepath)), ".unavailable") + + @debug "Downloading cache file for $name." + if isfile(dest_filepath_unavailable) + @debug "Cloud was unable to cache $name in the past, we won't try to retrieve it again." + return false + end + file = try + if Pkg.PlatformEngines.download_verify_unpack(link, nothing, download_dir) + mkpath(dirname(dest_filepath)) + if !isfile(download_filepath) && isfile(download_filepath_unavailable) + mv(download_filepath_unavailable, dest_filepath_unavailable) + @info "Cloud is unable to cache $name, we won't try to retrieve it again." + return false + end + mv(download_filepath, dest_filepath) + dest_filepath + else + @debug "Couldn't retrieve cache file for $name." + return false + end + catch err + @debug "Couldn't retrieve cache file for $name." exception = (err, catch_backtrace()) + return false + end + + cache = try + open(file, "r") do io + CacheStore.read(io) + end + catch + @warn "Couldn't read cache file for $name, deleting." + rm(file) + return false + end + + pkg_entry = Base.locate_package(Base.PkgId(uuid, name)) + if pkg_entry !== nothing && isfile(pkg_entry) + pkg_src = dirname(pkg_entry) + else + pkg_root = get_pkg_path(Base.PkgId(uuid, name), environment_path, depot_dir) + if pkg_root === nothing + @debug "Successfully downloaded and saved $(name), but with placeholder paths" + return false + end + pkg_src = joinpath(pkg_root, "src") + end + + # TODO: it would be better if the PLACEHOLDER replacement happens at runtime + # instead of "unpack-time", because we can use the current depot path + # in case the user switched to another one after downloading + + @debug "Replacing PLACEHOLDER with:" pkg_src + modify_dirs(cache.val, f -> modify_dir(f, r"^PLACEHOLDER", pkg_src)) + open(file, "w") do io + CacheStore.write(io, cache) + end + + @debug "Successfully downloaded, scrubbed and saved $(name)" + return true +end + +""" + validate_disc_store(store_path, manifest) + +This returns a list of non-jll packages in the manifest that don't have caches on disc. +""" +function validate_disc_store(store_path, manifest) + filter(manifest) do pkg + uuid = packageuuid(pkg) + endswith(packagename(manifest, uuid), "_jll") && return false + + file_name = joinpath(get_cache_path(manifest, uuid)...) + yield() + return !isfile(joinpath(store_path, file_name)) + end +end + +function find_project_file(env) + isdir(env) || return false + for filename in ("Project.toml", "JuliaProject.toml") + maybe_project_file = joinpath(env, filename) + if isfile(maybe_project_file) + return maybe_project_file + end + end + return false +end + +""" + get_pkg_path(pkg::Base.PkgId, env, depot_path) + +Find out where a package is installed without having to load it. +""" +function get_pkg_path(pkg::Base.PkgId, env, depot_path) + project_file = find_project_file(env) + project_file isa Bool && return nothing + manifest_file = Base.project_file_manifest_path(project_file) + + d = parsed_toml(manifest_file) + if get(d, "manifest_format", "0.0") == "2.0" + entries = get(d, "deps", nothing) + entries === nothing && return nothing + entries = map(e -> e[1], values(entries)) + else + entries = get(d, pkg.name, nothing) + end + entries === nothing && return nothing # TODO: allow name to mismatch? + for entry in entries + entry = entry::Dict{String,Any} + uuid = get(entry, "uuid", nothing)::Union{Nothing,String} + uuid === nothing && continue + if UUID(uuid) === pkg.uuid + path = get(entry, "path", nothing)::Union{Nothing,String} + # this can only be true for explicitly dev'ed packages + if path !== nothing + path = normpath(abspath(dirname(manifest_file), path)) + return path + end + hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing,String} + hash === nothing && return nothing + hash = Base.SHA1(hash) + # empty default path probably means that we should use the default Julia depots + if depot_path == "" + depot_paths = [] + if isdefined(Base, :append_default_depot_path!) + Base.append_default_depot_path!(depot_paths) + else + depot_paths = Pkg.depots() + end + else + depot_paths = [depot_path] + end + for depot in depot_paths + # Keep the 4 since it used to be the default + for slug in (Base.version_slug(pkg.uuid, hash, 4), Base.version_slug(pkg.uuid, hash)) + path = abspath(depot, "packages", pkg.name, slug) + ispath(path) && return path + end + end + return nothing + end + end + return nothing +end + +function load_package(c::Pkg.Types.Context, uuid, conn, loadingbay, percentage = missing) + isinmanifest(c, uuid isa String ? Base.UUID(uuid) : uuid) || return + pe_name = packagename(c, uuid) + + pid = Base.PkgId(uuid isa String ? Base.UUID(uuid) : uuid, pe_name) + if pid in keys(Base.loaded_modules) + conn !== nothing && println(conn, "PROCESSPKG;$pe_name;$uuid;noversion;$percentage") + loadingbay.eval(:($(Symbol(pe_name)) = $(Base.loaded_modules[pid]))) + m = getfield(loadingbay, Symbol(pe_name)) + else + m = try + conn !== nothing && println(conn, "STARTLOAD;$pe_name;$uuid;noversion;$percentage") + loadingbay.eval(:(import $(Symbol(pe_name)))) + conn !== nothing && println(conn, "STOPLOAD;$pe_name") + m = getfield(loadingbay, Symbol(pe_name)) + catch + return + end + end +end + +function write_cache(uuid, pkg::Package, outpath) + mkpath(dirname(outpath)) + @info "Now writing to disc $uuid" + open(outpath, "w") do io + CacheStore.write(io, pkg) + end + outpath +end + +""" + get_cache_path(manifest, uuid) + +Returns a vector containing the cache storage path for a package structured: [folder, folder, file]. +""" +function get_cache_path(manifest, uuid) + name = packagename(manifest, uuid) + pkg_info = frommanifest(manifest, uuid) + ver = version(pkg_info) + if ver === nothing + ver = "nothing" + if isdefined(Pkg.Types, :is_stdlib) && Pkg.Types.is_stdlib(uuid) + ver = VERSION + end + end + ver = replace(string(ver), '+'=>'_') + th = tree_hash(pkg_info) + th = th === nothing ? "nothing" : th + + [ + string(uppercase(string(name)[1])) + string(name, "_", uuid) + string("v", ver, "_", th, ".jstore") + ] +end + +function write_depot(server::Server, ctx, written_caches) + for (uuid, pkg) in server.depot + cache_paths = get_cache_path(manifest(ctx), uuid) + outpath = joinpath(server.storedir, cache_paths...) + outpath in written_caches && continue + + written_path = write_cache(uuid, pkg, outpath) + !isempty(written_path) && push!(written_caches, written_path) + end +end diff --git a/src/fileio.jl b/src/fileio.jl index 2b9e950..db634c1 100644 --- a/src/fileio.jl +++ b/src/fileio.jl @@ -133,8 +133,8 @@ function add_folder_from_disc!(jw::JuliaWorkspace, path; ignore_io_errors=false) end end -function workspace_from_folders(workspace_folders::Vector{String}) - jw = JuliaWorkspace() +function workspace_from_folders(workspace_folders::Vector{String}; symbol_cache_path=nothing, async_symbol_loading=false) + jw = JuliaWorkspace(; symbol_cache_path=symbol_cache_path, async_symbol_loading=async_symbol_loading) for folder in workspace_folders add_folder_from_disc!(jw, folder) diff --git a/src/inputs.jl b/src/inputs.jl index dd8ac1d..88b17f1 100644 --- a/src/inputs.jl +++ b/src/inputs.jl @@ -4,3 +4,5 @@ Salsa.@declare_input input_notebook_file(rt, uri)::NotebookFile Salsa.@declare_input input_fallback_test_project(rt)::Union{URI,Nothing} Salsa.@declare_input input_marked_diagnostics(rt)::DiagnosticsMark Salsa.@declare_input input_marked_testitems(rt)::TestitemsMark +Salsa.@declare_input input_package_symbols(rt)::Set{JuliaProjectEntryRegularPackage} +Salsa.@declare_input input_symbols_for_package(rt, JuliaProjectEntryRegularPackage)::@NamedTuple{status::Symbol,data} diff --git a/src/layer_symbols.jl b/src/layer_symbols.jl new file mode 100644 index 0000000..33dd1a5 --- /dev/null +++ b/src/layer_symbols.jl @@ -0,0 +1,7 @@ +Salsa.@derived function derived_required_symbol_info(rt) + all_projects = [derived_project(rt, i) for i in derived_project_folders(rt)] + + regular_packages = unique(Iterators.flatten(values(i.regular_packages) for i in all_projects)) + + return (;regular_packages) +end diff --git a/src/public.jl b/src/public.jl index 717717b..f5d016e 100644 --- a/src/public.jl +++ b/src/public.jl @@ -36,6 +36,34 @@ function add_file!(jw::JuliaWorkspace, file::TextFile) set_input_files!(jw.runtime, new_files) set_input_text_file!(jw.runtime, file.uri, file) + + if jw.symbol_cache_path!==nothing + required_symbols = derived_required_symbol_info(jw.runtime) + already_loaded_symbols = input_package_symbols(jw.runtime) + still_need_to_be_loaded = setdiff(required_symbols.regular_packages, already_loaded_symbols) + + for i in still_need_to_be_loaded + set_input_symbols_for_package!(jw.runtime, i, @NamedTuple{status::Symbol,data}((:loading, nothing))) + end + + new_already_loaded_symbols = Set{JuliaProjectEntryRegularPackage}(union(already_loaded_symbols, still_need_to_be_loaded)) + set_input_package_symbols!(jw.runtime, new_already_loaded_symbols) + + put!(jw.symbol_cache_channel_requests, still_need_to_be_loaded) + if !jw.symbol_cache_async + new_symbols = take!(jw.symbol_cache_channel_responses) + + for i in new_symbols + if i.data===nothing + set_input_symbols_for_package!(jw.runtime, i.package, @NamedTuple{status::Symbol,data}((:unavailable, nothing))) + else + set_input_symbols_for_package!(jw.runtime, i.package, @NamedTuple{status::Symbol,data}((:loaded, i.data))) + end + end + end + end + + # println("Done with add_file!") end function update_file!(jw::JuliaWorkspace, file::TextFile) diff --git a/src/types.jl b/src/types.jl index 4ebbe8a..e3346c8 100644 --- a/src/types.jl +++ b/src/types.jl @@ -119,13 +119,72 @@ end struct JuliaWorkspace runtime::Salsa.Runtime - function JuliaWorkspace() + symbol_cache_path::Union{Nothing,String} + symbol_cache_channel_requests::Union{Nothing,Channel{Any}} + symbol_cache_channel_responses::Union{Nothing,Channel{Any}} + symbol_cache_async::Bool + + function JuliaWorkspace(; symbol_cache_path=nothing, async_symbol_loading=true) rt = Salsa.Runtime() set_input_files!(rt, Set{URI}()) + set_input_package_symbols!(rt, Set{JuliaProjectEntryRegularPackage}()) set_input_fallback_test_project!(rt, nothing) - new(rt) + symbol_cache_channel_requests = symbol_cache_path===nothing ? nothing : Channel(Inf) + symbol_cache_channel_responses = symbol_cache_path===nothing ? nothing : Channel(Inf) + + if symbol_cache_path!==nothing + Threads.@spawn try + while true + still_need_to_be_loaded = take!(symbol_cache_channel_requests) + + new_symbols = map(still_need_to_be_loaded) do i + # Construct cache path + file_to_load_path = joinpath( + symbol_cache_path, + string(uppercase(string(i.name)[1])), # Capitalized first letter of the package name + string(i.name, "_", i.uuid), + string("v", i.version, "_", i.git_tree_sha1, ".jstore") + ) + + if isfile(file_to_load_path) + # println("We found cache file $file_to_load_path") + package_data = open(file_to_load_path) do io + SymbolServer.CacheStore.read(io) + end + + pkg_path = nothing + + git_tree_sha1 = Base.SHA1(i.git_tree_sha1) + # Keep the 4 since it used to be the default + slugs = (Base.version_slug(i.uuid, git_tree_sha1, 4), Base.version_slug(i.uuid, git_tree_sha1)) + for depot in Base.DEPOT_PATH, slug in slugs + path = abspath(depot, "packages", i.name, slug) + if ispath(path) + pkg_path = path + break + end + end + + if pkg_path !== nothing + SymbolServer.modify_dirs(package_data.val, f -> SymbolServer.modify_dir(f, r"^PLACEHOLDER", joinpath(pkg_path, "src"))) + end + + return @NamedTuple{package,data}((i, package_data)) + end + + return @NamedTuple{package,data}((i, nothing)) + end + + put!(symbol_cache_channel_responses, new_symbols) + end + catch err + Base.display_error(err, catch_backtrace()) + end + end + + new(rt, symbol_cache_path, symbol_cache_channel_requests, symbol_cache_channel_responses, async_symbol_loading) end end