diff --git a/Project.toml b/Project.toml
index 3bd583c..85bdec5 100644
--- a/Project.toml
+++ b/Project.toml
@@ -4,25 +4,27 @@ authors = ["David Anthoff <anthoff@berkeley.edu>"]
version = "4.6.1-DEV"
[deps]
+AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f"
+CancellationTokens = "2e8d271d-f2e2-407b-a864-17eb2156783e"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f"
Salsa = "1fbf2c77-44e2-4d5d-8131-0fa618a5c278"
-UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
TestItemDetection = "76b0de8b-5c4b-48ef-a724-914b33ca988d"
-CancellationTokens = "2e8d271d-f2e2-407b-a864-17eb2156783e"
-
-[extras]
-TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
[compat]
-JuliaSyntax = "0.4"
-julia = "1.6"
AutoHashEquals = "1,2"
+CancellationTokens = "1"
+InteractiveUtils = "1.11.0"
+JuliaSyntax = "0.4"
Salsa = "2.2.0"
TestItemDetection = "1.1"
-CancellationTokens = "1"
+julia = "1.6"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
[targets]
test = ["Test", "TestItemRunner"]
diff --git a/src/JuliaWorkspaces.jl b/src/JuliaWorkspaces.jl
index b3fce9e..dd9f2f4 100644
--- a/src/JuliaWorkspaces.jl
+++ b/src/JuliaWorkspaces.jl
@@ -17,6 +17,9 @@ using .URIs2: filepath2uri, uri2filepath
using .URIs2: URI, @uri_str
+include("SymbolServer/SymbolServer.jl")
+import .SymbolServer
+
include("exception_types.jl")
include("types.jl")
include("sourcetext.jl")
@@ -27,6 +30,7 @@ include("layer_projects.jl")
include("layer_testitems.jl")
include("layer_diagnostics.jl")
include("fileio.jl")
+include("layer_symbols.jl")
include("public.jl")
end
diff --git a/src/SymbolServer/SymbolServer.jl b/src/SymbolServer/SymbolServer.jl
new file mode 100644
index 0000000..ab6c309
--- /dev/null
+++ b/src/SymbolServer/SymbolServer.jl
@@ -0,0 +1,14 @@
+module SymbolServer
+
+import Pkg, InteractiveUtils, UUIDs
+
+using UUIDs: UUID
+
+include("faketypes.jl")
+include("symbols.jl")
+include("utils.jl")
+include("serialize.jl")
+
+using .CacheStore
+
+end
diff --git a/src/SymbolServer/faketypes.jl b/src/SymbolServer/faketypes.jl
new file mode 100644
index 0000000..94e1123
--- /dev/null
+++ b/src/SymbolServer/faketypes.jl
@@ -0,0 +1,172 @@
+########## Fake type-system
+
+
+# Used to label all objects
+@auto_hash_equals struct VarRef
+ parent::Union{VarRef,Nothing}
+ name::Symbol
+end
+VarRef(m::Module) = VarRef((parentmodule(m) == Main || parentmodule(m) == m) ? nothing : VarRef(parentmodule(m)), nameof(m))
+
+# These mirror Julia types (w/o the Fake prefix)
+@auto_hash_equals struct FakeTypeName
+ name::VarRef
+ parameters::Vector{Any}
+end
+
+function FakeTypeName(@nospecialize(x); justname=false)
+ @static if !(Vararg isa Type)
+ x isa typeof(Vararg) && return FakeTypeofVararg(x)
+ end
+ if x isa DataType
+ xname = x.name
+ xnamename = xname.name # necessary but unclear why.
+ if justname
+ FakeTypeName(VarRef(VarRef(x.name.module), x.name.name), [])
+ else
+ # FakeTypeName(VarRef(VarRef(x.name.module), x.name.name), _parameter.(x.parameters))
+ ft = FakeTypeName(VarRef(VarRef(x.name.module), x.name.name), [])
+ for p in x.parameters
+ push!(ft.parameters, _parameter(p))
+ end
+ ft
+ end
+ elseif x isa Union
+ FakeUnion(x)
+ elseif x isa UnionAll
+ FakeUnionAll(x)
+ elseif x isa TypeVar
+ FakeTypeVar(x)
+ elseif x isa Core.TypeofBottom
+ FakeTypeofBottom()
+ elseif x isa Module
+ VarRef(x)
+ else
+ error((x, typeof(x)))
+ end
+end
+
+@auto_hash_equals struct FakeTypeofBottom end
+@auto_hash_equals struct FakeUnion
+ a
+ b
+end
+FakeUnion(u::Union) = FakeUnion(FakeTypeName(u.a, justname=true), FakeTypeName(u.b, justname=true))
+@auto_hash_equals struct FakeTypeVar
+ name::Symbol
+ lb
+ ub
+end
+FakeTypeVar(tv::TypeVar) = FakeTypeVar(tv.name, FakeTypeName(tv.lb, justname=true), FakeTypeName(tv.ub, justname=true))
+@auto_hash_equals struct FakeUnionAll
+ var::FakeTypeVar
+ body::Any
+end
+FakeUnionAll(ua::UnionAll) = FakeUnionAll(FakeTypeVar(ua.var), FakeTypeName(ua.body, justname=true))
+
+function _parameter(@nospecialize(p))
+ if p isa Union{Int,Symbol,Bool,Char}
+ p
+ elseif !(p isa Type) && isbitstype(typeof(p))
+ 0
+ elseif p isa Tuple
+ _parameter.(p)
+ else
+ FakeTypeName(p, justname=true)
+ end
+end
+
+Base.print(io::IO, vr::VarRef) = vr.parent === nothing ? print(io, vr.name) : print(io, vr.parent, ".", vr.name)
+function Base.print(io::IO, tn::FakeTypeName)
+ print(io, tn.name)
+ if !isempty(tn.parameters)
+ print(io, "{")
+ for i = 1:length(tn.parameters)
+ print(io, tn.parameters[i])
+ i != length(tn.parameters) && print(io, ",")
+ end
+ print(io, "}")
+ end
+end
+Base.print(io::IO, x::FakeUnionAll) = print(io, x.body, " where ", x.var)
+function Base.print(io::IO, x::FakeUnion; inunion=false)
+ !inunion && print(io, "Union{")
+ print(io, x.a, ",")
+ if x.b isa FakeUnion
+ print(io, x.b, inunion=true)
+ else
+ print(io, x.b, "}")
+ end
+end
+function Base.print(io::IO, x::FakeTypeVar)
+ if isfakebottom(x.lb)
+ if isfakeany(x.ub)
+ print(io, x.name)
+ else
+ print(io, x.name, "<:", x.ub)
+ end
+ elseif isfakeany(x.ub)
+ print(io, x.lb, "<:", x.name)
+ else
+ print(io, x.lb, "<:", x.name, "<:", x.ub)
+ end
+end
+
+isfakeany(t) = false
+isfakeany(t::FakeTypeName) = isfakeany(t.name)
+isfakeany(vr::VarRef) = vr.name === :Any && vr.parent isa VarRef && vr.parent.name === :Core && vr.parent.parent === nothing
+
+isfakebottom(t) = false
+isfakebottom(t::FakeTypeofBottom) = true
+
+Base.:(==)(a::FakeTypeName, b::FakeTypeName) = a.name == b.name && a.parameters == b.parameters
+Base.:(==)(a::VarRef, b::VarRef) = a.parent == b.parent && a.name == b.name
+Base.:(==)(a::FakeTypeVar, b::FakeTypeVar) = a.lb == b.lb && a.name == b.name && a.ub == b.ub
+Base.:(==)(a::FakeUnionAll, b::FakeUnionAll) = a.var == b.var && a.body == b.body
+Base.:(==)(a::FakeUnion, b::FakeUnion) = a.a == b.a && a.b == b.b
+Base.:(==)(a::FakeTypeofBottom, b::FakeTypeofBottom) = true
+
+@static if !(Vararg isa Type)
+ @auto_hash_equals struct FakeTypeofVararg
+ T
+ N
+ FakeTypeofVararg() = new()
+ FakeTypeofVararg(T) = (new(T))
+ FakeTypeofVararg(T, N) = new(T, N)
+ end
+ function FakeTypeofVararg(va::typeof(Vararg))
+ if isdefined(va, :N)
+ vaN = va.N isa TypeVar ? FakeTypeVar(va.N) : va.N
+ FakeTypeofVararg(FakeTypeName(va.T; justname=true), vaN) # This should be FakeTypeName(va.N) but seems to crash inference.
+ elseif isdefined(va, :T)
+ FakeTypeofVararg(FakeTypeName(va.T; justname=true))
+ else
+ FakeTypeofVararg()
+ end
+ end
+ function Base.print(io::IO, va::FakeTypeofVararg)
+ print(io, "Vararg")
+ if isdefined(va, :T)
+ print(io, "{", va.T)
+ if isdefined(va, :N)
+ print(io, ",", va.N)
+ end
+ print(io, "}")
+ end
+ end
+ function Base.:(==)(a::FakeTypeofVararg, b::FakeTypeofVararg)
+ if isdefined(a, :T)
+ if isdefined(b, :T) && a.T == b.T
+ if isdefined(a, :N)
+ isdefined(b, :N) && a.N == b.N
+ else
+ !isdefined(b, :N)
+ end
+ else
+ false
+ end
+ else
+ !isdefined(b, :T)
+ end
+ end
+end
diff --git a/src/SymbolServer/serialize.jl b/src/SymbolServer/serialize.jl
new file mode 100644
index 0000000..d19344d
--- /dev/null
+++ b/src/SymbolServer/serialize.jl
@@ -0,0 +1,290 @@
+module CacheStore
+using ..SymbolServer: VarRef, FakeTypeName, FakeTypeofBottom, FakeTypeVar, FakeUnion, FakeUnionAll
+using ..SymbolServer: ModuleStore, Package, FunctionStore, MethodStore, DataTypeStore, GenericStore
+@static if !(Vararg isa Type)
+ using ..SymbolServer: FakeTypeofVararg
+end
+
+const NothingHeader = 0x01
+const SymbolHeader = 0x02
+const CharHeader = 0x03
+const IntegerHeader = 0x04
+const StringHeader = 0x05
+const VarRefHeader = 0x06
+const FakeTypeNameHeader = 0x07
+const FakeTypeofBottomHeader = 0x08
+const FakeTypeVarHeader = 0x09
+const FakeUnionHeader = 0x0a
+const FakeUnionAllHeader = 0xb
+const ModuleStoreHeader = 0x0c
+const MethodStoreHeader = 0x0d
+const FunctionStoreHeader = 0x0e
+const DataTypeStoreHeader = 0x0f
+const GenericStoreHeader = 0x10
+const PackageHeader = 0x11
+const TrueHeader = 0x12
+const FalseHeader = 0x13
+const TupleHeader = 0x14
+const FakeTypeofVarargHeader = 0x15
+const UndefHeader = 0x16
+
+
+function write(io, x::VarRef)
+ Base.write(io, VarRefHeader)
+ write(io, x.parent)
+ write(io, x.name)
+end
+function write(io, x::Nothing)
+ Base.write(io, NothingHeader)
+end
+function write(io, x::Char)
+ Base.write(io, CharHeader)
+ Base.write(io, UInt32(x))
+end
+function write(io, x::Bool)
+ x ? Base.write(io, TrueHeader) : Base.write(io, FalseHeader)
+end
+function write(io, x::Int)
+ Base.write(io, IntegerHeader)
+ Base.write(io, x)
+end
+function write(io, x::Symbol)
+ Base.write(io, SymbolHeader)
+ Base.write(io, sizeof(x))
+ Base.write(io, String(x))
+end
+function write(io, x::NTuple{N,Any}) where N
+ Base.write(io, TupleHeader)
+ Base.write(io, N)
+ for i = 1:N
+ write(io, x[i])
+ end
+end
+function write(io, x::String)
+ Base.write(io, StringHeader)
+ Base.write(io, sizeof(x))
+ Base.write(io, x)
+end
+function write(io, x::FakeTypeName)
+ Base.write(io, FakeTypeNameHeader)
+ write(io, x.name)
+ write_vector(io, x.parameters)
+end
+write(io, x::FakeTypeofBottom) = Base.write(io, FakeTypeofBottomHeader)
+function write(io, x::FakeTypeVar)
+ Base.write(io, FakeTypeVarHeader)
+ write(io, x.name)
+ write(io, x.lb)
+ write(io, x.ub)
+end
+function write(io, x::FakeUnion)
+ Base.write(io, FakeUnionHeader)
+ write(io, x.a)
+ write(io, x.b)
+end
+function write(io, x::FakeUnionAll)
+ Base.write(io, FakeUnionAllHeader)
+ write(io, x.var)
+ write(io, x.body)
+end
+
+@static if !(Vararg isa Type)
+ function write(io, x::FakeTypeofVararg)
+ Base.write(io, FakeTypeofVarargHeader)
+ isdefined(x, :T) ? write(io, x.T) : Base.write(io, UndefHeader)
+ isdefined(x, :N) ? write(io, x.N) : Base.write(io, UndefHeader)
+ end
+end
+
+function write(io, x::MethodStore)
+ Base.write(io, MethodStoreHeader)
+ write(io, x.name)
+ write(io, x.mod)
+ write(io, x.file)
+ Base.write(io, x.line)
+ Base.write(io, length(x.sig))
+ for p in x.sig
+ write(io, p[1])
+ write(io, p[2])
+ end
+ write_vector(io, x.kws)
+ write(io, x.rt)
+end
+
+function write(io, x::FunctionStore)
+ Base.write(io, FunctionStoreHeader)
+ write(io, x.name)
+ write_vector(io, x.methods)
+ write(io, x.doc)
+ write(io, x.extends)
+ write(io, x.exported)
+end
+
+function write(io, x::DataTypeStore)
+ Base.write(io, DataTypeStoreHeader)
+ write(io, x.name)
+ write(io, x.super)
+ write_vector(io, x.parameters)
+ write_vector(io, x.types)
+ write_vector(io, x.fieldnames)
+ write_vector(io, x.methods)
+ write(io, x.doc)
+ write(io, x.exported)
+end
+
+function write(io, x::GenericStore)
+ Base.write(io, GenericStoreHeader)
+ write(io, x.name)
+ write(io, x.typ)
+ write(io, x.doc)
+ write(io, x.exported)
+end
+
+function write(io, x::ModuleStore)
+ Base.write(io, ModuleStoreHeader)
+ write(io, x.name)
+ Base.write(io, length(x.vals))
+ for p in x.vals
+ write(io, p[1])
+ write(io, p[2])
+ end
+ write(io, x.doc)
+ write(io, x.exported)
+ write_vector(io, x.exportednames)
+ write_vector(io, x.used_modules)
+end
+
+function write(io, x::Package)
+ Base.write(io, PackageHeader)
+ write(io, x.name)
+ write(io, x.val)
+ Base.write(io, UInt128(x.uuid))
+ Base.write(io, x.sha === nothing ? zeros(UInt8, 32) : x.sha)
+end
+
+function write_vector(io, x)
+ Base.write(io, length(x))
+ for p in x
+ write(io, p)
+ end
+end
+
+function read(io, t = Base.read(io, UInt8))
+ # There are a bunch of `yield`s in potentially expensive code paths.
+ # One top-level `yield` would probably increase responsiveness in the
+ # LS, but increases runtime by 3x. This seems like a good compromise.
+
+ if t === VarRefHeader
+ VarRef(read(io), read(io))
+ elseif t === NothingHeader
+ nothing
+ elseif t === SymbolHeader
+ n = Base.read(io, Int)
+ out = Vector{UInt8}(undef, n)
+ readbytes!(io, out, n)
+ Symbol(String(out))
+ elseif t === StringHeader
+ # yield()
+ n = Base.read(io, Int)
+ out = Vector{UInt8}(undef, n)
+ readbytes!(io, out, n)
+ String(out)
+ elseif t === CharHeader
+ Char(Base.read(io, UInt32))
+ elseif t === IntegerHeader
+ Base.read(io, Int)
+ elseif t === FakeTypeNameHeader
+ FakeTypeName(read(io), read_vector(io, Any))
+ elseif t === FakeTypeofBottomHeader
+ FakeTypeofBottom()
+ elseif t === FakeTypeVarHeader
+ FakeTypeVar(read(io), read(io), read(io))
+ elseif t === FakeUnionHeader
+ FakeUnion(read(io), read(io))
+ elseif t === FakeUnionAllHeader
+ FakeUnionAll(read(io), read(io))
+ elseif t === FakeTypeofVarargHeader
+ T, N = read(io), read(io)
+ if T === nothing
+ FakeTypeofVararg()
+ elseif N === nothing
+ FakeTypeofVararg(T)
+ else
+ FakeTypeofVararg(T, N)
+ end
+ elseif t === UndefHeader
+ nothing
+ elseif t === MethodStoreHeader
+ # yield()
+ name = read(io)
+ mod = read(io)
+ file = read(io)
+ line = Base.read(io, UInt32)
+ nsig = Base.read(io, Int)
+ sig = Vector{Pair{Any, Any}}(undef, nsig)
+ for i in 1:nsig
+ sig[i] = read(io) => read(io)
+ end
+ kws = read_vector(io, Symbol)
+ rt = read(io)
+ MethodStore(name, mod, file, line, sig, kws, rt)
+ elseif t === FunctionStoreHeader
+ # yield()
+ FunctionStore(read(io), read_vector(io, MethodStore), read(io), read(io), read(io))
+ elseif t === DataTypeStoreHeader
+ # yield()
+ DataTypeStore(read(io), read(io), read_vector(io, Any), read_vector(io, Any), read_vector(io, Any), read_vector(io, MethodStore), read(io), read(io))
+ elseif t === GenericStoreHeader
+ # yield()
+ GenericStore(read(io), read(io), read(io), read(io))
+ elseif t === ModuleStoreHeader
+ # yield()
+ name = read(io)
+ n = Base.read(io, Int)
+ vals = Dict{Symbol,Any}()
+ sizehint!(vals, n)
+ for _ = 1:n
+ k = read(io)
+ v = read(io)
+ vals[k] = v
+ end
+ doc = read(io)
+ exported = read(io)
+ exportednames = read_vector(io, Symbol)
+ used_modules = read_vector(io, Symbol)
+ ModuleStore(name, vals, doc, exported, exportednames, used_modules)
+ elseif t === TrueHeader
+ true
+ elseif t === FalseHeader
+ false
+ elseif t === TupleHeader
+ N = Base.read(io, Int)
+ ntuple(i->read(io), N)
+ elseif t === PackageHeader
+ # yield()
+ name = read(io)
+ val = read(io)
+ uuid = Base.UUID(Base.read(io, UInt128))
+ sha = Base.read(io, 32)
+ Package(name, val, uuid, all(x == 0x00 for x in sha) ? nothing : sha)
+ else
+ error("Unknown type: $t")
+ end
+end
+
+function read_vector(io, T)
+ n = Base.read(io, Int)
+ v = Vector{T}(undef, n)
+ for i in 1:n
+ v[i] = read(io)
+ end
+ v
+end
+
+function storeunstore(x)
+ io = IOBuffer()
+ write(io, x)
+ bs = take!(io)
+ read(IOBuffer(bs))
+end
+end
diff --git a/src/SymbolServer/symbols.jl b/src/SymbolServer/symbols.jl
new file mode 100644
index 0000000..fc870e1
--- /dev/null
+++ b/src/SymbolServer/symbols.jl
@@ -0,0 +1,659 @@
+# using LibGit2, InteractiveUtils
+
+mutable struct Server
+ storedir::String
+ context::Pkg.Types.Context
+ depot::Dict
+end
+
+abstract type SymStore end
+@auto_hash_equals struct ModuleStore <: SymStore
+ name::VarRef
+ vals::Dict{Symbol,Any}
+ doc::String
+ exported::Bool
+ exportednames::Vector{Symbol}
+ used_modules::Vector{Symbol}
+end
+
+ModuleStore(m) = ModuleStore(VarRef(m), Dict{Symbol,Any}(), _doc(Base.Docs.Binding(m, nameof(m))), true, unsorted_names(m), Symbol[])
+Base.getindex(m::ModuleStore, k) = m.vals[k]
+Base.setindex!(m::ModuleStore, v, k) = (m.vals[k] = v)
+Base.haskey(m::ModuleStore, k) = haskey(m.vals, k)
+
+const EnvStore = Dict{Symbol,ModuleStore}
+
+@auto_hash_equals struct Package
+ name::String
+ val::ModuleStore
+ uuid::Base.UUID
+ sha::Union{Vector{UInt8},Nothing}
+end
+Package(name::String, val::ModuleStore, uuid::String, sha) = Package(name, val, Base.UUID(uuid), sha)
+
+@auto_hash_equals struct MethodStore
+ name::Symbol
+ mod::Symbol
+ file::String
+ line::Int32
+ sig::Vector{Pair{Any,Any}}
+ kws::Vector{Symbol}
+ rt::Any
+end
+
+@auto_hash_equals struct DataTypeStore <: SymStore
+ name::FakeTypeName
+ super::FakeTypeName
+ parameters::Vector{Any}
+ types::Vector{Any}
+ fieldnames::Vector{Any}
+ methods::Vector{MethodStore}
+ doc::String
+ exported::Bool
+ function DataTypeStore(names, super, parameters, fieldtypes, fieldnames, methods, doc, exported)
+ if length(fieldtypes) < length(fieldnames)
+ append!(fieldtypes, [Any for _ in 1:(length(fieldnames)-length(fieldtypes))])
+ end
+ new(names, super, parameters, fieldtypes, fieldnames, methods, doc, exported)
+ end
+end
+
+function DataTypeStore(@nospecialize(t), symbol, parent_mod, exported)
+ ur_t = Base.unwrap_unionall(t)
+ parameters = if isdefined(ur_t, :parameters)
+ map(ur_t.parameters) do p
+ _parameter(p)
+ end
+ else
+ []
+ end
+ types = if isdefined(ur_t, :types)
+ map(ur_t.types) do p
+ FakeTypeName(p)
+ end
+ else
+ []
+ end
+ DataTypeStore(FakeTypeName(ur_t), FakeTypeName(ur_t.super), parameters, types, isconcretetype(ur_t) && fieldcount(ur_t) > 0 ? collect(fieldnames(ur_t)) : Symbol[], MethodStore[], _doc(Base.Docs.Binding(parent_mod, symbol)), exported)
+end
+
+@auto_hash_equals struct FunctionStore <: SymStore
+ name::VarRef
+ methods::Vector{MethodStore}
+ doc::String
+ extends::VarRef
+ exported::Bool
+end
+
+function FunctionStore(@nospecialize(f), symbol, parent_mod, exported)
+ if f isa Core.IntrinsicFunction
+ FunctionStore(VarRef(VarRef(Core.Intrinsics), nameof(f)), MethodStore[], _doc(Base.Docs.Binding(parent_mod, symbol)), VarRef(VarRef(parentmodule(f)), nameof(f)), exported)
+ else
+ FunctionStore(VarRef(VarRef(parent_mod), nameof(f)), MethodStore[], _doc(Base.Docs.Binding(parent_mod, symbol)), VarRef(VarRef(parentmodule(f)), nameof(f)), exported)
+ end
+end
+
+@auto_hash_equals struct GenericStore <: SymStore
+ name::VarRef
+ typ::Any
+ doc::String
+ exported::Bool
+end
+
+# adapted from https://github.com/timholy/CodeTracking.jl/blob/afc73a957f5034cc7f02e084a91283c47882f92b/src/utils.jl#L87-L122
+
+"""
+ path = maybe_fix_path(path)
+
+Return a normalized, absolute path for a source file `path`.
+"""
+function maybe_fix_path(file)
+ if !isabspath(file)
+ # This may be a Base or Core method
+ newfile = Base.find_source_file(file)
+ if isa(newfile, AbstractString)
+ file = normpath(newfile)
+ end
+ end
+ return maybe_fixup_stdlib_path(file)
+end
+
+safe_isfile(x) = try isfile(x); catch; false end
+const BUILDBOT_STDLIB_PATH = dirname(abspath(joinpath(String((InteractiveUtils.@which InteractiveUtils.versioninfo()).file), "..", "..", "..")))
+replace_buildbot_stdlibpath(str::String) = replace(str, BUILDBOT_STDLIB_PATH => Sys.STDLIB)
+"""
+ path = maybe_fixup_stdlib_path(path::String)
+
+Return `path` corrected for julia issue [#26314](https://github.com/JuliaLang/julia/issues/26314) if applicable.
+Otherwise, return the input `path` unchanged.
+
+Due to the issue mentioned above, location info for methods defined one of Julia's standard libraries
+are, for non source Julia builds, given as absolute paths on the worker that built the `julia` executable.
+This function corrects such a path to instead refer to the local path on the users drive.
+"""
+function maybe_fixup_stdlib_path(path)
+ if !safe_isfile(path)
+ maybe_stdlib_path = replace_buildbot_stdlibpath(path)
+ safe_isfile(maybe_stdlib_path) && return maybe_stdlib_path
+ end
+ return path
+end
+
+_default_world_age() =
+ if isdefined(Base, :get_world_counter)
+ Base.get_world_counter()
+ else
+ typemax(UInt)
+ end
+
+const _global_method_cache = IdDict{Any,Vector{Any}}()
+function methodinfo(@nospecialize(f); types = Tuple, world = _default_world_age())
+ key = (f, types, world)
+ if haskey(_global_method_cache, key)
+ return _global_method_cache[key]
+ else
+ ms = Base._methods(f, types, -1, world)
+ ms isa Vector || (ms = [])
+ _global_method_cache[key] = ms
+ return ms
+ end
+end
+
+function methodlist(@nospecialize(f))
+ ms = methodinfo(f)
+ Method[x[3]::Method for x in ms]
+end
+
+function sparam_syms(meth::Method)
+ s = Symbol[]
+ sig = meth.sig
+ while sig isa UnionAll
+ push!(s, Symbol(sig.var.name))
+ sig = sig.body
+ end
+ return s
+end
+
+function cache_methods(@nospecialize(f), name, env, get_return_type)
+ if isa(f, Core.Builtin)
+ return MethodStore[]
+ end
+ types = Tuple
+ world = _default_world_age()
+ ms = Tuple{Module,MethodStore}[]
+ methods0 = try
+ methodinfo(f; types = types, world = world)
+ catch err
+ @debug "Error in method lookup for $f" ex=(err, catch_backtrace())
+ return ms
+ end
+ ind_of_method_w_kws = Int[] # stores the index of methods with kws.
+ i = 1
+ for m in methods0
+ # Get inferred method return type
+ if get_return_type
+ sparams = Core.svec(sparam_syms(m[3])...)
+ rt = try
+ @static if isdefined(Core.Compiler, :NativeInterpreter)
+ Core.Compiler.typeinf_type(Core.Compiler.NativeInterpreter(), m[3], m[3].sig, sparams)
+ else
+ Core.Compiler.typeinf_type(m[3], m[3].sig, sparams, Core.Compiler.Params(world))
+ end
+ catch e
+ Any
+ end
+ else
+ rt = Any
+ end
+ file = maybe_fix_path(String(m[3].file))
+ MS = MethodStore(m[3].name, nameof(m[3].module), file, m[3].line, [], Symbol[], FakeTypeName(rt))
+ # Get signature
+ sig = Base.unwrap_unionall(m[1])
+ argnames = getargnames(m[3])
+ for i = 2:m[3].nargs
+ push!(MS.sig, argnames[i] => FakeTypeName(sig.parameters[i]))
+ end
+ kws = getkws(m[3])
+ if !isempty(kws)
+ push!(ind_of_method_w_kws, i)
+ end
+ for kw in kws
+ push!(MS.kws, kw)
+ end
+ push!(ms, (m[3].module, MS))
+ i += 1
+ end
+
+ # Go back and add kws to methods defined in the same place as others with kws.
+ for i in ind_of_method_w_kws
+ for mj in ms
+ if mj[2].file == ms[i][2].file && mj[2].line == ms[i][2].line && isempty(mj[2].kws)
+ for kw in ms[i][2].kws
+ push!(mj[2].kws, kw)
+ end
+ end
+ end
+ end
+
+ func_vr = VarRef(VarRef(parentmodule(f)), name)
+ for m in ms
+ mvr = VarRef(m[1])
+ modstore = _lookup(mvr, env)
+ modstore === nothing && continue
+
+ if !haskey(modstore, name)
+ modstore[name] = FunctionStore(VarRef(mvr, name), MethodStore[m[2]], "", func_vr, false)
+ elseif !(modstore[name] isa DataTypeStore || modstore[name] isa FunctionStore)
+ modstore[name] = FunctionStore(VarRef(mvr, name), MethodStore[m[2]], "", func_vr, false)
+ else
+ push!(modstore[name].methods, m[2])
+ end
+ end
+ return ms
+end
+
+getargnames(m::Method) = Base.method_argnames(m)
+@static if length(first(methods(Base.kwarg_decl)).sig.parameters) == 2
+ getkws = Base.kwarg_decl
+else
+ function getkws(m::Method)
+ sig = Base.unwrap_unionall(m.sig)
+ length(sig.parameters) == 0 && return []
+ sig.parameters[1] isa Union && return []
+ !isdefined(Base.unwrap_unionall(sig.parameters[1]), :name) && return []
+ fname = Base.unwrap_unionall(sig.parameters[1]).name
+ if isdefined(fname.mt, :kwsorter)
+ Base.kwarg_decl(m, typeof(fname.mt.kwsorter))
+ else
+ []
+ end
+ end
+end
+
+function apply_to_everything(f, m = nothing, visited = Base.IdSet{Module}())
+ if m isa Module
+ push!(visited, m)
+ for s in unsorted_names(m, all = true, imported = true)
+ (!isdefined(m, s) || s == nameof(m)) && continue
+ x = getfield(m, s)
+ f(x)
+ if x isa Module && !in(x, visited)
+ apply_to_everything(f, x, visited)
+ end
+ end
+ else
+ for m in Base.loaded_modules_array()
+ in(m, visited) || apply_to_everything(f, m, visited)
+ end
+ end
+end
+
+
+
+function oneverything(f, m = nothing, visited = Base.IdSet{Module}())
+ if m isa Module
+ push!(visited, m)
+ state = nothing
+ for s in unsorted_names(m, all = true, imported = true)
+ !isdefined(m, s) && continue
+ x = getfield(m, s)
+ state = f(m, s, x, state)
+ if x isa Module && !in(x, visited)
+ oneverything(f, x, visited)
+ end
+ end
+ else
+ for m in Base.loaded_modules_array()
+ in(m, visited) || oneverything(f, m, visited)
+ end
+ end
+end
+
+const _global_symbol_cache_by_mod = IdDict{Module,Base.IdSet{Symbol}}()
+function build_namecache(m, s, @nospecialize(x), state::Union{Base.IdSet{Symbol},Nothing} = nothing)
+ if state === nothing
+ state = get(_global_symbol_cache_by_mod, m, nothing)
+ if state === nothing
+ state = _global_symbol_cache_by_mod[m] = Base.IdSet{Symbol}()
+ end
+ end
+ push!(state, s)
+end
+
+function getnames(m::Module)
+ cache = get(_global_symbol_cache_by_mod, m, nothing)
+ if cache === nothing
+ oneverything(build_namecache, m)
+ cache = _global_symbol_cache_by_mod[m]
+ end
+ return cache
+end
+
+function allmodulenames()
+ symbols = Base.IdSet{Symbol}()
+ oneverything((m, s, x, state) -> (x isa Module && push!(symbols, s); return state))
+ return symbols
+end
+
+function allthingswithmethods()
+ symbols = Base.IdSet{Any}()
+ oneverything(function (m, s, x, state)
+ if !Base.isvarargtype(x) && !isempty(methodlist(x))
+ push!(symbols, x)
+ end
+ return state
+ end)
+ return symbols
+end
+
+function allmethods()
+ ms = Method[]
+ oneverything(function (m, s, x, state)
+ if !Base.isvarargtype(x) && !isempty(methodlist(x))
+ append!(ms, methodlist(x))
+ end
+ return state
+ end)
+ return ms
+end
+
+usedby(outer, inner) = outer !== inner && isdefined(outer, nameof(inner)) && getproperty(outer, nameof(inner)) === inner && all(isdefined(outer, name) || !isdefined(inner, name) for name in unsorted_names(inner))
+istoplevelmodule(m) = parentmodule(m) === m || parentmodule(m) === Main
+
+function getmoduletree(m::Module, amn, visited = Base.IdSet{Module}())
+ push!(visited, m)
+ cache = ModuleStore(m)
+ for s in unsorted_names(m, all = true, imported = true)
+ !isdefined(m, s) && continue
+ x = getfield(m, s)
+ if x isa Module
+ if istoplevelmodule(x)
+ cache[s] = VarRef(x)
+ elseif m === parentmodule(x)
+ cache[s] = getmoduletree(x, amn, visited)
+ else
+ cache[s] = VarRef(x)
+ end
+ end
+ end
+ for n in amn
+ if n !== nameof(m) && isdefined(m, n)
+ x = getfield(m, n)
+ if x isa Module
+ if !haskey(cache, n)
+ cache[n] = VarRef(x)
+ end
+ if x !== Main && usedby(m, x)
+ push!(cache.used_modules, n)
+ end
+ end
+ end
+ end
+ cache
+end
+
+function getenvtree(names = nothing)
+ amn = allmodulenames()
+ EnvStore(nameof(m) => getmoduletree(m, amn) for m in Base.loaded_modules_array() if names === nothing || nameof(m) in names)
+end
+
+# faster and more correct split_module_names
+all_names(m) = all_names(m, x -> isdefined(m, x))
+function all_names(m, pred, symbols = Set(Symbol[]), seen = Set(Module[]))
+ push!(seen, m)
+ ns = unsorted_names(m; all = true, imported = false)
+ for n in ns
+ isdefined(m, n) || continue
+ Base.isdeprecated(m, n) && continue
+ val = getfield(m, n)
+ if val isa Module && !(val in seen)
+ all_names(val, pred, symbols, seen)
+ end
+ if pred(n)
+ push!(symbols, n)
+ end
+ end
+ symbols
+end
+
+function symbols(env::EnvStore, m::Union{Module,Nothing} = nothing, allnames::Base.IdSet{Symbol} = getallns(), visited = Base.IdSet{Module}(); get_return_type = false)
+ if m isa Module
+ cache = _lookup(VarRef(m), env, true)
+ cache === nothing && return
+ push!(visited, m)
+ ns = all_names(m)
+ for s in ns
+ !isdefined(m, s) && continue
+ x = getfield(m, s)
+ if Base.unwrap_unionall(x) isa DataType # Unions aren't handled here.
+ if parentmodule((x)) === m
+ cache[s] = DataTypeStore(x, s, m, s in getnames(m))
+ cache_methods(x, s, env, get_return_type)
+ elseif nameof(x) !== s
+ # This needs some finessing.
+ cache[s] = DataTypeStore(x, s, m, s in getnames(m))
+ ms = cache_methods(x, s, env, get_return_type)
+ # A slightly difficult case. `s` is probably a shadow binding of `x` but we should store the methods nonetheless.
+ # Example: DataFrames.Not points to InvertedIndices.InvertedIndex
+ for m in ms
+ push!(cache[s].methods, m[2])
+ end
+ else
+ # These are imported variables that are reexported.
+ cache[s] = VarRef(VarRef(parentmodule(x)), nameof(x))
+ end
+ elseif x isa Function
+ if parentmodule(x) === m || (x isa Core.IntrinsicFunction && m === Core.Intrinsics)
+ cache[s] = FunctionStore(x, s, m, s in getnames(m))
+ cache_methods(x, s, env, get_return_type)
+ elseif !haskey(cache, s)
+ # This will be replaced at a later point by a FunctionStore if methods for `x` are defined within `m`.
+ if x isa Core.IntrinsicFunction
+ cache[s] = VarRef(VarRef(Core.Intrinsics), nameof(x))
+ else
+ cache[s] = VarRef(VarRef(parentmodule(x)), nameof(x))
+ end
+ elseif !((cache[s] isa FunctionStore || cache[s] isa DataTypeStore) && !isempty(cache[s].methods))
+ # These are imported variables that are reexported.
+ # We don't want to remove Func/DT stores that have methods (these will be specific to the module)
+ if x isa Core.IntrinsicFunction
+ cache[s] = VarRef(VarRef(Core.Intrinsics), nameof(x))
+ else
+ cache[s] = VarRef(VarRef(parentmodule(x)), nameof(x))
+ end
+ end
+ elseif x isa Module
+ if x === m
+ cache[s] = VarRef(x)
+ elseif parentmodule(x) === m
+ symbols(env, x, allnames, visited, get_return_type = get_return_type)
+ else
+ cache[s] = VarRef(x)
+ end
+ else
+ cache[s] = GenericStore(VarRef(VarRef(m), s), FakeTypeName(typeof(x)), _doc(Base.Docs.Binding(m, s)), s in getnames(m))
+ end
+ end
+ else
+ for m in Base.loaded_modules_array()
+ in(m, visited) || symbols(env, m, allnames, visited, get_return_type = get_return_type)
+ end
+ end
+end
+
+
+function load_core(; get_return_type = false)
+ c = Pkg.Types.Context()
+ cache = getenvtree([:Core,:Base])
+ symbols(cache, get_return_type = get_return_type)
+ cache[:Main] = ModuleStore(VarRef(nothing, :Main), Dict(), "", true, [], [])
+
+ # This is wrong. Every module contains it's own include function.
+ push!(cache[:Base].exportednames, :include)
+ let f = cache[:Base][:include]
+ if haskey(cache[:Base][:MainInclude], :include)
+ cache[:Base][:include] = FunctionStore(f.name, cache[:Base][:MainInclude][:include].methods, f.doc, f.extends, true)
+ else
+ m1 = first(f.methods)
+ push!(f.methods, MethodStore(
+ m1.name,
+ m1.mod,
+ m1.file,
+ m1.line,
+ Pair{Any,Any}[
+ :x => SymbolServer.FakeTypeName(SymbolServer.VarRef(SymbolServer.VarRef(nothing, :Core), :AbstractString), Any[])
+ ],
+ [],
+ m1.rt
+ ))
+ end
+ end
+
+ cache[:Base][Symbol("@.")] = cache[:Base][Symbol("@__dot__")]
+ cache[:Core][:Main] = GenericStore(VarRef(nothing, :Main), FakeTypeName(Module), _doc(Base.Docs.Binding(Main, :Main)), true)
+ # Add built-ins
+ builtins = Symbol[nameof(getfield(Core, n).instance) for n in unsorted_names(Core, all = true) if isdefined(Core, n) && getfield(Core, n) isa DataType && isdefined(getfield(Core, n), :instance) && getfield(Core, n).instance isa Core.Builtin]
+ cnames = unsorted_names(Core)
+ for f in builtins
+ if !haskey(cache[:Core], f)
+ cache[:Core][f] = FunctionStore(getfield(Core, Symbol(f)), Symbol(f), Core, Symbol(f) in cnames)
+ end
+ end
+ haskey(cache[:Core], :_typevar) && push!(cache[:Core][:_typevar].methods, MethodStore(:_typevar, :Core, "built-in", 0, [:n => FakeTypeName(Symbol), :lb => FakeTypeName(Any), :ub => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:_apply].methods, MethodStore(:_apply, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ haskey(cache[:Core].vals, :_apply_iterate) && push!(cache[:Core][:_apply_iterate].methods, MethodStore(:_apply_iterate, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ if isdefined(Core, :_call_latest)
+ push!(cache[:Core][:_call_latest].methods, MethodStore(:_call_latest, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:_call_in_world].methods, MethodStore(:_call_in_world, :Core, "built-in", 0, [:world => FakeTypeName(UInt), :f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ else
+ if isdefined(Core, :_apply_in_world)
+ push!(cache[:Core][:_apply_in_world].methods, MethodStore(:_apply_in_world, :Core, "built-in", 0, [:world => FakeTypeName(UInt), :f => FakeTypeName(Function), :args => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ end
+ push!(cache[:Core][:_apply_latest].methods, MethodStore(:_apply_latest, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ end
+ push!(cache[:Core][:_apply_pure].methods, MethodStore(:_apply_pure, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:_expr].methods, MethodStore(:_expr, :Core, "built-in", 0, [:head => FakeTypeName(Symbol), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Expr)))
+ haskey(cache[:Core].vals, :_typevar) && push!(cache[:Core][:_typevar].methods, MethodStore(:_typevar, :Core, "built-in", 0, [:name => FakeTypeName(Symbol), :lb => FakeTypeName(Any), :ub => FakeTypeName(Any)], Symbol[], FakeTypeName(TypeVar)))
+ push!(cache[:Core][:applicable].methods, MethodStore(:applicable, :Core, "built-in", 0, [:f => FakeTypeName(Function), :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Bool)))
+ push!(cache[:Core][:apply_type].methods, MethodStore(:apply_type, :Core, "built-in", 0, [:T => FakeTypeName(UnionAll), :types => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(UnionAll)))
+ push!(cache[:Core][:arrayref].methods, MethodStore(:arrayref, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any), :c => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:arrayset].methods, MethodStore(:arrayset, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any), :c => FakeTypeName(Any), :d => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:arraysize].methods, MethodStore(:arraysize, :Core, "built-in", 0, [:a => FakeTypeName(Array), :i => FakeTypeName(Int)], Symbol[], FakeTypeName(Int)))
+ haskey(cache[:Core], :const_arrayref) && push!(cache[:Core][:const_arrayref].methods, MethodStore(:const_arrayref, :Core, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:fieldtype].methods, MethodStore(:fieldtype, :Core, "built-in", 0, [:t => FakeTypeName(DataType), :field => FakeTypeName(Symbol)], Symbol[], FakeTypeName(Type{T} where T)))
+ push!(cache[:Core][:getfield].methods, MethodStore(:setfield, :Core, "built-in", 0, [:object => FakeTypeName(Any), :item => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:ifelse].methods, MethodStore(:ifelse, :Core, "built-in", 0, [:condition => FakeTypeName(Bool), :x => FakeTypeName(Any), :y => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:invoke].methods, MethodStore(:invoke, :Core, "built-in", 0, [:f => FakeTypeName(Function), :x => FakeTypeName(Any), :argtypes => FakeTypeName(Type{T} where T) , :args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:isa].methods, MethodStore(:isa, :Core, "built-in", 0, [:a => FakeTypeName(Any), :T => FakeTypeName(Type{T} where T)], Symbol[], FakeTypeName(Bool)))
+ push!(cache[:Core][:isdefined].methods, MethodStore(:getproperty, :Core, "built-in", 0, [:value => FakeTypeName(Any), :field => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:nfields].methods, MethodStore(:nfields, :Core, "built-in", 0, [:x => FakeTypeName(Any)], Symbol[], FakeTypeName(Int)))
+ push!(cache[:Core][:setfield!].methods, MethodStore(:setfield!, :Core, "built-in", 0, [:value => FakeTypeName(Any), :name => FakeTypeName(Symbol), :x => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:sizeof].methods, MethodStore(:sizeof, :Core, "built-in", 0, [:obj => FakeTypeName(Any)], Symbol[], FakeTypeName(Int)))
+ push!(cache[:Core][:svec].methods, MethodStore(:svec, :Core, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:throw].methods, MethodStore(:throw, :Core, "built-in", 0, [:e => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:tuple].methods, MethodStore(:tuple, :Core, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:typeassert].methods, MethodStore(:typeassert, :Core, "built-in", 0, [:x => FakeTypeName(Any), :T => FakeTypeName(Type{T} where T)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:typeof].methods, MethodStore(:typeof, :Core, "built-in", 0, [:x => FakeTypeName(Any)], Symbol[], FakeTypeName(Type{T} where T)))
+
+ push!(cache[:Core][:getproperty].methods, MethodStore(:getproperty, :Core, "built-in", 0, [:value => FakeTypeName(Any), :name => FakeTypeName(Symbol)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:setproperty!].methods, MethodStore(:setproperty!, :Core, "built-in", 0, [:value => FakeTypeName(Any), :name => FakeTypeName(Symbol), :x => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:setproperty!].methods, MethodStore(:setproperty!, :Core, "built-in", 0, [:value => FakeTypeName(Any), :name => FakeTypeName(Symbol), :x => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ haskey(cache[:Core], :_abstracttype) && push!(cache[:Core][:_abstracttype].methods, MethodStore(:_abstracttype, :Core, "built-in", 0, [:m => FakeTypeName(Module), :x => FakeTypeName(Symbol), :p => FakeTypeName(Core.SimpleVector)], Symbol[], FakeTypeName(Any)))
+ haskey(cache[:Core], :_primitivetype) && push!(cache[:Core][:_primitivetype].methods, MethodStore(:_primitivetype, :Core, "built-in", 0, [:m => FakeTypeName(Module), :x => FakeTypeName(Symbol), :p => FakeTypeName(Core.SimpleVector), :n => FakeTypeName(Core.Int)], Symbol[], FakeTypeName(Any)))
+ haskey(cache[:Core], :_equiv_typedef) && push!(cache[:Core][:_equiv_typedef].methods, MethodStore(:_equiv_typedef, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ haskey(cache[:Core], :_setsuper!) && push!(cache[:Core][:_setsuper!].methods, MethodStore(:_setsuper!, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ haskey(cache[:Core], :_structtype) && push!(cache[:Core][:_structtype].methods, MethodStore(:_structtype, :Core, "built-in", 0, [:m => FakeTypeName(Module), :x => FakeTypeName(Symbol), :p => FakeTypeName(Core.SimpleVector), :fields => FakeTypeName(Core.SimpleVector), :mut => FakeTypeName(Bool), :z => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ haskey(cache[:Core], :_typebody) && push!(cache[:Core][:_typebody!].methods, MethodStore(:_typebody!, :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:(===)].methods, MethodStore(:(===), :Core, "built-in", 0, [:a => FakeTypeName(Any), :b => FakeTypeName(Any)], Symbol[], FakeTypeName(Any)))
+ push!(cache[:Core][:(<:)].methods, MethodStore(:(<:), :Core, "built-in", 0, [:a => FakeTypeName(Type{T} where T), :b => FakeTypeName(Type{T} where T)], Symbol[], FakeTypeName(Any)))
+ # Add unspecified methods for Intrinsics, working out the actual methods will need to be done by hand?
+ for n in names(Core.Intrinsics)
+ if getfield(Core.Intrinsics, n) isa Core.IntrinsicFunction
+ push!(cache[:Core][:Intrinsics][n].methods, MethodStore(n, :Intrinsics, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ :args => FakeTypeName(Vararg{Any})
+ end
+ end
+
+ for bi in builtins
+ if haskey(cache[:Core], bi) && isempty(cache[:Core][bi].methods)
+ # Add at least one arbitrary method for anything left over
+ push!(cache[:Core][bi].methods, MethodStore(bi, :none, "built-in", 0, [:x => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)))
+ end
+ end
+
+ cache[:Core][:ccall] = FunctionStore(VarRef(VarRef(Core), :ccall),
+ MethodStore[
+ MethodStore(:ccall, :Core, "built-in", 0, [:args => FakeTypeName(Vararg{Any})], Symbol[], FakeTypeName(Any)) # General method - should be fixed
+ ],
+ "`ccall((function_name, library), returntype, (argtype1, ...), argvalue1, ...)`\n`ccall(function_name, returntype, (argtype1, ...), argvalue1, ...)`\n`ccall(function_pointer, returntype, (argtype1, ...), argvalue1, ...)`\n\nCall a function in a C-exported shared library, specified by the tuple (`function_name`, `library`), where each component is either a string or symbol. Instead of specifying a library, one\ncan also use a `function_name` symbol or string, which is resolved in the current process. Alternatively, `ccall` may also be used to call a function pointer `function_pointer`, such as one\nreturned by `dlsym`.\n\nNote that the argument type tuple must be a literal tuple, and not a tuple-valued variable or expression.\n\nEach `argvalue` to the `ccall` will be converted to the corresponding `argtype`, by automatic insertion of calls to `unsafe_convert(argtype, cconvert(argtype, argvalue))`. (See also the documentation for `unsafe_convert` and `cconvert` for further details.) In most cases, this simply results in a call to `convert(argtype, argvalue)`.",
+ VarRef(VarRef(Core), :ccall),
+ true)
+ push!(cache[:Core].exportednames, :ccall)
+ cache[:Core][Symbol("@__doc__")] = FunctionStore(VarRef(VarRef(Core), Symbol("@__doc__")), [], "", VarRef(VarRef(Core), Symbol("@__doc__")), true)
+ cache_methods(getfield(Core, Symbol("@__doc__")), Symbol("@__doc__"), cache, false)
+ # Accounts for the dd situation where Base.rand only has methods from Random which doesn't appear to be explicitly used.
+ # append!(cache[:Base][:rand].methods, cache_methods(Base.rand, cache))
+ for m in cache_methods(Base.rand, :rand, cache, get_return_type)
+ push!(cache[:Base][:rand].methods, m[2])
+ end
+ for m in cache_methods(Base.randn, :randn, cache, get_return_type)
+ push!(cache[:Base][:randn].methods, m[2])
+ end
+
+ # Intrinsics
+ cache[:Core][:add_int] = VarRef(VarRef(VarRef(nothing, :Core), :Intrinsics), :add_int)
+ cache[:Core][:sle_int] = VarRef(VarRef(VarRef(nothing, :Core), :Intrinsics), :sle_int)
+ return cache
+end
+
+
+function collect_extended_methods(depot::EnvStore, extendeds = Dict{VarRef,Vector{VarRef}}())
+ for m in depot
+ collect_extended_methods(m[2], extendeds, m[2].name)
+ end
+ extendeds
+end
+
+function collect_extended_methods(mod::ModuleStore, extendeds, mname)
+ for (n, v) in mod.vals
+ if (v isa FunctionStore) && v.extends != v.name
+ haskey(extendeds, v.extends) ? push!(extendeds[v.extends], mname) : (extendeds[v.extends] = VarRef[v.extends.parent, mname])
+ elseif v isa ModuleStore
+ collect_extended_methods(v, extendeds, v.name)
+ end
+ end
+end
+
+getallns() = let allns = Base.IdSet{Symbol}(); oneverything((m, s, x, state) -> push!(allns, s)); allns end
+
+"""
+ split_module_names(m::Module, allns)
+
+Return two lists of names accessible from calling `getfield(m, somename)`. The first
+contains those symbols returned by `Base.names(m, all = true)`. The second contains
+all others, including imported symbols and those introduced by the `using` of modules.
+"""
+function split_module_names(m::Module, allns)
+ internal_names = getnames(m)
+ availablenames = Set{Symbol}([s for s in allns if isdefined(m, s)])
+ # usinged_names = Set{Symbol}()
+
+ for n in availablenames
+ if (n in internal_names)
+ pop!(availablenames, n)
+ end
+ end
+ allms = get_all_modules()
+ for u in get_used_modules(m, allms)
+ for n in unsorted_names(u)
+ if n in availablenames
+ pop!(availablenames, n)
+ # push!(usinged_names, pop!(availablenames, n))
+ end
+ end
+ end
+ internal_names, availablenames
+end
+
+get_all_modules() = let allms = Base.IdSet{Module}(); apply_to_everything(x -> if x isa Module push!(allms, x) end); allms end
+get_used_modules(M, allms = get_all_modules()) = [m for m in allms if usedby(M, m)]
diff --git a/src/SymbolServer/utils.jl b/src/SymbolServer/utils.jl
new file mode 100644
index 0000000..4efdb7e
--- /dev/null
+++ b/src/SymbolServer/utils.jl
@@ -0,0 +1,688 @@
+using Pkg
+
+@static if VERSION < v"1.1"
+ const PackageEntry = Vector{Dict{String,Any}}
+else
+ using Pkg.Types: PackageEntry
+end
+
+@static if isdefined(Base, :parsed_toml)
+ parsed_toml(args...) = Base.parsed_toml(args...)
+else
+ parsed_toml(file) = Pkg.TOML.parsefile(file)
+end
+
+"""
+ manifest(c::Pkg.Types.Context)
+Retrieves the UUID -> PackageEntry map from the manifest of a Context.
+"""
+function manifest(c::Pkg.Types.Context)
+ m = c.env.manifest
+ if m isa Dict
+ return m
+ else
+ return m.deps
+ end
+end
+
+"""
+ read_manifest(manifest_filename)
+
+Read the manifest from the path and return the UUID -> PackageEntry map.
+If the file can't be read, return `nothing`.
+"""
+function read_manifest(manifest_filename)
+ try
+ m = Pkg.API.read_manifest(manifest_filename)
+ if m isa Dict
+ return m
+ else
+ return m.deps
+ end
+ catch err
+ @warn "Could not load manifest." exception=(err, catch_backtrace())
+ return nothing
+ end
+end
+
+"""
+ project(c::Pkg.Types.Context)
+Retrieves the project of a Context.
+"""
+project(c::Pkg.Types.Context) = c.env.project
+
+"""
+ isinproject(context, package::Union{String,UUID})
+Checks whether a package is in the dependencies of a given context, e.g. is directly loadable.
+"""
+function isinproject end
+
+"""
+ isinmanifest(context, package::Union{String,UUID})
+Checks whether a package is in the manifest of a given context, e.g. is either directly loadable or is a dependency of an loadable package.
+"""
+function isinmanifest end
+
+@static if VERSION < v"1.1"
+ isinmanifest(context::Pkg.Types.Context, module_name::String) = module_name in keys(manifest(context))
+ isinmanifest(context::Pkg.Types.Context, uuid::UUID) = any(get(p[1], "uuid", "") == string(uuid) for (u, p) in manifest(context))
+ isinmanifest(manifest::Dict{String,Any}, uuid::AbstractString) = any(get(p[1], "uuid", "") == uuid for (u, p) in manifest)
+ isinmanifest(manifest::Dict{String,Any}, uuid::UUID) = isinmanifest(manifest, string(uuid))
+
+ isinproject(context::Pkg.Types.Context, package_name::String) = haskey(deps(project(context)), package_name)
+ isinproject(context::Pkg.Types.Context, package_uuid::UUID) = any(u == package_uuid for (n, u) in deps(project(context)))
+
+ function packageuuid(c::Pkg.Types.Context, name::String)
+ for pkg in manifest(c)
+ if first(pkg) == name
+ return UUID(last(pkg)[1]["uuid"])
+ end
+ end
+ end
+ packageuuid(pkg::Pair{Any,Any}) = last(pkg) isa String ? UUID(last(pkg)) : UUID(first(last(pkg))["uuid"])
+ packageuuid(pkg::Pair{String,Any}) = last(pkg) isa String ? UUID(last(pkg)) : UUID(first(last(pkg))["uuid"])
+
+ packagename(pkg::Pair{String,Any})::String = first(pkg)
+ function packagename(c::Pkg.Types.Context, uuid)
+ for (n, p) in manifest(c)
+ if get(first(p), "uuid", "") == string(uuid)
+ return n
+ end
+ end
+ return nothing
+ end
+ function packagename(manifest::Dict{String,Any}, uuid::String)
+ for (n, p) in manifest
+ if get(first(p), "uuid", "") == string(uuid)
+ return n
+ end
+ end
+ return nothing
+ end
+ packagename(manifest::Dict{String,Any}, uuid::UUID) = packagename(manifest, string(uuid))
+
+ function deps(uuid::UUID, c::Pkg.Types.Context)
+ if any(p[1]["uuid"] == string(uuid) for (n, p) in manifest(c))
+ return manifest(c)[string(uuid)][1].deps
+ else
+ return Dict{Any,Any}()
+ end
+ end
+ deps(d::Dict{String,Any}) = get(d, "deps", Dict{String,Any}())
+ deps(pe::PackageEntry) = get(pe[1], "deps", Dict{String,Any}())
+ path(pe::PackageEntry) = get(pe[1], "path", nothing)
+ version(pe::PackageEntry) = get(pe[1], "version", nothing)
+ tree_hash(pe) = get(pe[1], "git-tree-sha1", nothing)
+
+ frommanifest(c::Pkg.Types.Context, uuid) = frommanifest(manifest(c), uuid)
+
+ function frommanifest(manifest::Dict{String,Any}, uuid)
+ for p in values(manifest)
+ if get(first(p), "uuid", "") == string(uuid)
+ return (p)
+ end
+ end
+ return nothing
+ end
+ is_package_deved(manifest, uuid) = get(first([p[2][1] for p in manifest if get(p[2][1], "uuid", "") == string(uuid)]), "path", "") != ""
+else
+ isinmanifest(context::Pkg.Types.Context, module_name::String) = any(p.name == module_name for (u, p) in manifest(context))
+ isinmanifest(context::Pkg.Types.Context, uuid::UUID) = haskey(manifest(context), uuid)
+ isinmanifest(manifest::Dict{UUID,PackageEntry}, uuid::UUID) = haskey(manifest, uuid)
+
+ isinproject(context::Pkg.Types.Context, package_name::String) = haskey(deps(project(context)), package_name)
+ isinproject(context::Pkg.Types.Context, package_uuid::UUID) = any(u == package_uuid for (n, u) in deps(project(context)))
+
+ function packageuuid(c::Pkg.Types.Context, name::String)
+ for pkg in manifest(c)
+ if last(pkg).name == name
+ return first(pkg)
+ end
+ end
+ end
+ packageuuid(pkg::Pair{String,UUID}) = last(pkg)
+ packageuuid(pkg::Pair{UUID,PackageEntry}) = first(pkg)
+
+ packagename(pkg::Pair{UUID,PackageEntry})::Union{Nothing,String} = last(pkg).name
+ packagename(c::Pkg.Types.Context, uuid::UUID) = manifest(c)[uuid].name
+ packagename(manifest::Dict{UUID,PackageEntry}, uuid::UUID) = manifest[uuid].name
+
+ function deps(uuid::UUID, c::Pkg.Types.Context)
+ if haskey(manifest(c), uuid)
+ return deps(manifest(c)[uuid])
+ else
+ return Dict{String,Base.UUID}()
+ end
+ end
+ deps(pe::PackageEntry) = pe.deps
+ deps(proj::Pkg.Types.Project) = proj.deps
+ deps(pkg::Pair{String,UUID}, c::Pkg.Types.Context) = deps(packageuuid(pkg), c)
+ path(pe::PackageEntry) = pe.path
+ version(pe::PackageEntry) = pe.version
+ version(pe::Pair{UUID,PackageEntry}) = last(pe).version
+ frommanifest(c::Pkg.Types.Context, uuid) = manifest(c)[uuid]
+ frommanifest(manifest::Dict{UUID,PackageEntry}, uuid) = manifest[uuid]
+ tree_hash(pkg::Pair{UUID,PackageEntry}) = tree_hash(last(pkg))
+
+ @static if VERSION >= v"1.3"
+ tree_hash(pe::PackageEntry) = pe.tree_hash
+ else
+ tree_hash(pe::PackageEntry) = (pe.other === nothing ? nothing : get(pe.other, "git-tree-sha1", nothing))
+ end
+
+ is_package_deved(manifest, uuid) = manifest[uuid].path !== nothing
+end
+
+function sha2_256_dir(path, sha=zeros(UInt8, 32))
+ (uperm(path) & 0x04) != 0x04 && return
+ startswith(path, ".") && return
+ if isfile(path) && endswith(path, ".jl")
+ s1 = open(path) do f
+ sha2_256(f)
+ end
+ sha .+= s1
+ elseif isdir(path)
+ for f in readdir(path)
+ sha = sha2_256_dir(joinpath(path, f), sha)
+ end
+ end
+ return sha
+end
+
+function sha_pkg(manifest_dir::AbstractString, pe::PackageEntry)
+ relpath = path(pe)
+ isa(relpath, String) || return nothing
+ src_path = normpath(joinpath(manifest_dir, relpath, "src"))
+ return isdir(src_path) ? sha2_256_dir(src_path) : nothing
+end
+
+function _doc(binding::Base.Docs.Binding)
+ try
+ sig = Union{}
+ if Base.Docs.defined(binding)
+ result = Base.Docs.getdoc(Base.Docs.resolve(binding), sig)
+ result === nothing || return string(result)
+ end
+ results, groups = Base.Docs.DocStr[], Base.Docs.MultiDoc[]
+ # Lookup `binding` and `sig` for matches in all modules of the docsystem.
+ for mod in Base.Docs.modules
+ dict = Base.Docs.meta(mod)::IdDict{Any,Any}
+ if haskey(dict, binding)
+ multidoc = dict[binding]
+ push!(groups, multidoc)
+ for msig in multidoc.order
+ sig <: msig && push!(results, multidoc.docs[msig])
+ end
+ end
+ end
+ if isempty(results)
+ for group in groups, each in group.order
+ push!(results, group.docs[each])
+ end
+ end
+ md = try
+ Base.Docs.catdoc(map(Base.Docs.parsedoc, results)...)
+ catch
+ nothing
+ end
+ return md === nothing ? "" : string(md)
+ catch
+ return ""
+ end
+end
+
+_lookup(vr::FakeUnion, depot::EnvStore, cont=false) = nothing
+_lookup(vr::FakeTypeName, depot::EnvStore, cont=false) = _lookup(vr.name, depot, cont)
+_lookup(vr::FakeUnionAll, depot::EnvStore, cont=false) = _lookup(vr.body, depot, cont)
+function _lookup(vr::VarRef, depot::EnvStore, cont=false)
+ if vr.parent === nothing
+ if haskey(depot, vr.name)
+ val = depot[vr.name]
+ if cont && val isa VarRef
+ return _lookup(val, depot, cont)
+ else
+ return val
+ end
+ else
+ return nothing
+ end
+ else
+ par = _lookup(vr.parent, depot, cont)
+ if par !== nothing && par isa ModuleStore && haskey(par, vr.name)
+ val = par[vr.name]
+ if cont && val isa VarRef
+ return _lookup(val, depot, cont)
+ else
+ return val
+ end
+ else
+ return nothing
+ end
+ end
+end
+
+maybe_lookup(x, env) = x isa VarRef ? _lookup(x, env, true) : x
+
+"""
+ maybe_getfield(k::Symbol , m::ModuleStore, server)
+
+Try to get `k` from `m`. This includes: unexported variables, and variables
+exported by modules used within `m`.
+"""
+function maybe_getfield(k::Symbol, m::ModuleStore, envstore)
+ if haskey(m.vals, k)
+ return m.vals[k]
+ else
+ for v in m.used_modules
+ !haskey(m.vals, v) && continue
+ submod = m.vals[v]
+ if submod isa ModuleStore && k in submod.exportednames && haskey(submod.vals, k)
+ return submod.vals[k]
+ elseif submod isa VarRef
+ submod = _lookup(submod, envstore, true)
+ if submod isa ModuleStore && k in submod.exportednames && haskey(submod.vals, k)
+ return submod.vals[k]
+ end
+ end
+ end
+ end
+end
+
+function issubmodof(m::Module, M::Module)
+ if m == M
+ return true
+ elseif parentmodule(m) === m
+ return false
+ elseif parentmodule(m) == M
+ return true
+ else
+ return issubmodof(parentmodule(m), M)
+ end
+end
+
+function Base.print(io::IO, f::FunctionStore)
+ println(io, f.name, " is a Function.")
+ nm = length(f.methods)
+ println(io, "# $nm method", nm == 1 ? "" : "s", " for function ", f.name)
+ for i = 1:nm
+ print(io, "[$i] ")
+ println(io, f.methods[i])
+ end
+end
+
+const JULIA_DIR = normpath(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia"))
+
+function Base.print(io::IO, m::MethodStore)
+ print(io, m.name, "(")
+ for i = 1:length(m.sig)
+ if m.sig[i][1] != Symbol("#unused#")
+ print(io, m.sig[i][1])
+ end
+ print(io, "::", m.sig[i][2])
+ i != length(m.sig) && print(io, ", ")
+ end
+ print(io, ")")
+ path = replace(m.file, JULIA_DIR => "")
+ print(io, " in ", m.mod, " at ", path, ':', m.line)
+end
+
+function Base.print(io::IO, t::DataTypeStore)
+ print(io, t.name, " <: ", t.super)
+ for i = 1:length(t.fieldnames)
+ print(io, "\n ", t.fieldnames[i], "::", t.types[i])
+ end
+end
+
+Base.print(io::IO, m::ModuleStore) = print(io, m.name)
+Base.print(io::IO, x::GenericStore) = print(io, x.name, "::", x.typ)
+
+extends_methods(f) = false
+extends_methods(f::FunctionStore) = f.name != f.extends
+get_top_module(vr::VarRef) = vr.parent === nothing ? vr.name : get_top_module(vr.parent)
+
+# Sorting is the main performance of calling `names`
+unsorted_names(m::Module; all::Bool=false, imported::Bool=false) =
+ ccall(:jl_module_names, Array{Symbol,1}, (Any, Cint, Cint), m, all, imported)
+
+## recursive_copy
+#
+# `deepcopy` is reliable but incredibly slow. Its slowness comes from two factors:
+# - generically iterating over, e.g., `fieldnames(typeof(x))` rather than having a method
+# optimized for each struct type
+# - its care to protect against circular depenency graphs
+# When you don't need to worry about cycles, you can do much better by defining your own function.
+
+recursive_copy(x) = deepcopy(x)
+
+recursive_copy(::Nothing) = nothing
+
+recursive_copy(s::Symbol) = s
+
+recursive_copy(c::Char) = c
+
+recursive_copy(str::String) = str
+
+recursive_copy(x::Number) = x
+
+recursive_copy(p::Pair) = typeof(p)(recursive_copy(p.first), recursive_copy(p.second))
+
+recursive_copy(A::Array) = eltype(A)[recursive_copy(a) for a in A]
+
+recursive_copy(d::Dict) = typeof(d)(recursive_copy(p) for p in d)
+
+
+recursive_copy(ref::VarRef) = VarRef(recursive_copy(ref.parent), ref.name)
+
+recursive_copy(tn::FakeTypeName) = FakeTypeName(recursive_copy(tn.name), recursive_copy(tn.parameters))
+
+recursive_copy(tb::FakeTypeofBottom) = tb
+
+recursive_copy(u::FakeUnion) = FakeUnion(recursive_copy(u.a), recursive_copy(u.b))
+
+recursive_copy(tv::FakeTypeVar) = FakeTypeVar(tv.name, recursive_copy(tv.lb), recursive_copy(tv.ub))
+
+recursive_copy(ua::FakeUnionAll) = FakeUnionAll(recursive_copy(ua.var), recursive_copy(ua.body))
+
+@static if !(Vararg isa Type)
+ function recursive_copy(va::FakeTypeofVararg)
+ if isdefined(va, :N)
+ FakeTypeofVararg(recursive_copy(va.T), va.N)
+ elseif isdefined(va, :T)
+ FakeTypeofVararg(recursive_copy(va.T))
+ else
+ FakeTypeofVararg()
+ end
+ end
+end
+
+recursive_copy(m::ModuleStore) = ModuleStore(recursive_copy(m.name), recursive_copy(m.vals), m.doc,
+ m.exported, copy(m.exportednames), copy(m.used_modules))
+
+recursive_copy(p::Package) = Package(p.name,
+ recursive_copy(p.val),
+ p.uuid,
+ recursive_copy(p.sha))
+
+recursive_copy(ms::MethodStore) = MethodStore(ms.name,
+ ms.mod,
+ ms.file,
+ ms.line,
+ recursive_copy(ms.sig),
+ copy(ms.kws),
+ recursive_copy(ms.rt))
+
+recursive_copy(dts::DataTypeStore) = DataTypeStore(recursive_copy(dts.name),
+ recursive_copy(dts.super),
+ recursive_copy(dts.parameters),
+ recursive_copy(dts.types),
+ recursive_copy(dts.fieldnames),
+ recursive_copy(dts.methods),
+ dts.doc,
+ dts.exported)
+
+recursive_copy(fs::FunctionStore) = FunctionStore(recursive_copy(fs.name),
+ recursive_copy(fs.methods),
+ fs.doc,
+ recursive_copy(fs.extends),
+ fs.exported)
+
+recursive_copy(gs::GenericStore) = GenericStore(recursive_copy(gs.name),
+ recursive_copy(gs.typ),
+ gs.doc,
+ gs.exported)
+
+
+# Tools for modifying source location
+# env = getenvtree([:somepackage])
+# symbols(env, somepackage)
+# m = env[:somepackage]
+# To strip actual src path:
+# modify_dirs(m, f -> modify_dir(f, pkg_src_dir(somepackage), "PLACEHOLDER"))
+# To replace the placeholder:
+# modify_dirs(m, f -> modify_dir(f, "PLACEHOLDER", new_src_dir))
+function modify_dirs(m::ModuleStore, f)
+ for (k, v) in m.vals
+ if v isa FunctionStore
+ m.vals[k] = FunctionStore(v.name, MethodStore[MethodStore(m.name, m.mod, f(m.file), m.line, m.sig, m.kws, m.rt) for m in v.methods], v.doc, v.extends, v.exported)
+ elseif v isa DataTypeStore
+ m.vals[k] = DataTypeStore(v.name, v.super, v.parameters, v.types, v.fieldnames, MethodStore[MethodStore(m.name, m.mod, f(m.file), m.line, m.sig, m.kws, m.rt) for m in v.methods], v.doc, v.exported)
+ elseif v isa ModuleStore
+ modify_dirs(v, f)
+ end
+ end
+end
+
+pkg_src_dir(m::Module) = dirname(pathof(m))
+
+# replace s1 with s2 at the start of a string
+function modify_dir(f, s1, s2)
+ # @assert startswith(f, s1)
+ # Removed assertion because of Enums issue
+ replace(f, s1 => s2)
+end
+
+
+# tools to retrieve cache from the cloud
+
+function get_file_from_cloud(manifest, uuid, environment_path, depot_dir, cache_dir="../cache", download_dir="../downloads/", symbolcache_upstream="https://www.julia-vscode.org/symbolcache")
+ paths = get_cache_path(manifest, uuid)
+ name = packagename(manifest, uuid)
+ link = string(first(splitext(join([symbolcache_upstream, "store/v1/packages", paths...], '/'))), ".tar.gz")
+
+ dest_filepath = joinpath(cache_dir, paths...)
+ dest_filepath_unavailable = string(first(splitext(dest_filepath)), ".unavailable")
+
+ download_dir = joinpath(download_dir, first(splitext(last(paths))))
+ download_filepath = joinpath(download_dir, last(paths))
+ download_filepath_unavailable = string(first(splitext(download_filepath)), ".unavailable")
+
+ @debug "Downloading cache file for $name."
+ if isfile(dest_filepath_unavailable)
+ @debug "Cloud was unable to cache $name in the past, we won't try to retrieve it again."
+ return false
+ end
+ file = try
+ if Pkg.PlatformEngines.download_verify_unpack(link, nothing, download_dir)
+ mkpath(dirname(dest_filepath))
+ if !isfile(download_filepath) && isfile(download_filepath_unavailable)
+ mv(download_filepath_unavailable, dest_filepath_unavailable)
+ @info "Cloud is unable to cache $name, we won't try to retrieve it again."
+ return false
+ end
+ mv(download_filepath, dest_filepath)
+ dest_filepath
+ else
+ @debug "Couldn't retrieve cache file for $name."
+ return false
+ end
+ catch err
+ @debug "Couldn't retrieve cache file for $name." exception = (err, catch_backtrace())
+ return false
+ end
+
+ cache = try
+ open(file, "r") do io
+ CacheStore.read(io)
+ end
+ catch
+ @warn "Couldn't read cache file for $name, deleting."
+ rm(file)
+ return false
+ end
+
+ pkg_entry = Base.locate_package(Base.PkgId(uuid, name))
+ if pkg_entry !== nothing && isfile(pkg_entry)
+ pkg_src = dirname(pkg_entry)
+ else
+ pkg_root = get_pkg_path(Base.PkgId(uuid, name), environment_path, depot_dir)
+ if pkg_root === nothing
+ @debug "Successfully downloaded and saved $(name), but with placeholder paths"
+ return false
+ end
+ pkg_src = joinpath(pkg_root, "src")
+ end
+
+ # TODO: it would be better if the PLACEHOLDER replacement happens at runtime
+ # instead of "unpack-time", because we can use the current depot path
+ # in case the user switched to another one after downloading
+
+ @debug "Replacing PLACEHOLDER with:" pkg_src
+ modify_dirs(cache.val, f -> modify_dir(f, r"^PLACEHOLDER", pkg_src))
+ open(file, "w") do io
+ CacheStore.write(io, cache)
+ end
+
+ @debug "Successfully downloaded, scrubbed and saved $(name)"
+ return true
+end
+
+"""
+ validate_disc_store(store_path, manifest)
+
+This returns a list of non-jll packages in the manifest that don't have caches on disc.
+"""
+function validate_disc_store(store_path, manifest)
+ filter(manifest) do pkg
+ uuid = packageuuid(pkg)
+ endswith(packagename(manifest, uuid), "_jll") && return false
+
+ file_name = joinpath(get_cache_path(manifest, uuid)...)
+ yield()
+ return !isfile(joinpath(store_path, file_name))
+ end
+end
+
+function find_project_file(env)
+ isdir(env) || return false
+ for filename in ("Project.toml", "JuliaProject.toml")
+ maybe_project_file = joinpath(env, filename)
+ if isfile(maybe_project_file)
+ return maybe_project_file
+ end
+ end
+ return false
+end
+
+"""
+ get_pkg_path(pkg::Base.PkgId, env, depot_path)
+
+Find out where a package is installed without having to load it.
+"""
+function get_pkg_path(pkg::Base.PkgId, env, depot_path)
+ project_file = find_project_file(env)
+ project_file isa Bool && return nothing
+ manifest_file = Base.project_file_manifest_path(project_file)
+
+ d = parsed_toml(manifest_file)
+ if get(d, "manifest_format", "0.0") == "2.0"
+ entries = get(d, "deps", nothing)
+ entries === nothing && return nothing
+ entries = map(e -> e[1], values(entries))
+ else
+ entries = get(d, pkg.name, nothing)
+ end
+ entries === nothing && return nothing # TODO: allow name to mismatch?
+ for entry in entries
+ entry = entry::Dict{String,Any}
+ uuid = get(entry, "uuid", nothing)::Union{Nothing,String}
+ uuid === nothing && continue
+ if UUID(uuid) === pkg.uuid
+ path = get(entry, "path", nothing)::Union{Nothing,String}
+ # this can only be true for explicitly dev'ed packages
+ if path !== nothing
+ path = normpath(abspath(dirname(manifest_file), path))
+ return path
+ end
+ hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing,String}
+ hash === nothing && return nothing
+ hash = Base.SHA1(hash)
+ # empty default path probably means that we should use the default Julia depots
+ if depot_path == ""
+ depot_paths = []
+ if isdefined(Base, :append_default_depot_path!)
+ Base.append_default_depot_path!(depot_paths)
+ else
+ depot_paths = Pkg.depots()
+ end
+ else
+ depot_paths = [depot_path]
+ end
+ for depot in depot_paths
+ # Keep the 4 since it used to be the default
+ for slug in (Base.version_slug(pkg.uuid, hash, 4), Base.version_slug(pkg.uuid, hash))
+ path = abspath(depot, "packages", pkg.name, slug)
+ ispath(path) && return path
+ end
+ end
+ return nothing
+ end
+ end
+ return nothing
+end
+
+function load_package(c::Pkg.Types.Context, uuid, conn, loadingbay, percentage = missing)
+ isinmanifest(c, uuid isa String ? Base.UUID(uuid) : uuid) || return
+ pe_name = packagename(c, uuid)
+
+ pid = Base.PkgId(uuid isa String ? Base.UUID(uuid) : uuid, pe_name)
+ if pid in keys(Base.loaded_modules)
+ conn !== nothing && println(conn, "PROCESSPKG;$pe_name;$uuid;noversion;$percentage")
+ loadingbay.eval(:($(Symbol(pe_name)) = $(Base.loaded_modules[pid])))
+ m = getfield(loadingbay, Symbol(pe_name))
+ else
+ m = try
+ conn !== nothing && println(conn, "STARTLOAD;$pe_name;$uuid;noversion;$percentage")
+ loadingbay.eval(:(import $(Symbol(pe_name))))
+ conn !== nothing && println(conn, "STOPLOAD;$pe_name")
+ m = getfield(loadingbay, Symbol(pe_name))
+ catch
+ return
+ end
+ end
+end
+
+function write_cache(uuid, pkg::Package, outpath)
+ mkpath(dirname(outpath))
+ @info "Now writing to disc $uuid"
+ open(outpath, "w") do io
+ CacheStore.write(io, pkg)
+ end
+ outpath
+end
+
+"""
+ get_cache_path(manifest, uuid)
+
+Returns a vector containing the cache storage path for a package structured: [folder, folder, file].
+"""
+function get_cache_path(manifest, uuid)
+ name = packagename(manifest, uuid)
+ pkg_info = frommanifest(manifest, uuid)
+ ver = version(pkg_info)
+ if ver === nothing
+ ver = "nothing"
+ if isdefined(Pkg.Types, :is_stdlib) && Pkg.Types.is_stdlib(uuid)
+ ver = VERSION
+ end
+ end
+ ver = replace(string(ver), '+'=>'_')
+ th = tree_hash(pkg_info)
+ th = th === nothing ? "nothing" : th
+
+ [
+ string(uppercase(string(name)[1]))
+ string(name, "_", uuid)
+ string("v", ver, "_", th, ".jstore")
+ ]
+end
+
+function write_depot(server::Server, ctx, written_caches)
+ for (uuid, pkg) in server.depot
+ cache_paths = get_cache_path(manifest(ctx), uuid)
+ outpath = joinpath(server.storedir, cache_paths...)
+ outpath in written_caches && continue
+
+ written_path = write_cache(uuid, pkg, outpath)
+ !isempty(written_path) && push!(written_caches, written_path)
+ end
+end
diff --git a/src/fileio.jl b/src/fileio.jl
index 2b9e950..db634c1 100644
--- a/src/fileio.jl
+++ b/src/fileio.jl
@@ -133,8 +133,8 @@ function add_folder_from_disc!(jw::JuliaWorkspace, path; ignore_io_errors=false)
end
end
-function workspace_from_folders(workspace_folders::Vector{String})
- jw = JuliaWorkspace()
+function workspace_from_folders(workspace_folders::Vector{String}; symbol_cache_path=nothing, async_symbol_loading=false)
+ jw = JuliaWorkspace(; symbol_cache_path=symbol_cache_path, async_symbol_loading=async_symbol_loading)
for folder in workspace_folders
add_folder_from_disc!(jw, folder)
diff --git a/src/inputs.jl b/src/inputs.jl
index dd8ac1d..88b17f1 100644
--- a/src/inputs.jl
+++ b/src/inputs.jl
@@ -4,3 +4,5 @@ Salsa.@declare_input input_notebook_file(rt, uri)::NotebookFile
Salsa.@declare_input input_fallback_test_project(rt)::Union{URI,Nothing}
Salsa.@declare_input input_marked_diagnostics(rt)::DiagnosticsMark
Salsa.@declare_input input_marked_testitems(rt)::TestitemsMark
+Salsa.@declare_input input_package_symbols(rt)::Set{JuliaProjectEntryRegularPackage}
+Salsa.@declare_input input_symbols_for_package(rt, JuliaProjectEntryRegularPackage)::@NamedTuple{status::Symbol,data}
diff --git a/src/layer_symbols.jl b/src/layer_symbols.jl
new file mode 100644
index 0000000..33dd1a5
--- /dev/null
+++ b/src/layer_symbols.jl
@@ -0,0 +1,7 @@
+Salsa.@derived function derived_required_symbol_info(rt)
+ all_projects = [derived_project(rt, i) for i in derived_project_folders(rt)]
+
+ regular_packages = unique(Iterators.flatten(values(i.regular_packages) for i in all_projects))
+
+ return (;regular_packages)
+end
diff --git a/src/public.jl b/src/public.jl
index 717717b..f5d016e 100644
--- a/src/public.jl
+++ b/src/public.jl
@@ -36,6 +36,34 @@ function add_file!(jw::JuliaWorkspace, file::TextFile)
set_input_files!(jw.runtime, new_files)
set_input_text_file!(jw.runtime, file.uri, file)
+
+ if jw.symbol_cache_path!==nothing
+ required_symbols = derived_required_symbol_info(jw.runtime)
+ already_loaded_symbols = input_package_symbols(jw.runtime)
+ still_need_to_be_loaded = setdiff(required_symbols.regular_packages, already_loaded_symbols)
+
+ for i in still_need_to_be_loaded
+ set_input_symbols_for_package!(jw.runtime, i, @NamedTuple{status::Symbol,data}((:loading, nothing)))
+ end
+
+ new_already_loaded_symbols = Set{JuliaProjectEntryRegularPackage}(union(already_loaded_symbols, still_need_to_be_loaded))
+ set_input_package_symbols!(jw.runtime, new_already_loaded_symbols)
+
+ put!(jw.symbol_cache_channel_requests, still_need_to_be_loaded)
+ if !jw.symbol_cache_async
+ new_symbols = take!(jw.symbol_cache_channel_responses)
+
+ for i in new_symbols
+ if i.data===nothing
+ set_input_symbols_for_package!(jw.runtime, i.package, @NamedTuple{status::Symbol,data}((:unavailable, nothing)))
+ else
+ set_input_symbols_for_package!(jw.runtime, i.package, @NamedTuple{status::Symbol,data}((:loaded, i.data)))
+ end
+ end
+ end
+ end
+
+ # println("Done with add_file!")
end
function update_file!(jw::JuliaWorkspace, file::TextFile)
diff --git a/src/types.jl b/src/types.jl
index 4ebbe8a..e3346c8 100644
--- a/src/types.jl
+++ b/src/types.jl
@@ -119,13 +119,72 @@ end
struct JuliaWorkspace
runtime::Salsa.Runtime
- function JuliaWorkspace()
+ symbol_cache_path::Union{Nothing,String}
+ symbol_cache_channel_requests::Union{Nothing,Channel{Any}}
+ symbol_cache_channel_responses::Union{Nothing,Channel{Any}}
+ symbol_cache_async::Bool
+
+ function JuliaWorkspace(; symbol_cache_path=nothing, async_symbol_loading=true)
rt = Salsa.Runtime()
set_input_files!(rt, Set{URI}())
+ set_input_package_symbols!(rt, Set{JuliaProjectEntryRegularPackage}())
set_input_fallback_test_project!(rt, nothing)
- new(rt)
+ symbol_cache_channel_requests = symbol_cache_path===nothing ? nothing : Channel(Inf)
+ symbol_cache_channel_responses = symbol_cache_path===nothing ? nothing : Channel(Inf)
+
+ if symbol_cache_path!==nothing
+ Threads.@spawn try
+ while true
+ still_need_to_be_loaded = take!(symbol_cache_channel_requests)
+
+ new_symbols = map(still_need_to_be_loaded) do i
+ # Construct cache path
+ file_to_load_path = joinpath(
+ symbol_cache_path,
+ string(uppercase(string(i.name)[1])), # Capitalized first letter of the package name
+ string(i.name, "_", i.uuid),
+ string("v", i.version, "_", i.git_tree_sha1, ".jstore")
+ )
+
+ if isfile(file_to_load_path)
+ # println("We found cache file $file_to_load_path")
+ package_data = open(file_to_load_path) do io
+ SymbolServer.CacheStore.read(io)
+ end
+
+ pkg_path = nothing
+
+ git_tree_sha1 = Base.SHA1(i.git_tree_sha1)
+ # Keep the 4 since it used to be the default
+ slugs = (Base.version_slug(i.uuid, git_tree_sha1, 4), Base.version_slug(i.uuid, git_tree_sha1))
+ for depot in Base.DEPOT_PATH, slug in slugs
+ path = abspath(depot, "packages", i.name, slug)
+ if ispath(path)
+ pkg_path = path
+ break
+ end
+ end
+
+ if pkg_path !== nothing
+ SymbolServer.modify_dirs(package_data.val, f -> SymbolServer.modify_dir(f, r"^PLACEHOLDER", joinpath(pkg_path, "src")))
+ end
+
+ return @NamedTuple{package,data}((i, package_data))
+ end
+
+ return @NamedTuple{package,data}((i, nothing))
+ end
+
+ put!(symbol_cache_channel_responses, new_symbols)
+ end
+ catch err
+ Base.display_error(err, catch_backtrace())
+ end
+ end
+
+ new(rt, symbol_cache_path, symbol_cache_channel_requests, symbol_cache_channel_responses, async_symbol_loading)
end
end