Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 38 additions & 38 deletions registryindexer/main.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

max_n = 1_000_000
max_versions = 1_000_000
max_tasks = length(ARGS)>1 ? parse(Int, ARGS[2]) : 1
max_tasks = length(ARGS) > 1 ? parse(Int, ARGS[2]) : 1

julia_versions = [v"1.10.4"]

Expand All @@ -13,26 +13,26 @@ Pkg.instantiate()

using ProgressMeter, Query, JSON, UUIDs, Tar, CodecZlib

function get_all_package_versions(;max_versions=typemax(Int))
function get_all_package_versions(; max_versions=typemax(Int))
registry_folder_path = joinpath(homedir(), ".julia", "registries", "General")
registry_path = joinpath(registry_folder_path, "Registry.toml")

registry_content = Pkg.TOML.parsefile(registry_path)

packages = registry_content["packages"] |>
@map({
name = _[2]["name"],
uuid = UUID(_[1]),
path = _[2]["path"]
}) |>
@mutate(
versions = (Pkg.TOML.parsefile(joinpath(registry_folder_path, _.path, "Versions.toml")) |>
@map(i->{version=VersionNumber(i[1]), treehash=i[2]["git-tree-sha1"]}) |>
@orderby_descending(i->i.version) |>
@take(max_versions) |>
collect)
) |>
collect
@map({
name = _[2]["name"],
uuid = UUID(_[1]),
path = _[2]["path"]
}) |>
@mutate(
versions = (Pkg.TOML.parsefile(joinpath(registry_folder_path, _.path, "Versions.toml")) |>
@map(i -> {version = VersionNumber(i[1]), treehash = i[2]["git-tree-sha1"]}) |>
@orderby_descending(i -> i.version) |>
@take(max_versions) |>
collect)
) |>
collect
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The indentation here seems to fall back to the align defaults, whereas it should just be based on indent levels.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

weird, this looks like a bug


return packages
end
Expand All @@ -42,7 +42,7 @@ function get_flattened_package_versions(packages)

for p in packages
for v in p.versions
push!(flattened_packageversions, (;p.name, p.uuid, p.path, version=v.version, treehash=v.treehash))
push!(flattened_packageversions, (; p.name, p.uuid, p.path, version=v.version, treehash=v.treehash))
end
end

Expand All @@ -54,11 +54,11 @@ function execute(cmd::Base.Cmd)
err = IOBuffer()
process = run(pipeline(ignorestatus(cmd), stdout=out, stderr=err))

out_string =String(take!(out))
out_string = String(take!(out))
err_string = String(take!(err))
return (stdout = out_string,
stderr = err_string,
code = process.exitcode)
return (stdout=out_string,
stderr=err_string,
code=process.exitcode)
end

@info "Indexing started..."
Expand All @@ -69,7 +69,7 @@ flattened_packageversions = get_flattened_package_versions(all_packages)

@info "Loaded package versions from registry..."

cache_folder = length(ARGS)>0 ? ARGS[1] : joinpath(@__DIR__, "..", "registryindexcache")
cache_folder = length(ARGS) > 0 ? ARGS[1] : joinpath(@__DIR__, "..", "registryindexcache")

@info "Using the following folder as the cache folder: " cache_folder

Expand All @@ -92,7 +92,7 @@ asyncmap(julia_versions) do v
print(f, res.stderr)
end

if res.code!=0
if res.code != 0
error("Could not create docker image.")
end
end
Expand All @@ -108,10 +108,10 @@ true || asyncmap(julia_versions) do v
else
res = execute(`docker run --rm --mount type=bind,source="$cache_folder",target=/symcache juliavscodesymbolindexer:$v julia SymbolServer/src/indexbasestdlib.jl $v`)

if res.code==10 || res.code==20
if res.code==10
if res.code == 10 || res.code == 20
if res.code == 10
# global count_failed_to_load += 1
elseif res.code==20
elseif res.code == 20
# global count_failed_to_install += 1
end

Expand All @@ -121,7 +121,7 @@ true || asyncmap(julia_versions) do v
# # Write them to a file
# open(joinpath(path, error_filename), "w") do io
# end

# Pkg.PlatformEngines.package(path, cache_path)
# end

Expand All @@ -136,7 +136,7 @@ true || asyncmap(julia_versions) do v
# global status_db

# push!(status_db, Dict("name"=>v.name, "uuid"=>string(v.uuid), "version"=>string(v.version), "treehash"=>v.treehash, "status"=>res.code==20 ? "install_error" : "load_error", "indexattempts"=>[Dict("juliaversion"=>string(VERSION), "stdout"=>res.stdout, "stderr"=>res.stderr)]))
elseif res.code==0
elseif res.code == 0
# global count_successfully_cached += 1
else
# global count_failed_to_index += 1
Expand All @@ -154,7 +154,7 @@ end
@info "Now computing which of the total $(length(flattened_packageversions)) package versions that exist still need to be indexed..."

unindexed_packageversions = filter(collect(Iterators.take(flattened_packageversions, max_n))) do v
versionwithoutplus = replace(string(v.version), '+'=>'_')
versionwithoutplus = replace(string(v.version), '+' => '_')

cache_path = joinpath(cache_folder, "v1", "packages", string(uppercase(v.name[1])), "$(v.name)_$(v.uuid)", "v$(versionwithoutplus)_$(v.treehash).tar.gz")

Expand All @@ -179,7 +179,7 @@ status_db = isfile(statusdb_filename) ? JSON.parsefile(statusdb_filename) : []
@info "Starting the actual indexing process..."

asyncmap(unindexed_packageversions, ntasks=max_tasks) do v
versionwithoutplus = replace(string(v.version), '+'=>'_')
versionwithoutplus = replace(string(v.version), '+' => '_')

cache_path = joinpath(cache_folder, "v1", "packages", string(uppercase(v.name[1])), "$(v.name)_$(v.uuid)")
mkpath(cache_path)
Expand All @@ -188,12 +188,12 @@ asyncmap(unindexed_packageversions, ntasks=max_tasks) do v
mktempdir() do path
res = execute(`docker run --rm --mount type=bind,source="$path",target=/symcache juliavscodesymbolindexer:$(first(julia_versions)) julia SymbolServer/src/indexpackage.jl $(v.name) $(v.version) $(v.uuid) $(v.treehash)`)

if res.code==37 # This is our magic error code that indicates everything worked
if res.code == 37 # This is our magic error code that indicates everything worked
global count_successfully_cached += 1
else
if res.code==10
if res.code == 10
global count_failed_to_load += 1
elseif res.code==20
elseif res.code == 20
global count_failed_to_install += 1
else
global count_failed_to_index += 1
Expand All @@ -209,20 +209,20 @@ asyncmap(unindexed_packageversions, ntasks=max_tasks) do v
isfile(joinpath(path, error_filename)) && rm(joinpath(path, error_filename))

# Write them to a file
open(joinpath(path, error_filename), "w") do io
open(joinpath(path, error_filename), "w") do io
end

open(joinpath(cache_folder, "logs", res.code==10 ? "packageloadfailure" : res.code==20 ? "packageinstallfailure" : "packageindexfailure", "log_$(v.name)_v$(versionwithoutplus)_stdout.txt"), "w") do f
open(joinpath(cache_folder, "logs", res.code == 10 ? "packageloadfailure" : res.code == 20 ? "packageinstallfailure" : "packageindexfailure", "log_$(v.name)_v$(versionwithoutplus)_stdout.txt"), "w") do f
print(f, res.stdout)
end

open(joinpath(cache_folder, "logs", res.code==10 ? "packageloadfailure" : res.code==20 ? "packageinstallfailure" : "packageindexfailure", "log_$(v.name)_v$(versionwithoutplus)_stderr.txt"), "w") do f
open(joinpath(cache_folder, "logs", res.code == 10 ? "packageloadfailure" : res.code == 20 ? "packageinstallfailure" : "packageindexfailure", "log_$(v.name)_v$(versionwithoutplus)_stderr.txt"), "w") do f
print(f, res.stderr)
end

global status_db

push!(status_db, Dict("name"=>v.name, "uuid"=>string(v.uuid), "version"=>string(v.version), "treehash"=>v.treehash, "status"=>res.code==20 ? "install_error" : res.code==10 ? "load_error" : "index_error", "indexattempts"=>[Dict("juliaversion"=>string(VERSION), "stdout"=>res.stdout, "stderr"=>res.stderr)]))
push!(status_db, Dict("name" => v.name, "uuid" => string(v.uuid), "version" => string(v.version), "treehash" => v.treehash, "status" => res.code == 20 ? "install_error" : res.code == 10 ? "load_error" : "index_error", "indexattempts" => [Dict("juliaversion" => string(VERSION), "stdout" => res.stdout, "stderr" => res.stderr)]))
end

# @info "Files to be compressed" path readdir(path, join=true) ispath(cache_path) isfile(cache_path_compressed)
Expand All @@ -237,8 +237,8 @@ asyncmap(unindexed_packageversions, ntasks=max_tasks) do v
end
end

next!(p, showvalues = [
(:finished_package_count,p.counter+1),
next!(p, showvalues=[
(:finished_package_count, p.counter + 1),
(:count_successfully_cached, count_successfully_cached),
(:count_failed_to_install, count_failed_to_install),
(:count_failed_to_load, count_failed_to_load),
Expand Down
29 changes: 15 additions & 14 deletions src/SymbolServer.jl
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ mutable struct SymbolServerInstance
store_path::String
symbolcache_upstream::String

function SymbolServerInstance(depot_path::String="", store_path::Union{String,Nothing}=nothing, julia_exe::Union{NamedTuple{(:path,:version),Tuple{String,VersionNumber}},Nothing}=nothing; symbolcache_upstream = nothing)
function SymbolServerInstance(depot_path::String="", store_path::Union{String,Nothing}=nothing, julia_exe::Union{NamedTuple{(:path, :version),Tuple{String,VersionNumber}},Nothing}=nothing; symbolcache_upstream=nothing)
if symbolcache_upstream === nothing
symbolcache_upstream = "https://www.julia-vscode.org/symbolcache"
end
Expand Down Expand Up @@ -134,23 +134,23 @@ function download_cache_files(ssi, environment_path, progress_callback)
get_file_from_cloud(manifest, uuid, environment_path, ssi.depot_path, ssi.store_path, download_dir, ssi.symbolcache_upstream)
yield()
n_done += 1
percentage = round(Int, 100*(n_done/n_total))
percentage = round(Int, 100 * (n_done / n_total))
if percentage < 100
progress_callback("Downloading cache files...", percentage)
end
end
end
end
took = round(time() - t0, sigdigits = 2)
took = round(time() - t0, sigdigits=2)
progress_callback("All cache files downloaded (took $(took)s).", 100)
end
end
end

function getstore(ssi::SymbolServerInstance, environment_path::AbstractString, progress_callback=nothing, error_handler=nothing; download = false)
function getstore(ssi::SymbolServerInstance, environment_path::AbstractString, progress_callback=nothing, error_handler=nothing; download=false)
!ispath(environment_path) && return :success, recursive_copy(stdlibs)
_progress_callback = (msg, p) -> progress_callback === nothing ?
println(lpad(p, 4), "% - ", msg) : progress_callback(msg, p)
println(lpad(p, 4), "% - ", msg) : progress_callback(msg, p)

# see if we can download any package caches before local indexing
if download
Expand Down Expand Up @@ -241,9 +241,9 @@ function getstore(ssi::SymbolServerInstance, environment_path::AbstractString, p
end
cmd = Cmd(path)
if ssi.julia_exe_version > v"1.11-"
open(pipeline(Cmd(`$(cmd) --code-coverage=$(use_code_coverage==0 ? "none" : "user") --startup-file=no --compiled-modules=existing --history-file=no --project=$environment_path $server_script $(ssi.store_path) $pipename`, env=env_to_use), stderr=stderr), read=true, write=true)
open(pipeline(Cmd(`$(cmd) --code-coverage=$(use_code_coverage==0 ? "none" : "user") --startup-file=no --compiled-modules=existing --history-file=no --project=$environment_path $server_script $(ssi.store_path) $pipename`, env=env_to_use), stderr=stderr), read=true, write=true)
else
open(pipeline(Cmd(`$(cmd) --code-coverage=$(use_code_coverage==0 ? "none" : "user") --startup-file=no --compiled-modules=no --history-file=no --project=$environment_path $server_script $(ssi.store_path) $pipename`, env=env_to_use), stderr=stderr), read=true, write=true)
open(pipeline(Cmd(`$(cmd) --code-coverage=$(use_code_coverage==0 ? "none" : "user") --startup-file=no --compiled-modules=no --history-file=no --project=$environment_path $server_script $(ssi.store_path) $pipename`, env=env_to_use), stderr=stderr), read=true, write=true)
end
catch err
if stderr_for_client_process !== nothing
Expand All @@ -270,7 +270,7 @@ function getstore(ssi::SymbolServerInstance, environment_path::AbstractString, p
else
@debug "SymbolStore: store failure"
if currently_loading_a_package
return :package_load_crash, (package_name = current_package_name, stderr = stderr_for_client_process)
return :package_load_crash, (package_name=current_package_name, stderr=stderr_for_client_process)
else
return :failure, stderr_for_client_process
end
Expand All @@ -291,15 +291,16 @@ function pipe_name()
# Try to use /tmp and if that fails, hope the long pipe name works anyway
maybe = "/tmp/" * prefix * uuid
try
touch(maybe); rm(maybe) # Check permissions on this path
touch(maybe)
rm(maybe) # Check permissions on this path
pipename = maybe
catch
end
end
return pipename
end

function load_project_packages_into_store!(ssi::SymbolServerInstance, environment_path, store, progress_callback = nothing)
function load_project_packages_into_store!(ssi::SymbolServerInstance, environment_path, store, progress_callback=nothing)
project_filename = isfile(joinpath(environment_path, "JuliaProject.toml")) ? joinpath(environment_path, "JuliaProject.toml") : joinpath(environment_path, "Project.toml")
project = try
Pkg.API.read_project(project_filename)
Expand All @@ -317,7 +318,7 @@ function load_project_packages_into_store!(ssi::SymbolServerInstance, environmen
for (i, uuid) in enumerate(uuids)
load_package_from_cache_into_store!(ssi, uuid isa UUID ? uuid : UUID(uuid), environment_path, manifest, store, progress_callback, round(Int, 100 * (i - 1) / num_uuids))
end
took = round(time() - t0, sigdigits = 2)
took = round(time() - t0, sigdigits=2)
progress_callback("Loaded all packages into cache in $(took)s", 100)
end

Expand All @@ -326,7 +327,7 @@ end

Tries to load the on-disc stored cache for a package (uuid). Attempts to generate (and save to disc) a new cache if the file does not exist or is unopenable.
"""
function load_package_from_cache_into_store!(ssi::SymbolServerInstance, uuid::UUID, environment_path, manifest, store, progress_callback = nothing, percentage = missing)
function load_package_from_cache_into_store!(ssi::SymbolServerInstance, uuid::UUID, environment_path, manifest, store, progress_callback=nothing, percentage=missing)
yield()
isinmanifest(manifest, uuid) || return
pe = frommanifest(manifest, uuid)
Expand All @@ -353,7 +354,7 @@ function load_package_from_cache_into_store!(ssi::SymbolServerInstance, uuid::UU
end

store[Symbol(pe_name)] = package_data.val
took = round(time() - t0, sigdigits = 2)
took = round(time() - t0, sigdigits=2)
msg = "Done loading $pe_name from cache..."
if took > 0.01
msg *= " (took $(took)s)"
Expand Down Expand Up @@ -383,7 +384,7 @@ end
function clear_disc_store(ssi::SymbolServerInstance)
for f in readdir(ssi.store_path)
if occursin(f, "ABCDEFGHIJKLMNOPQRSTUVWXYZ")
rm(joinpath(ssi.store_path, f), recursive = true)
rm(joinpath(ssi.store_path, f), recursive=true)
end
end
end
Expand Down
2 changes: 1 addition & 1 deletion src/faketypes.jl
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ function Base.print(io::IO, tn::FakeTypeName)
end
Base.print(io::IO, x::FakeUnionAll) = print(io, x.body, " where ", x.var)
function Base.print(io::IO, x::FakeUnion; inunion=false)
!inunion && print(io, "Union{")
!inunion && print(io, "Union{")
print(io, x.a, ",")
if x.b isa FakeUnion
print(io, x.b, inunion=true)
Expand Down
4 changes: 2 additions & 2 deletions src/indexpackage.jl
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ current_package_treehash = ARGS[4]
# This path will always be mounted in the docker container in which we are running
store_path = "/symcache"

current_package_versionwithoutplus = replace(string(current_package_version), '+'=>'_')
current_package_versionwithoutplus = replace(string(current_package_version), '+' => '_')
filename_with_extension = "v$(current_package_versionwithoutplus)_$current_package_treehash.jstore"

module LoadingBay end
Expand Down Expand Up @@ -45,7 +45,7 @@ end
env = getenvtree([current_package_name])
symbols(env, m, get_return_type=true)

# Strip out paths
# Strip out paths
modify_dirs(env[current_package_name], f -> modify_dir(f, pkg_src_dir(Base.loaded_modules[Base.PkgId(current_package_uuid, string(current_package_name))]), "PLACEHOLDER"))

# There's an issue here - @enum used within CSTParser seems to add a method that is introduced from Enums.jl...
Expand Down
6 changes: 3 additions & 3 deletions src/serialize.jl
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ function write_vector(io, x)
end
end

function read(io, t = Base.read(io, UInt8))
function read(io, t=Base.read(io, UInt8))
# There are a bunch of `yield`s in potentially expensive code paths.
# One top-level `yield` would probably increase responsiveness in the
# LS, but increases runtime by 3x. This seems like a good compromise.
Expand Down Expand Up @@ -221,7 +221,7 @@ function read(io, t = Base.read(io, UInt8))
file = read(io)
line = Base.read(io, UInt32)
nsig = Base.read(io, Int)
sig = Vector{Pair{Any, Any}}(undef, nsig)
sig = Vector{Pair{Any,Any}}(undef, nsig)
for i in 1:nsig
sig[i] = read(io) => read(io)
end
Expand Down Expand Up @@ -259,7 +259,7 @@ function read(io, t = Base.read(io, UInt8))
false
elseif t === TupleHeader
N = Base.read(io, Int)
ntuple(i->read(io), N)
ntuple(i -> read(io), N)
elseif t === PackageHeader
yield()
name = read(io)
Expand Down
6 changes: 3 additions & 3 deletions src/server.jl
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ end
# Load all packages together
# This is important, or methods added to functions in other packages that are loaded earlier would not be in the cache
for (i, uuid) in enumerate(packages_to_load)
load_package(ctx, uuid, conn, LoadingBay, round(Int, 100*(i - 1)/length(packages_to_load)))
load_package(ctx, uuid, conn, LoadingBay, round(Int, 100 * (i - 1) / length(packages_to_load)))
end

# Create image of whole package env. This creates the module structure only.
Expand All @@ -109,8 +109,8 @@ visited = Base.IdSet{Module}([Base, Core])

for (pid, m) in Base.loaded_modules
if pid.uuid !== nothing && is_stdlib(pid.uuid) &&
isinmanifest(ctx, pid.uuid) &&
isfile(joinpath(server.storedir, SymbolServer.get_cache_path(manifest(ctx), pid.uuid)...))
isinmanifest(ctx, pid.uuid) &&
isfile(joinpath(server.storedir, SymbolServer.get_cache_path(manifest(ctx), pid.uuid)...))
push!(visited, m)
delete!(env_symbols, Symbol(pid.name))
end
Expand Down
Loading