diff --git a/base/loading.jl b/base/loading.jl index 7f6e6e5ace55eb..cfb141c266be42 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -690,7 +690,8 @@ function parse_cache_header(f::IO) push!(files, (modname, filename, ntoh(read(f, Float64)))) totbytes -= 8 + n1 + n2 + 8 end - @assert totbytes == 4 "header of cache file appears to be corrupt" + @assert totbytes == 12 "header of cache file appears to be corrupt" + srctextpos = ntoh(read(f, Int64)) # read the list of modules that are required to be present during loading required_modules = Dict{Symbol,UInt64}() while true @@ -700,7 +701,7 @@ function parse_cache_header(f::IO) uuid = ntoh(read(f, UInt64)) # module UUID required_modules[sym] = uuid end - return modules, files, required_modules + return modules, files, required_modules, srctextpos end function parse_cache_header(cachefile::String) @@ -728,6 +729,34 @@ function cache_dependencies(cachefile::String) end end +function read_dependency_src(io::IO, filename::AbstractString) + modules, files, required_modules, srctextpos = parse_cache_header(io) + filenames = map(x->x[2], files) + idx = findfirst(x->x==filename, filenames) + idx == 0 && error(filename, " not found in ", filenames) + srctextpos == 0 && error("source text for $filename not stored in cache file") + seek(io, srctextpos) + len = UInt64(0) + while idx > 0 + len = ntoh(read(io, UInt64)) + idx -= 1 + if idx > 0 + seek(io, position(io) + len) + end + end + String(read(io, len)) +end + +function read_dependency_src(cachefile::String, filename::AbstractString) + io = open(cachefile, "r") + try + !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile.")) + return read_dependency_src(io, filename) + finally + close(io) + end +end + function stale_cachefile(modpath::String, cachefile::String) io = open(cachefile, "r") try diff --git a/src/dump.c b/src/dump.c index 4a2a13ff8fa57c..0b668edd855675 100644 --- a/src/dump.c +++ b/src/dump.c @@ -156,6 +156,12 @@ static uint64_t read_uint64(ios_t *s) return b0 | (b1<<32); } +static void write_int64(ios_t *s, int64_t i) +{ + write_int32(s, (i>>32) & 0xffffffff); + write_int32(s, i & 0xffffffff); +} + static void write_uint16(ios_t *s, uint16_t i) { write_uint8(s, (i>> 8) & 0xff); @@ -1065,9 +1071,10 @@ static void write_work_list(ios_t *s) // serialize the global _require_dependencies array of pathnames that // are include depenencies -static void write_dependency_list(ios_t *s) +static int64_t write_dependency_list(ios_t *s, jl_array_t **udepsp) { size_t total_size = 0; + int64_t pos = 0; static jl_array_t *deps = NULL; if (!deps) deps = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("_require_dependencies")); @@ -1080,10 +1087,9 @@ static void write_dependency_list(ios_t *s) jl_value_t *uniqargs[2] = {unique_func, (jl_value_t*)deps}; size_t last_age = jl_get_ptls_states()->world_age; jl_get_ptls_states()->world_age = jl_world_counter; - jl_array_t *udeps = deps && unique_func ? (jl_array_t*)jl_apply(uniqargs, 2) : NULL; + jl_array_t *udeps = (*udepsp = deps && unique_func ? (jl_array_t*)jl_apply(uniqargs, 2) : NULL); jl_get_ptls_states()->world_age = last_age; - JL_GC_PUSH1(&udeps); if (udeps) { size_t l = jl_array_len(udeps); for (size_t i=0; i < l; i++) { @@ -1093,7 +1099,7 @@ static void write_dependency_list(ios_t *s) slen += jl_string_len(dep); total_size += 8 + slen + 8; } - total_size += 4; + total_size += 4 + 8; } // write the total size so that we can quickly seek past all of the // dependencies if we don't need them @@ -1113,8 +1119,11 @@ static void write_dependency_list(ios_t *s) write_float64(s, jl_unbox_float64(jl_fieldref(deptuple, 2))); // mtime } write_int32(s, 0); // terminator, for ease of reading + // write a dummy file position to indicate the beginning of the source-text + pos = ios_pos(s); + write_int64(s, 0); } - JL_GC_POP(); + return pos; } // --- deserialize --- @@ -2286,18 +2295,18 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) { char *tmpfname = strcat(strcpy((char *) alloca(strlen(fname)+8), fname), ".XXXXXX"); ios_t f; - jl_array_t *mod_array; + jl_array_t *mod_array, *udeps; if (ios_mkstemp(&f, tmpfname) == NULL) { jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", tmpfname); return 1; } - JL_GC_PUSH1(&mod_array); + JL_GC_PUSH2(&mod_array, &udeps); mod_array = jl_get_loaded_modules(); serializer_worklist = worklist; write_header(&f); write_work_list(&f); - write_dependency_list(&f); + int64_t srctextpos = write_dependency_list(&f, &udeps); write_mod_list(&f, mod_array); // this can return errors during deserialize, // best to keep it early (before any actual initialization) @@ -2319,7 +2328,6 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) assert(jl_is_module(m)); jl_collect_lambdas_from_mod(lambdas, m); } - JL_GC_POP(); jl_collect_backedges(edges); @@ -2332,15 +2340,40 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) jl_serialize_value(&s, worklist); jl_serialize_value(&s, lambdas); jl_serialize_value(&s, edges); - jl_finalize_serializer(&s); // done with f + jl_finalize_serializer(&s); serializer_worklist = NULL; jl_gc_enable(en); htable_reset(&edges_map, 0); htable_reset(&backref_table, 0); arraylist_free(&reinit_list); + + // Write the source-text for the dependent files + if (udeps) { + int64_t posfile = ios_pos(&f); + ios_seek(&f, srctextpos); + write_int64(&f, posfile); + ios_seek_end(&f); + len = jl_array_len(udeps); + ios_t *srctext; + uint64_t filelen; + for (i = 0; i < len; i++) { + jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); + jl_value_t *dep = jl_fieldref(deptuple, 1); // file abspath + posfile = ios_pos(&f); + write_uint64(&f, 0); // placeholder for length of this file in bytes + srctext = ios_file(srctext, jl_string_data(dep), 1, 0, 0, 0); + filelen = (uint64_t) ios_copyall(&f, srctext); + ios_close(srctext); + assert(srctext != NULL); + ios_seek(&f, posfile); + write_uint64(&f, filelen); + ios_seek_end(&f); + } + } ios_close(&f); + JL_GC_POP(); if (jl_fs_rename(tmpfname, fname) < 0) { jl_printf(JL_STDERR, "Cannot write cache file \"%s\".\n", fname); return 1; diff --git a/test/compile.jl b/test/compile.jl index 57a1875b92a65e..ba07e60e42842f 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -207,6 +207,8 @@ try discard_module = mod_fl_mt -> (mod_fl_mt[2], mod_fl_mt[3]) @test modules == Dict(Foo_module => Base.module_uuid(Foo)) @test map(x -> x[1], sort(discard_module.(deps))) == [Foo_file, joinpath(dir, "bar.jl"), joinpath(dir, "foo.jl")] + srctxt = Base.read_dependency_src(cachefile, Foo_file) + @test !isempty(srctxt) && srctxt == read(Foo_file, String) modules, deps1 = Base.cache_dependencies(cachefile) @test modules == merge(Dict(s => Base.module_uuid(getfield(Foo, s)) for s in