Merge pull request #24120 from JuliaLang/teh/srccache_base

Cache the source text for Base and update guidelines for using Revise

CONTRIBUTING.md

@@ -225,10 +225,9 @@ your changes.
 Here is the standard procedure:
 
 1. If you are planning changes to any types or macros, make those
-   changes, commit them, and build julia using `make`. (This is
+   changes and build julia using `make`. (This is
    necessary because `Revise` cannot handle changes to type
-   definitions or macros.) By making a git commit, you "shield" these
-   changes from the `git stash` procedure described below. Unless it's
+   definitions or macros.) Unless it's
    required to get Julia to build, you do not have to add any
    functionality based on the new types, just the type definitions
    themselves.
@@ -241,27 +240,12 @@ Revise.track(Base)
 ```
 
 3. Edit files in `base/`, save your edits, and test the
-   functionality. Once you are satisfied that things work as desired,
-   make another commit and rebuild julia.
-
-Should you for some reason need to quit and restart your REPL session
-before finishing your changes, the procedure above will fail because
-`Revise.track`
-[cannot detect changes in source files that occurred after Julia was built](https://github.com/JuliaLang/julia/issues/23448)---it
-will only detect changes to source files that occur after tracking is
-initiated.  Consequently, any changes made prior to
-`Revise.track(Base)` will not be incorporated into your new REPL
-session. You can work around this by temporarily reverting all source
-files to their original state. From somewhere in the `julia`
-directory, start your REPL session and do the following:
+   functionality.
 
-```julia
-shell> git stash  # ensure that the code in `base/` matches its state when you built julia
-
-julia> Revise.track(Base)  # Revise's source code cache is synchronized with what's running
-
-shell> git stash pop  # restore any in-progress changes (will now be tracked)
-```
+If you need to restart your Julia session, just start at step 2 above.
+`Revise.track(Base)` will note any changes from when Julia was last
+built and incorporate them automatically. You only need to rebuild
+Julia if you made code-changes that Revise cannot handle.
 
 ### Code Formatting Guidelines
 

Makefile

@@ -108,7 +108,7 @@ julia-sysimg-release : julia-inference julia-ui-release
 julia-sysimg-debug : julia-inference julia-ui-debug
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_BUILD_MODE=debug
 
-julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest
+julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-base-cache
 
 debug release : % : julia-%
 
@@ -229,6 +229,8 @@ $(eval $(call sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug)))
 $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_depsbindir)
 	@$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c)
 
+julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia
+	@$(call exec,$(JULIA_EXECUTABLE) $(JULIAHOME)/etc/write_base_cache.jl $(build_datarootdir)/julia/base.cache)
 
 # public libraries, that are installed in $(prefix)/lib
 JL_LIBS := julia julia-debug

base/loading.jl

@@ -218,7 +218,7 @@ function _include_dependency(modstring::AbstractString, _path::AbstractString)
         path = joinpath(dirname(prev), _path)
     end
     if _track_dependencies[]
-        push!(_require_dependencies, (modstring, path, mtime(path)))
+        push!(_require_dependencies, (modstring, normpath(path), mtime(path)))
     end
     return path, prev
 end
@@ -733,6 +733,10 @@ function read_dependency_src(io::IO, filename::AbstractString)
     modules, files, required_modules, srctextpos = parse_cache_header(io)
     srctextpos == 0 && error("no source-text stored in cache file")
     seek(io, srctextpos)
+    _read_dependency_src(io, filename)
+end
+
+function _read_dependency_src(io::IO, filename::AbstractString)
     while !eof(io)
         filenamelen = ntoh(read(io, Int32))
         filenamelen == 0 && break

base/sysimg.jl

@@ -4,10 +4,11 @@ baremodule Base
 
 using Core.Intrinsics
 ccall(:jl_set_istopmod, Void, (Any, Bool), Base, true)
+
 function include(mod::Module, path::AbstractString)
     local result
     if INCLUDE_STATE === 1
-        result = Core.include(mod, path)
+        result = _include1(mod, path)
     elseif INCLUDE_STATE === 2
         result = _include(mod, path)
     elseif INCLUDE_STATE === 3
@@ -18,7 +19,7 @@ end
 function include(path::AbstractString)
     local result
     if INCLUDE_STATE === 1
-        result = Core.include(Base, path)
+        result = _include1(Base, path)
     elseif INCLUDE_STATE === 2
         result = _include(Base, path)
     else
@@ -28,12 +29,18 @@ function include(path::AbstractString)
     end
     result
 end
+const _included_files = Array{Tuple{Module,String}}(0)
+function _include1(mod::Module, path)
+    Core.Inference.push!(_included_files, (mod, ccall(:jl_prepend_cwd, Any, (Any,), path)))
+    Core.include(mod, path)
+end
 let SOURCE_PATH = ""
     # simple, race-y TLS, relative include
     global _include
     function _include(mod::Module, path)
         prev = SOURCE_PATH
         path = joinpath(dirname(prev), path)
+        push!(_included_files, (mod, abspath(path)))
         SOURCE_PATH = path
         result = Core.include(mod, path)
         SOURCE_PATH = prev
@@ -447,6 +454,9 @@ end # baremodule Base
 
 using Base
 
+# Ensure this file is also tracked
+unshift!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl")))
+
 # set up load path to be able to find stdlib packages
 Base.init_load_path(ccall(:jl_get_julia_home, Any, ()))
 

etc/write_base_cache.jl

@@ -0,0 +1,12 @@
+# Write the sys source cache in format readable by Base._read_dependency_src
+cachefile = ARGS[1]
+open(cachefile, "w") do io
+    for (_, filename) in Base._included_files
+        src = read(filename, String)
+        write(io, hton(Int32(sizeof(filename))))
+        write(io, filename)
+        write(io, hton(UInt64(sizeof(src))))
+        write(io, src)
+    end
+    write(io, Int32(0))
+end

src/toplevel.c

@@ -694,6 +694,23 @@ JL_DLLEXPORT jl_value_t *jl_load_(jl_module_t *module, jl_value_t *str)
     return jl_load(module, (const char*)jl_string_data(str));
 }
 
+JL_DLLEXPORT jl_value_t *jl_prepend_cwd(jl_value_t *str)
+{
+    size_t sz = 1024;
+    char path[1024];
+    int c = uv_cwd(path, &sz);
+    if (c < 0) {
+        jl_errorf("could not get current directory");
+    }
+    path[sz] = '/';  // fix later with normpath if Windows
+    const char *fstr = (const char*)jl_string_data(str);
+    if (strlen(fstr) + sz >= 1024) {
+        jl_errorf("use a bigger buffer for jl_fullpath");
+    }
+    strcpy(path + sz + 1, fstr);
+    return jl_cstr_to_string(path);
+}
+
 #ifdef __cplusplus
 }
 #endif