Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support non-preprocess build object cache #6108

Draft
wants to merge 1 commit into
base: dev
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions xmake/modules/core/tools/nvcc.lua
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import("core.project.project")
import("core.platform.platform")
import("core.language.language")
import("core.project.policy")
import("private.cache.build_cache")
import("utils.progress")

-- init it
Expand Down Expand Up @@ -380,8 +381,15 @@ function compile(self, sourcefile, objectfile, dependinfo, flags, opt)

-- do compile
local program, argv = compargv(self, sourcefile, objectfile, compflags)
local outdata, errdata = os.iorunv(program, argv, {envs = self:runenvs()})
return (outdata or "") .. (errdata or "")
if build_cache.is_enabled(opt.target) then
local program_, origin_flags = compargv(self, sourcefile, objectfile, flags)
local info = build_cache.build(program, argv, {envs = self:runenvs(),
origin_flags = origin_flags, outputfile = objectfile, target = opt.target, tool = self})
return (info.outdata or "") .. (info.errdata or "")
else
local outdata, errdata = os.iorunv(program, argv, {envs = self:runenvs()})
return (outdata or "") .. (errdata or "")
end
end,
catch
{
Expand Down
148 changes: 99 additions & 49 deletions xmake/modules/private/cache/build_cache.lua
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,8 @@ function is_supported(sourcekind)
end

-- get cache key
function cachekey(program, cppinfo, envs)
local cppfile = cppinfo.cppfile
local cppflags = cppinfo.cppflags
function cachekey(program, info, opt)
local cppflags = info.cppflags
local items = {program}
for _, cppflag in ipairs(cppflags) do
if cppflag:startswith("-D") or cppflag:startswith("/D") then
Expand All @@ -116,12 +115,16 @@ function cachekey(program, cppinfo, envs)
end
end
table.sort(items)
table.insert(items, hash.xxhash128(cppfile))
if envs then

if opt.preprocess then
table.insert(items, hash.xxhash128(info.cppfile))
end

if opt.envs then
local basename = path.basename(program)
if basename == "cl" then
for _, name in ipairs({"WindowsSDKVersion", "VCToolsVersion", "LIB"}) do
local val = envs[name]
local val = opt.envs[name]
if val then
table.insert(items, val)
end
Expand Down Expand Up @@ -275,74 +278,121 @@ end

-- build with cache
function build(program, argv, opt)

-- do preprocess
opt = opt or {}
local preprocess = assert(opt.preprocess, "preprocessor not found!")
local compile = assert(opt.compile, "compiler not found!")
local cppinfo = preprocess(program, argv, opt)
if cppinfo then
local cachekey = cachekey(program, cppinfo, opt.envs)

local info
-- support preprocess?
if is_supported(opt.tool:kind()) then
-- do preprocess
local preprocess = assert(opt.preprocess, "preprocessor not found!")
local compile = assert(opt.compile, "compiler not found!")
info = preprocess(program, argv, opt)
if info then
local cachekey = cachekey(program, info, {envs = opt.envs, preprocess = true})
local cache_hit_start_time = os.mclock()
local objectfile_cached, objectfile_infofile = get(cachekey)
if objectfile_cached then
os.cp(objectfile_cached, info.objectfile)
-- we need to update mtime for incremental compilation
-- @see https://github.com/xmake-io/xmake/issues/2620
os.touch(info.objectfile, {mtime = os.time()})
-- we need to get outdata/errdata to show warnings,
-- @see https://github.com/xmake-io/xmake/issues/2452
if objectfile_infofile and os.isfile(objectfile_infofile) then
local extrainfo = io.load(objectfile_infofile)
info.outdata = extrainfo.outdata
info.errdata = extrainfo.errdata
end
_g.cache_hit_total_time = (_g.cache_hit_total_time or 0) + (os.mclock() - cache_hit_start_time)
else
-- do compile
local preprocess_outdata = info.outdata
local preprocess_errdata = info.errdata
local compile_start_time = os.mclock()
local compile_fallback = opt.compile_fallback
if compile_fallback then
local ok = try {function () compile(program, info, opt); return true end}
if not ok then
-- we fallback to compile original source file if compiling preprocessed file fails.
-- https://github.com/xmake-io/xmake/issues/2467
local outdata, errdata = compile_fallback()
info.outdata = outdata
info.errdata = errdata
_g.compile_fallback_count = (_g.compile_fallback_count or 0) + 1
end
else
compile(program, info, opt)
end
-- if no compiler output, we need use preprocessor output, because it maybe contains warning output
if not info.outdata or #info.outdata == 0 then
info.outdata = preprocess_outdata
end
if not info.errdata or #info.errdata == 0 then
info.errdata = preprocess_errdata
end
_g.compile_total_time = (_g.compile_total_time or 0) + (os.mclock() - compile_start_time)
if cachekey then
local extrainfo
if info.outdata and #info.outdata ~= 0 then
extrainfo = extrainfo or {}
extrainfo.outdata = info.outdata
end
if info.errdata and #info.errdata ~= 0 then
extrainfo = extrainfo or {}
extrainfo.errdata = info.errdata
end
local cache_miss_start_time = os.mclock()
put(cachekey, info.objectfile, extrainfo)
_g.cache_miss_total_time = (_g.cache_miss_total_time or 0) + (os.mclock() - cache_miss_start_time)
end
end
os.rm(info.cppfile)
else
_g.preprocess_error_count = (_g.preprocess_error_count or 0) + 1
end
else
info = {
-- Do not use depfile flag to generate cache key
cppflags = opt.origin_flags,
outputfile = opt.outputfile
}
local cachekey = cachekey(program, info, {envs = envs})
local cache_hit_start_time = os.mclock()
local objectfile_cached, objectfile_infofile = get(cachekey)
if objectfile_cached then
os.cp(objectfile_cached, cppinfo.objectfile)
os.cp(objectfile_cached, info.outputfile)
-- we need to update mtime for incremental compilation
-- @see https://github.com/xmake-io/xmake/issues/2620
os.touch(cppinfo.objectfile, {mtime = os.time()})
os.touch(info.outputfile, {mtime = os.time()})
-- we need to get outdata/errdata to show warnings,
-- @see https://github.com/xmake-io/xmake/issues/2452
if objectfile_infofile and os.isfile(objectfile_infofile) then
local extrainfo = io.load(objectfile_infofile)
cppinfo.outdata = extrainfo.outdata
cppinfo.errdata = extrainfo.errdata
info.outdata = extrainfo.outdata
info.errdata = extrainfo.errdata
end
_g.cache_hit_total_time = (_g.cache_hit_total_time or 0) + (os.mclock() - cache_hit_start_time)
else
-- do compile
local preprocess_outdata = cppinfo.outdata
local preprocess_errdata = cppinfo.errdata
local compile_start_time = os.mclock()
local compile_fallback = opt.compile_fallback
if compile_fallback then
local ok = try {function () compile(program, cppinfo, opt); return true end}
if not ok then
-- we fallback to compile original source file if compiling preprocessed file fails.
-- https://github.com/xmake-io/xmake/issues/2467
local outdata, errdata = compile_fallback()
cppinfo.outdata = outdata
cppinfo.errdata = errdata
_g.compile_fallback_count = (_g.compile_fallback_count or 0) + 1
end
else
compile(program, cppinfo, opt)
end
-- if no compiler output, we need use preprocessor output, because it maybe contains warning output
if not cppinfo.outdata or #cppinfo.outdata == 0 then
cppinfo.outdata = preprocess_outdata
end
if not cppinfo.errdata or #cppinfo.errdata == 0 then
cppinfo.errdata = preprocess_errdata
end
local outdata, errdata = os.iorunv(program, argv, {envs = opt.envs})
_g.compile_total_time = (_g.compile_total_time or 0) + (os.mclock() - compile_start_time)
if cachekey then
local extrainfo
if cppinfo.outdata and #cppinfo.outdata ~= 0 then
if info.outdata and #info.outdata ~= 0 then
extrainfo = extrainfo or {}
extrainfo.outdata = cppinfo.outdata
extrainfo.outdata = info.outdata
end
if cppinfo.errdata and #cppinfo.errdata ~= 0 then
if info.errdata and #info.errdata ~= 0 then
extrainfo = extrainfo or {}
extrainfo.errdata = cppinfo.errdata
extrainfo.errdata = info.errdata
end
local cache_miss_start_time = os.mclock()
put(cachekey, cppinfo.objectfile, extrainfo)
put(cachekey, info.outputfile, extrainfo)
_g.cache_miss_total_time = (_g.cache_miss_total_time or 0) + (os.mclock() - cache_miss_start_time)
end
end
os.rm(cppinfo.cppfile)
else
_g.preprocess_error_count = (_g.preprocess_error_count or 0) + 1
end
return cppinfo

return info
end
Loading