From 8d56bcc59c92c5770f585cd433230e7771da2dad Mon Sep 17 00:00:00 2001 From: Guanqun Lu Date: Tue, 29 Oct 2019 16:48:05 +0800 Subject: [PATCH 1/2] download .tar.xz if python3 is used --- src/bootstrap/bootstrap.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 4caf36a6f2a51..9cc58e03d5649 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -102,10 +102,10 @@ def verify(path, sha_path, verbose): return verified -def unpack(tarball, dst, verbose=False, match=None): +def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): """Unpack the given tarball file""" print("extracting", tarball) - fname = os.path.basename(tarball).replace(".tar.gz", "") + fname = os.path.basename(tarball).replace(tarball_suffix, "") with contextlib.closing(tarfile.open(tarball)) as tar: for member in tar.getnames(): if "/" not in member: @@ -331,6 +331,18 @@ def __init__(self): self.use_vendored_sources = '' self.verbose = False + def support_xz(): + try: + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + with tarfile.open(temp_path, "w:xz") as tar: + pass + return True + except tarfile.CompressionError: + return False + + self.tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' + def download_stage0(self): """Fetch the build system for Rust, written in Rust @@ -349,12 +361,13 @@ def download_stage0(self): self.program_out_of_date(self.rustc_stamp())): if os.path.exists(self.bin_root()): shutil.rmtree(self.bin_root()) - filename = "rust-std-{}-{}.tar.gz".format( - rustc_channel, self.build) + filename = "rust-std-{}-{}{}".format( + rustc_channel, self.build, self.tarball_suffix) pattern = "rust-std-{}".format(self.build) self._download_stage0_helper(filename, pattern) - filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build) + filename = "rustc-{}-{}{}".format(rustc_channel, self.build, + self.tarball_suffix) self._download_stage0_helper(filename, "rustc") self.fix_executable("{}/bin/rustc".format(self.bin_root())) self.fix_executable("{}/bin/rustdoc".format(self.bin_root())) @@ -365,14 +378,15 @@ def download_stage0(self): # libraries/binaries that are included in rust-std with # the system MinGW ones. if "pc-windows-gnu" in self.build: - filename = "rust-mingw-{}-{}.tar.gz".format( - rustc_channel, self.build) + filename = "rust-mingw-{}-{}{}".format( + rustc_channel, self.build, self.tarball_suffix) self._download_stage0_helper(filename, "rust-mingw") if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.program_out_of_date(self.cargo_stamp())): - filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build) + filename = "cargo-{}-{}{}".format(cargo_channel, self.build, + self.tarball_suffix) self._download_stage0_helper(filename, "cargo") self.fix_executable("{}/bin/cargo".format(self.bin_root())) with output(self.cargo_stamp()) as cargo_stamp: @@ -388,7 +402,7 @@ def _download_stage0_helper(self, filename, pattern): tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): get("{}/{}".format(url, filename), tarball, verbose=self.verbose) - unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose) + unpack(tarball, self.tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) @staticmethod def fix_executable(fname): From 0019371e3d878c1031bd7395b52ab40f2441049c Mon Sep 17 00:00:00 2001 From: Guanqun Lu Date: Tue, 12 Nov 2019 00:16:05 +0800 Subject: [PATCH 2/2] bootstrap: don't call support_xz in hot-path --- src/bootstrap/bootstrap.py | 43 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 9cc58e03d5649..730e8cf05d41d 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -331,17 +331,6 @@ def __init__(self): self.use_vendored_sources = '' self.verbose = False - def support_xz(): - try: - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - with tarfile.open(temp_path, "w:xz") as tar: - pass - return True - except tarfile.CompressionError: - return False - - self.tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' def download_stage0(self): """Fetch the build system for Rust, written in Rust @@ -356,19 +345,30 @@ def download_stage0(self): rustc_channel = self.rustc_channel cargo_channel = self.cargo_channel + def support_xz(): + try: + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + with tarfile.open(temp_path, "w:xz") as tar: + pass + return True + except tarfile.CompressionError: + return False + if self.rustc().startswith(self.bin_root()) and \ (not os.path.exists(self.rustc()) or self.program_out_of_date(self.rustc_stamp())): if os.path.exists(self.bin_root()): shutil.rmtree(self.bin_root()) + tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' filename = "rust-std-{}-{}{}".format( - rustc_channel, self.build, self.tarball_suffix) + rustc_channel, self.build, tarball_suffix) pattern = "rust-std-{}".format(self.build) - self._download_stage0_helper(filename, pattern) + self._download_stage0_helper(filename, pattern, tarball_suffix) filename = "rustc-{}-{}{}".format(rustc_channel, self.build, - self.tarball_suffix) - self._download_stage0_helper(filename, "rustc") + tarball_suffix) + self._download_stage0_helper(filename, "rustc", tarball_suffix) self.fix_executable("{}/bin/rustc".format(self.bin_root())) self.fix_executable("{}/bin/rustdoc".format(self.bin_root())) with output(self.rustc_stamp()) as rust_stamp: @@ -379,20 +379,21 @@ def download_stage0(self): # the system MinGW ones. if "pc-windows-gnu" in self.build: filename = "rust-mingw-{}-{}{}".format( - rustc_channel, self.build, self.tarball_suffix) - self._download_stage0_helper(filename, "rust-mingw") + rustc_channel, self.build, tarball_suffix) + self._download_stage0_helper(filename, "rust-mingw", tarball_suffix) if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.program_out_of_date(self.cargo_stamp())): + tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' filename = "cargo-{}-{}{}".format(cargo_channel, self.build, - self.tarball_suffix) - self._download_stage0_helper(filename, "cargo") + tarball_suffix) + self._download_stage0_helper(filename, "cargo", tarball_suffix) self.fix_executable("{}/bin/cargo".format(self.bin_root())) with output(self.cargo_stamp()) as cargo_stamp: cargo_stamp.write(self.date) - def _download_stage0_helper(self, filename, pattern): + def _download_stage0_helper(self, filename, pattern, tarball_suffix): cache_dst = os.path.join(self.build_dir, "cache") rustc_cache = os.path.join(cache_dst, self.date) if not os.path.exists(rustc_cache): @@ -402,7 +403,7 @@ def _download_stage0_helper(self, filename, pattern): tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): get("{}/{}".format(url, filename), tarball, verbose=self.verbose) - unpack(tarball, self.tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) + unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) @staticmethod def fix_executable(fname):