Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
node_modules/
build/
cache/
.DS_Store
2 changes: 1 addition & 1 deletion Dockerfile.centos7
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ RUN sed -i 's/mirror.centos.org/vault.centos.org/g' /etc/yum.repos.d/*.repo && \

# use node 10 as it's guaranteed to run on CentOS6
RUN yum-config-manager --enable rhel-server-rhscl-7-rpms && \
yum install -y devtoolset-8 python36 wget patch && \
yum install -y devtoolset-10 python36 wget patch && \
wget -q https://nodejs.org/dist/v10.23.0/node-v10.23.0-linux-x64.tar.gz && \
tar -xf node-v10.23.0-linux-x64.tar.gz && \
rm node-v10.23.0-linux-x64.tar.gz && \
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile.centos7.arm64
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ RUN sed -i 's|mirror.centos.org/centos|vault.centos.org/altarch|g' /etc/yum.repo

# use node 10 as it's guaranteed to run on CentOS6
RUN yum-config-manager --enable rhel-server-rhscl-7-rpms && \
yum install -y devtoolset-8 python36 wget patch && \
yum install -y devtoolset-10 python36 wget patch && \
wget -q https://nodejs.org/dist/v10.23.0/node-v10.23.0-linux-arm64.tar.gz && \
gunzip node-v10.23.0-linux-arm64.tar.gz && tar xf node-v10.23.0-linux-arm64.tar && rm node-v10.23.0-linux-arm64.tar && \
ln -s /node-v10.23.0-linux-arm64/bin/node /bin/node && \
Expand Down
10 changes: 5 additions & 5 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,27 @@ jobs:
linux_arm64:
imageName: 'ubuntu-20.04'
arch: 'linux/arm64'
node: '18.20.4'
node: '20.18.0'

linux_amd64:
imageName: 'ubuntu-20.04'
arch: 'linux/amd64'
node: '18.20.4'
node: '20.18.0'
ptrcompress: false

linux_amd64_ptrc:
imageName: 'ubuntu-20.04'
arch: 'linux/amd64'
node: '18.20.4'
node: '20.18.0'
ptrcompress: true

windows_2019:
imageName: 'windows-2019'
node: '18.20.4'
node: '20.18.0'
ptrcompress: false
windows_2019_ptrc:
imageName: 'windows-2019'
node: '18.20.4'
node: '20.18.0'
ptrcompress: true

# mac:
Expand Down
7 changes: 4 additions & 3 deletions build-builder.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ if [ $(docker buildx ls | grep js2bin-builder | wc -l) -eq 0 ]; then
fi
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
docker buildx use js2bin-builder
docker buildx build -t "cribl/js2bin-builder:latest-nonx64" --platform linux/arm64/v8 --push -f Dockerfile.centos7.arm64 .
docker build -t "cribl/js2bin-builder:latest" -f Dockerfile.centos7 .
docker push "cribl/js2bin-builder:latest"
BUILDER_IMAGE_VERSION="2"
docker buildx build -t "cribl/js2bin-builder:${BUILDER_IMAGE_VERSION}-nonx64" --platform linux/arm64/v8 --push -f Dockerfile.centos7.arm64 .
docker build -t "cribl/js2bin-builder:${BUILDER_IMAGE_VERSION}" -f Dockerfile.centos7 .
docker push "cribl/js2bin-builder:${BUILDER_IMAGE_VERSION}"
15 changes: 10 additions & 5 deletions src/NodeBuilder.js
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ class NodeJsBuilder {
this.cacheDir = join(cwd || process.cwd(), 'cache');
this.resultFile = isWindows ? join(this.nodeSrcDir, 'Release', 'node.exe') : join(this.nodeSrcDir, 'out', 'Release', 'node');
this.placeHolderSizeMB = -1;
this.builderImageVersion = 2;
}

static platform() {
Expand Down Expand Up @@ -208,6 +209,10 @@ class NodeJsBuilder {
this.nodePath('node.gyp'),
join(this.patchDir, 'node.gyp.patch'));

await patchFile(
this.nodePath('configure.py'),
join(this.patchDir, 'configure.py.patch'));

isWindows && await patchFile(
this.nodePath('vcbuild.bat'),
join(this.patchDir, 'vcbuild.bat.patch'));
Expand All @@ -228,26 +233,26 @@ class NodeJsBuilder {
}

buildInContainer() {
const containerTag = 'cribl/js2bin-builder:latest';
const containerTag = `cribl/js2bin-builder:${this.builderImageVersion}`;
return runCommand(
'docker', ['run',
'-v', `${process.cwd()}:/js2bin/`,
'-t', containerTag,
'/bin/bash', '-c',
`source /opt/rh/devtoolset-8/enable && cd /js2bin && npm install && ./js2bin.js --ci --node=${this.version} --size=${this.placeHolderSizeMB}MB`
`source /opt/rh/devtoolset-10/enable && cd /js2bin && npm install && ./js2bin.js --ci --node=${this.version} --size=${this.placeHolderSizeMB}MB`
]
);
}

buildInContainerNonX64(arch) {
const containerTag = 'cribl/js2bin-builder:latest-nonx64';
const containerTag = `cribl/js2bin-builder:${this.builderImageVersion}-nonx64`;
return runCommand(
'docker', ['run',
'--platform', arch,
'-v', `${process.cwd()}:/js2bin/`,
'-t', containerTag,
'/bin/bash', '-c',
`source /opt/rh/devtoolset-8/enable && cd /js2bin && npm install && ./js2bin.js --ci --node=${this.version} --size=${this.placeHolderSizeMB}MB`
`source /opt/rh/devtoolset-10/enable && cd /js2bin && npm install && ./js2bin.js --ci --node=${this.version} --size=${this.placeHolderSizeMB}MB`
]
);
}
Expand All @@ -258,7 +263,7 @@ class NodeJsBuilder {
// 4. process mainAppFile (gzip, base64 encode it) - could be a placeholder file
// 5. kick off ./configure & build
buildFromSource(uploadBuild, cache, container, arch, ptrCompression) {
const makeArgs = isWindows ? ['x64', 'noetw', 'no-cctest'] : [`-j${os.cpus().length}`];
const makeArgs = isWindows ? ['x64', 'no-cctest'] : [`-j${os.cpus().length}`];
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Support for etw (Event Tracing for Windows) has been removed since nodejs v19.0.0 onward. nodejs/node#43652
It's mentioned in the nodejs v19 changelog as well.

const configArgs = [];
if(ptrCompression) {
if(isWindows) makeArgs.push('v8_ptr_compress');
Expand Down
13 changes: 13 additions & 0 deletions src/patch/20.18.0/configure.py.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
--- a/configure.py
+++ b/configure.py
@@ -1598,7 +1598,9 @@ def configure_v8(o):
o['variables']['v8_use_siphash'] = 0 if options.without_siphash else 1
o['variables']['v8_enable_maglev'] = 1 if options.v8_enable_maglev else 0
o['variables']['v8_enable_pointer_compression'] = 1 if options.enable_pointer_compression else 0
- o['variables']['v8_enable_sandbox'] = 1 if options.enable_pointer_compression else 0
+
+ # o['variables']['v8_enable_sandbox'] = 1 if options.enable_pointer_compression else 0
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need to disable this option?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There has been a commit that was introduced in nodejs 20.17.0 that enforces enablingv8_enable_sandbox flag whenever v8_enable_pointer_compression flag is enabled. This causes failure of building nodejs for Windows OS when we enable the pointer compression flag. Apparently, such a thing does not work well in the Windows OS. I also did not find any build with the pointer compression flag enabled to run and test for Windows OS on the nodejs CI pipeline. The build failure message can be seen in CRIBL-28701 ticket.

+
o['variables']['v8_enable_31bit_smis_on_64bit_arch'] = 1 if options.enable_pointer_compression else 0
o['variables']['v8_enable_shared_ro_heap'] = 0 if options.enable_pointer_compression or options.disable_shared_ro_heap else 1
o['variables']['v8_enable_extensible_ro_snapshot'] = 0
11 changes: 11 additions & 0 deletions src/patch/20.18.0/fs-event.c.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
--- a/deps/uv/src/win/fs-event.c
+++ b/deps/uv/src/win/fs-event.c
@@ -266,6 +266,8 @@ short_path_done:
}

dir_to_watch = dir;
+ uv__free(short_path);
+ short_path = NULL;
uv__free(pathw);
pathw = NULL;
}
24 changes: 24 additions & 0 deletions src/patch/20.18.0/no_rand_on_glibc.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
disable usage of <sys/random.h>

--- deps/cares/config/linux/ares_config.h
+++ deps/cares/config/linux/ares_config.h
@@ -116,7 +116,9 @@
#define HAVE_GETNAMEINFO 1

/* Define to 1 if you have `getrandom` */
+#if defined(__GLIBC__) && __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 25
#define HAVE_GETRANDOM 1
+#endif

/* Define to 1 if you have `getservbyport_r` */
#define HAVE_GETSERVBYPORT_R 1
@@ -329,7 +331,9 @@
#define HAVE_SYS_PARAM_H 1

/* Define to 1 if you have the <sys/random.h> header file. */
+#if defined(__GLIBC__) && __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 25
#define HAVE_SYS_RANDOM_H 1
+#endif

/* Define to 1 if you have the <sys/select.h> header file. */
#define HAVE_SYS_SELECT_H 1
17 changes: 17 additions & 0 deletions src/patch/20.18.0/node.cc.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
--- src/node.cc
+++ src/node.cc
@@ -377,6 +377,14 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) {
return env->RunSnapshotDeserializeMain();
}

+ // To allow people to extend Node in different ways, this hook allows
+ // one to drop a file lib/_third_party_main.js into the build
+ // directory which will be executed instead of Node's normal loading.
+ if (env->builtin_loader()->Exists("_third_party_main")) {
+ return StartExecution(env, "internal/main/run_third_party_main");
+ }
+
+
if (env->worker_context() != nullptr) {
return StartExecution(env, "internal/main/worker_thread");
}
13 changes: 13 additions & 0 deletions src/patch/20.18.0/node.gyp.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
disable building of cctest as it (a) it fails to build due to align_alloc and (b) it is just a waste of time

--- node.gyp
+++ node.gyp
@@ -1144,7 +1144,7 @@
}, # fuzz_strings
{
'target_name': 'cctest',
- 'type': 'executable',
+ 'type': 'none',

'dependencies': [
'<(node_lib_target_name)',
17 changes: 17 additions & 0 deletions src/patch/20.18.0/run_third_party_main.js.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
--- /dev/null
+++ lib/internal/main/run_third_party_main.js
@@ -0,0 +1,14 @@
+'use strict';
+
+const {
+ prepareMainThreadExecution,
+ markBootstrapComplete
+} = require('internal/process/pre_execution');
+
+prepareMainThreadExecution();
+markBootstrapComplete();
+
+// Legacy _third_party_main.js support
+process.nextTick(() => {
+ require('_third_party_main');
+});
28 changes: 28 additions & 0 deletions src/patch/20.18.0/vcbuild.bat.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
expose option for configuring pointer compression

--- vcbuild.bat
+++ vcbuild.bat
@@ -67,6 +67,7 @@ set no_shared_roheap=
set doc=
set extra_msbuild_args=
set exit_code=0
+set v8_ptr_compress=

:next-arg
if "%1"=="" goto args-done
@@ -139,6 +140,7 @@ if /i "%1"=="cctest" set cctest=1&goto arg-ok
if /i "%1"=="openssl-no-asm" set openssl_no_asm=1&goto arg-ok
if /i "%1"=="no-shared-roheap" set no_shared_roheap=1&goto arg-ok
if /i "%1"=="doc" set doc=1&goto arg-ok
+if /i "%1"=="v8_ptr_compress" set v8_ptr_compress=1&goto arg-ok
if /i "%1"=="binlog" set extra_msbuild_args=/binaryLogger:out\%config%\node.binlog&goto arg-ok

echo Error: invalid command line option `%1`.
@@ -196,6 +198,7 @@ if defined debug_nghttp2 set configure_flags=%configure_flags% --debug-nghttp
if defined openssl_no_asm set configure_flags=%configure_flags% --openssl-no-asm
if defined no_shared_roheap set configure_flags=%configure_flags% --disable-shared-readonly-heap
if defined DEBUG_HELPER set configure_flags=%configure_flags% --verbose
+if defined v8_ptr_compress set configure_flags=%configure_flags% --experimental-enable-pointer-compression
if "%target_arch%"=="x86" if "%PROCESSOR_ARCHITECTURE%"=="AMD64" set configure_flags=%configure_flags% --no-cross-compiling

if not exist "%~dp0deps\icu" goto no-depsicu