[ci] Use available CPUs in builds

This adds a script to determine the CPUs for each job based on the number of executors per node, assuming and equal allocation to each executor on a node.

Author: driazati

tests/scripts/find_nproc.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import multiprocessing
+
+PREFIXES = {
+    "i-": 1,
+    "octo.aws.g4dn.52.42.180.210": 2,
+    "octo.aws.g4dn.54.185.57.115": 2,
+    "octo.aws.g4dn": 1,
+    "octo.aws.c4": 4,
+    "octo.aws.g4dn.cudabuild": 4,
+    "Built-In Node": 2,
+    "docs": 1,
+    "octo.aws.arm": 4,
+}
+
+
+def guess_num_executors(node_name: str) -> int:
+    # There is not a great way to get this from Jenkins directly apparently, so
+    # guess it based on node nodes
+    for prefix, count in PREFIXES.items():
+        if node_name.startswith(prefix):
+            return count
+
+    # Fallback to a safe option (4 is the wor
+    return 4
+
+
+if __name__ == "__main__":
+    help = "Determine max CPUs possible for this job"
+    parser = argparse.ArgumentParser(description=help)
+    parser.add_argument("--node-name", required=True, help="current node name")
+    args = parser.parse_args()
+
+    executors = guess_num_executors(args.node_name)
+    nproc = multiprocessing.cpu_count()
+
+    available_cpus = nproc // executors
+    available_cpus = max(available_cpus, 1)
+
+    print(available_cpus)

tests/scripts/task_build.sh

@@ -46,8 +46,15 @@ if [ "$HAS_SCCACHE" -eq "1" ]; then
     sccache --show-stats
 fi
 
+
+# Figure out the number of concurrent builds that can be running on this machine
+# TODO: Is there a way to get this from Jenkins directly? I don't think so
+# https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables
+NUM_CPUS=$(python tests/scripts/find_nproc.py --node-name "$NODE_NAME")
+MAKE_PROCESSES_ARG="-j$NUM_CPUS"
+
 cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
-cmake --build . -- VERBOSE=1 "$2"
+cmake --build . -- VERBOSE=1 "$MAKE_PROCESSES_ARG"
 
 if [ "$HAS_SCCACHE" -eq "1" ]; then
     sccache --show-stats