scripts: Reduce parallelism in "all" builds

Kernel builds are a bit asymmetrical in as far as CPU and memory
resources go.  Compilation itself tends to scale fairly linearly in
RAM usage per number of CPUs.  However, the link phase tends to use a
fairly constant large amount of RAM and generally only one CPU.

The existing attempt to build more in parallel has mostly resulted in
too many link steps running, which ends up swapping.

Instead of dividing the CPUs by the number of targets, just divide it
by two, and assume this will at least get us some compilation
happening during the link phase of one target build.

Change-Id: I0f8cac2d73600700d8ad6cb2a1b98e9529bb53a8
Signed-off-by: David Brown <davidb@codeaurora.org>

1 files changed, 9 insertions, 13 deletions

diff --git a/scripts/build-all.py b/scripts/build-all.py
index 5cb4da53791a..92f266ff8b03 100755
--- a/scripts/build-all.py
+++ b/scripts/build-all.py
@@ -74,12 +74,6 @@ def check_build():
             else:
                 raise
 
-def build_threads():
-    """Determine the number of build threads requested by the user"""
-    if all_options.load_average:
-        return all_options.load_average
-    return all_options.jobs or 1
-
 failed_targets = []
 
 BuildResult = namedtuple('BuildResult', ['status', 'messages'])
@@ -116,9 +110,10 @@ class BuildTracker:
     sequences can be processed independently, while the steps within a
     sequence must be done in order."""
 
-    def __init__(self):
+    def __init__(self, parallel_builds):
         self.sequence = []
         self.lock = threading.Lock()
+        self.parallel_builds = parallel_builds
 
     def add_sequence(self, log_name, short_name, steps):
         self.sequence.append(BuildSequence(log_name, short_name, steps))
@@ -148,7 +143,7 @@ class BuildTracker:
         children = []
         errors = []
         self.build_tokens = Queue.Queue()
-        nthreads = build_threads()
+        nthreads = self.parallel_builds
         print "Building with", nthreads, "threads"
         for i in range(nthreads):
             self.build_tokens.put(True)
@@ -327,14 +322,15 @@ def scan_configs():
 def build_many(targets):
     print "Building %d target(s)" % len(targets)
 
-    # If we are requesting multiple builds, divide down the job number
-    # to construct the make_command, giving it a floor of 2, so there
-    # is still some parallelism.
+    # To try and make up for the link phase being serial, try to do
+    # two full builds in parallel.  Don't do too many because lots of
+    # parallel builds tends to use up available memory rather quickly.
+    parallel = 2
     if all_options.jobs and all_options.jobs > 1:
-        j = max(all_options.jobs / len(targets), 2)
+        j = min(all_options.jobs / parallel, 2)
         make_command.append("-j" + str(j))
 
-    tracker = BuildTracker()
+    tracker = BuildTracker(parallel)
     for target in targets:
         if all_options.updateconfigs:
             update_config(target.defconfig, all_options.updateconfigs)