From af16cb65132e73ab1b921f9a40c4e66722ff79c6 Mon Sep 17 00:00:00 2001
From: Evan Lloyd New-Schmidt <evan@new-schmidt.com>
Date: Tue, 3 Oct 2023 12:23:36 -0400
Subject: [PATCH] Exit early with error if any jobs fail

I've tried a number of ways to do this, and this has been the simplest
and most reliable.

- Catches jobs that exit before calling the function.
- Doesn't mess with the kill_jobs hook and leave orphan processes.
- Bubbles up the exit code.

Signed-off-by: Evan Lloyd New-Schmidt <evan@new-schmidt.com>
---
 run.sh | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/run.sh b/run.sh
index 5954390..8d4c0df 100755
--- a/run.sh
+++ b/run.sh
@@ -139,11 +139,30 @@ kill_jobs() {
         log "Killing background jobs"
         # shellcheck disable=SC2086 # PIDs are intentionally expanded.
         kill $pids
-        log "Waiting for background jobs to stop"
         wait
     fi
 }
 
+# Exit early with error if any jobs fail.
+watch_jobs() {
+    # Must be provided, calling `jobs` clears any failed jobs from returning with `wait -n`.
+    num_jobs="$1"
+    if [ 0 -ge "$num_jobs" ]; then
+        echo "num_jobs is not a positive integer: '$num_jobs'" >&2
+        exit 1
+    fi
+
+    for ((i=1; i <= num_jobs; i++)); do
+        if wait -n; then
+            : # Cannot use `! wait -n`, it changes the exit code.
+        else
+            status=$?
+            log "ERROR: job failed with exit code $status"
+            exit $status
+        fi
+    done
+}
+
 trap 'kill_jobs' SIGINT SIGTERM EXIT
 
 for dump in "${DUMP_FILES[@]}"; do
@@ -154,7 +173,7 @@ for dump in "${DUMP_FILES[@]}"; do
     "$OUTPUT_DIR" &
 done
 
-wait
+watch_jobs "${#DUMP_FILES[@]}"
 
 log "Beginning extraction of discovered QIDs"
 
@@ -165,6 +184,6 @@ for dump in "${DUMP_FILES[@]}"; do
     "$OUTPUT_DIR" &
 done
 
-wait
+watch_jobs "${#DUMP_FILES[@]}"
 
 log "Finished"