Harden CH watchdog diagnostics against pid race

- Validate selected pid is alive before gcore/gdb attach

- Log attach context (cmdline/parent pid) for troubleshooting

- Retry gdb attach against parent pid when target exits

- Use less aggressive default watchdog intervals

Author: vyadavmsft

lisa/microsoft/testsuites/cloud_hypervisor/ch_tests_tool.py

@@ -807,9 +807,9 @@ def _run_with_enhanced_diagnostics(
         inactivity watchdog, and comprehensive logging.
         """
         # Tunables (pull from env if provided; else use sane defaults)
-        idle_secs = int(os.environ.get("CH_IDLE_SECS", "60"))
-        hang_kill_secs = int(os.environ.get("CH_HANG_KILL_SECS", "900"))
-        check_interval = int(os.environ.get("CH_CHECK_INTERVAL", "5"))
+        idle_secs = int(os.environ.get("CH_IDLE_SECS", "300"))
+        hang_kill_secs = int(os.environ.get("CH_HANG_KILL_SECS", "1200"))
+        check_interval = int(os.environ.get("CH_CHECK_INTERVAL", "10"))
 
         # --- 1) Rich Rust diagnostics ---
         enhanced_env_vars = self.env_vars.copy()
@@ -904,40 +904,64 @@ def _run_with_enhanced_diagnostics(
     ps -eL -o pid,tid,ppid,stat,etime,comm,cmd | head -200 \\
       | tee -a "$log_file" || true
 
-    # Find a good target: prefer the integration test binary; otherwise a child
-    # of the cargo/dev_cli process; otherwise fall back to the main pid.
-    tpid="$(pgrep -n -f 'target/.*/deps/integration-' || true)"
-    if [ -z "$tpid" ]; then
-      # newest child of $pid (often cargo test or the binary)
-      tpid="$(pgrep -P "$pid" | tail -n1 || true)"
-    fi
-    [ -z "$tpid" ] && tpid="$pid"
-
-    # Best-effort freeze to avoid the attach race
-    sudo kill -STOP "$tpid" 2>/dev/null || true
-
-    # Use consistent core filename pattern that matches search pattern
-    core_out="core.integration-$(date +%s)"
-    echo "[watchdog] Generating core: $core_out" | tee -a "$log_file"
-    if command -v gcore >/dev/null 2>&1; then
-      sudo gcore -o "$core_out" "$tpid" 2>&1 | tee -a "$log_file" || true
-    else
-      sudo gdb -batch -p "$tpid" \\
-        -ex "set pagination off" \\
-        -ex "generate-core-file $core_out" \\
-        -ex "detach" -ex "quit" 2>&1 | tee -a "$log_file" || true
-    fi
-
-    # Write live backtrace to BOTH main log and side file
-    echo "[watchdog] Attaching gdb to pid $tpid for live backtrace" \\
-      | tee -a "$log_file"
-    sudo gdb -batch -p "$tpid" \\
-      -ex "set pagination off" \\
-      -ex "set print elements 0" \\
-      -ex "set backtrace limit 64" \\
-      -ex "thread apply all bt" \\
-      -ex "info threads" \\
-      2>&1 | tee -a "$log_file" > "$live_bt_file" || true
+        # Find a good target: prefer the integration test binary; otherwise a child
+        # of the cargo/dev_cli process; otherwise fall back to the main pid.
+        tpid="$(pgrep -n -f 'target/.*/deps/integration-' || true)"
+        if [ -z "$tpid" ]; then
+            # newest child of $pid (often cargo test or the binary)
+            tpid="$(pgrep -P "$pid" | tail -n1 || true)"
+        fi
+        [ -z "$tpid" ] && tpid="$pid"
+
+        # Verify the target pid is still alive. If it raced away, fall back to $pid.
+        if ! kill -0 "$tpid" 2>/dev/null; then
+            echo "[watchdog] Selected pid $tpid is not alive; falling back to pid $pid" \\
+                | tee -a "$log_file"
+            tpid="$pid"
+        fi
+
+        # Best-effort freeze to avoid the attach race
+        sudo kill -STOP "$tpid" 2>/dev/null || true
+
+        # Use consistent core filename pattern that matches search pattern
+        core_out="core.integration-$(date +%s)"
+        echo "[watchdog] Generating core: $core_out" | tee -a "$log_file"
+        if command -v gcore >/dev/null 2>&1; then
+            sudo gcore -o "$core_out" "$tpid" 2>&1 | tee -a "$log_file" || true
+        else
+            sudo gdb -batch -p "$tpid" \\
+                -ex "set pagination off" \\
+                -ex "generate-core-file $core_out" \\
+                -ex "detach" -ex "quit" 2>&1 | tee -a "$log_file" || true
+        fi
+
+        # Write live backtrace to BOTH main log and side file
+        echo "[watchdog] Attaching gdb to pid $tpid for live backtrace" \\
+            | tee -a "$log_file"
+            {{
+                echo "[watchdog] gdb attach target pid=$tpid parent_pid=$pid";
+                echo "[watchdog] cmdline(target)=$(tr '\\0' ' ' < /proc/$tpid/cmdline 2>/dev/null || echo n/a)";
+                echo "[watchdog] cmdline(parent)=$(tr '\\0' ' ' < /proc/$pid/cmdline 2>/dev/null || echo n/a)";
+            }} 2>/dev/null | tee -a "$log_file" || true
+        sudo gdb -batch -p "$tpid" \\
+            -ex "set pagination off" \\
+            -ex "set print elements 0" \\
+            -ex "set backtrace limit 64" \\
+            -ex "thread apply all bt" \\
+            -ex "info threads" \\
+            2>&1 | tee -a "$log_file" > "$live_bt_file" || true
+        # If attach failed (e.g. tpid exited/raced), retry once against the main pid
+        if grep -q "No such process" "$live_bt_file" 2>/dev/null; then
+            echo "[watchdog] gdb attach on pid $tpid failed; retrying against pid $pid" \\
+                | tee -a "$log_file"
+            sudo gdb -batch -p "$pid" \\
+                -ex "set pagination off" \\
+                -ex "set print elements 0" \\
+                -ex "set backtrace limit 64" \\
+                -ex "thread apply all bt" \\
+                -ex "info threads" \\
+                2>&1 | tee -a "$log_file" > "$live_bt_file" || true
+        fi
 
     # Let it run again
     sudo kill -CONT "$tpid" 2>/dev/null || true