Brancher: ping host before logbook poll

This makes it possible to reuse a brancher node that does not have any items in the logbook anymore, which typically happens on older Brancher nodes.

Fixes #185

Author: tdgroot

grumphp.yml

@@ -5,6 +5,7 @@ grumphp:
             warning_severity: 0
             whitelist_patterns:
                 - /^src\/(.*)/
+                - /^tests\/(.*)/
             triggered_by: [php]
         psalm:
             config: psalm.xml

psalm.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 <psalm
-    phpVersion="8.4"
+    phpVersion="8.1"
     errorLevel="4"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xmlns="https://getpsalm.org/schema/config"

src/Brancher/BrancherHypernodeManager.php

@@ -1,5 +1,7 @@
 <?php
 
+declare(strict_types=1);
+
 namespace Hypernode\Deploy\Brancher;
 
 use Hypernode\Api\Exception\HypernodeApiClientException;
@@ -14,13 +16,26 @@
 
 class BrancherHypernodeManager
 {
+    /**
+     * Relevant flow names to poll for delivery
+     *
+     * @var string[]
+     */
+    public const RELEVANT_FLOW_NAMES = ['ensure_app', 'ensure_copied_app'];
+
     private LoggerInterface $log;
     private HypernodeClient $hypernodeClient;
+    private SshPoller $sshPoller;
 
-    public function __construct(LoggerInterface $log)
-    {
+    public function __construct(
+        LoggerInterface $log,
+        ?HypernodeClient $hypernodeClient = null,
+        ?SshPoller $sshPoller = null
+    ) {
         $this->log = $log;
-        $this->hypernodeClient = HypernodeClientFactory::create(getenv('HYPERNODE_API_TOKEN') ?: '');
+        $this->hypernodeClient = $hypernodeClient
+            ?? HypernodeClientFactory::create(getenv('HYPERNODE_API_TOKEN') ?: '');
+        $this->sshPoller = $sshPoller ?? new SshPoller();
     }
 
     /**
@@ -105,6 +120,11 @@ public function createForHypernode(string $hypernode, array $data = []): string
     /**
      * Wait for brancher Hypernode to become available.
      *
+     * This method first attempts a quick SSH connectivity check. If the brancher is already
+     * reachable (e.g., when reusing an existing brancher), it returns early. Otherwise, it
+     * falls back to polling the API logbook for delivery status, then performs a final SSH
+     * reachability check.
+     *
      * @param string $brancherHypernode Name of the brancher Hypernode
      * @param int $timeout Maximum time to wait for availability
      * @param int $reachabilityCheckCount Number of consecutive successful checks required
@@ -121,24 +141,58 @@ public function waitForAvailability(
         int $reachabilityCheckCount = 6,
         int $reachabilityCheckInterval = 10
     ): void {
-        $latest = microtime(true);
-        $timeElapsed = 0;
+        $latest = $this->sshPoller->microtime();
+        $timeElapsed = 0.0;
+
+        // Phase 1: SSH-first check, early return for reused delivered branchers
+        $this->log->info(
+            sprintf('Attempting SSH connectivity check for brancher Hypernode %s...', $brancherHypernode)
+        );
+
+        $isReachable = $this->pollSshConnectivity(
+            $brancherHypernode,
+            3,
+            5,
+            $reachabilityCheckInterval,
+            $timeElapsed,
+            $latest,
+            $timeout
+        );
+        if ($isReachable) {
+            $this->log->info(
+                sprintf('Brancher Hypernode %s is reachable!', $brancherHypernode)
+            );
+            return;
+        }
+
+        $this->log->info(
+            sprintf(
+                'SSH check inconclusive for brancher Hypernode %s, falling back to delivery check...',
+                $brancherHypernode
+            )
+        );
+
+        // Phase 2: Wait for delivery by polling the logbook
         $resolved = false;
         $interval = 3;
         $allowedErrorWindow = 3;
+        $logbookStartTime = $timeElapsed;
 
         while ($timeElapsed < $timeout) {
-            $now = microtime(true);
+            $now = $this->sshPoller->microtime();
             $timeElapsed += $now - $latest;
             $latest = $now;
 
             try {
                 $flows = $this->hypernodeClient->logbook->getList($brancherHypernode);
-                $relevantFlows = array_filter($flows, fn(Flow $flow) => in_array($flow->name, ["ensure_app", "ensure_copied_app"], true));
+                $relevantFlows = array_filter(
+                    $flows,
+                    fn(Flow $flow) => in_array($flow->name, self::RELEVANT_FLOW_NAMES, true)
+                );
                 $failedFlows = array_filter($relevantFlows, fn(Flow $flow) => $flow->isReverted());
                 $completedFlows = array_filter($relevantFlows, fn(Flow $flow) => $flow->isComplete());
 
-                if (count($failedFlows) === count($relevantFlows)) {
+                if (count($relevantFlows) > 0 && count($failedFlows) === count($relevantFlows)) {
                     throw new CreateBrancherHypernodeFailedException();
                 }
 
@@ -151,21 +205,26 @@ public function waitForAvailability(
                 // Otherwise, there's an error, and it should be propagated.
                 if ($e->getCode() !== 404) {
                     throw $e;
-                } elseif ($timeElapsed < $allowedErrorWindow) {
+                } elseif (($timeElapsed - $logbookStartTime) < $allowedErrorWindow) {
                     // Sometimes we get an error where the logbook is not yet available, but it will be soon.
-                    // We allow a small window for this to happen, and then we throw an exception.
+                    // We allow a small window for this to happen, and then we continue polling.
                     $this->log->info(
                         sprintf(
                             'Got an expected exception during the allowed error window of HTTP code %d, waiting for %s to become available.',
                             $e->getCode(),
                             $brancherHypernode
                         )
                     );
-                    continue;
                 }
             }
 
-            sleep($interval);
+            $this->sshPoller->sleep($interval);
+        }
+
+        if (!$resolved) {
+            throw new TimeoutException(
+                sprintf('Timed out waiting for brancher Hypernode %s to be delivered', $brancherHypernode)
+            );
         }
 
         $this->log->info(
@@ -175,63 +234,94 @@ public function waitForAvailability(
             )
         );
 
-        if (!$resolved) {
+        // Phase 3: Final SSH reachability check
+        $isReachable = $this->pollSshConnectivity(
+            $brancherHypernode,
+            $reachabilityCheckCount,
+            0, // No max failures, rely on timeout
+            $reachabilityCheckInterval,
+            $timeElapsed,
+            $latest,
+            $timeout
+        );
+        if (!$isReachable) {
             throw new TimeoutException(
-                sprintf('Timed out waiting for brancher Hypernode %s to be delivered', $brancherHypernode)
+                sprintf('Timed out waiting for brancher Hypernode %s to become reachable', $brancherHypernode)
             );
         }
 
+        $this->log->info(
+            sprintf('Brancher Hypernode %s became reachable!', $brancherHypernode)
+        );
+    }
+
+    /**
+     * Poll SSH connectivity until we get enough consecutive successes or hit a limit.
+     *
+     * @param string $brancherHypernode Hostname to check
+     * @param int $requiredConsecutiveSuccesses Number of consecutive successes required
+     * @param int $maxFailedAttempts Maximum failed attempts before giving up (0 = no limit, use timeout only)
+     * @param int $checkInterval Seconds between checks
+     * @param float $timeElapsed Reference to track elapsed time
+     * @param float $latest Reference to track latest timestamp
+     * @param int $timeout Maximum time allowed
+     * @return bool True if SSH check succeeded, false if we should fall back to other methods
+     */
+    private function pollSshConnectivity(
+        string $brancherHypernode,
+        int $requiredConsecutiveSuccesses,
+        int $maxFailedAttempts,
+        int $checkInterval,
+        float &$timeElapsed,
+        float &$latest,
+        int $timeout
+    ): bool {
         $consecutiveSuccesses = 0;
+        $failedAttempts = 0;
+
         while ($timeElapsed < $timeout) {
-            $now = microtime(true);
+            $now = $this->sshPoller->microtime();
             $timeElapsed += $now - $latest;
             $latest = $now;
 
-            $connection = @fsockopen(sprintf("%s.hypernode.io", $brancherHypernode), 22);
-            if ($connection) {
-                fclose($connection);
+            // Check if we've hit the max failed attempts limit (0 = unlimited)
+            if ($maxFailedAttempts > 0 && $failedAttempts >= $maxFailedAttempts) {
+                return false;
+            }
+
+            if ($this->sshPoller->poll($brancherHypernode)) {
                 $consecutiveSuccesses++;
                 $this->log->info(
                     sprintf(
                         'Brancher Hypernode %s reachability check %d/%d succeeded.',
                         $brancherHypernode,
                         $consecutiveSuccesses,
-                        $reachabilityCheckCount
+                        $requiredConsecutiveSuccesses
                     )
                 );
 
-                if ($consecutiveSuccesses >= $reachabilityCheckCount) {
-                    break;
+                if ($consecutiveSuccesses >= $requiredConsecutiveSuccesses) {
+                    return true;
                 }
-                sleep($reachabilityCheckInterval);
             } else {
                 if ($consecutiveSuccesses > 0) {
                     $this->log->info(
                         sprintf(
                             'Brancher Hypernode %s reachability check failed, resetting counter (was at %d/%d).',
                             $brancherHypernode,
                             $consecutiveSuccesses,
-                            $reachabilityCheckCount
+                            $requiredConsecutiveSuccesses
                         )
                     );
                 }
                 $consecutiveSuccesses = 0;
-                sleep($reachabilityCheckInterval);
+                $failedAttempts++;
             }
-        }
 
-        if ($consecutiveSuccesses < $reachabilityCheckCount) {
-            throw new TimeoutException(
-                sprintf('Timed out waiting for brancher Hypernode %s to become reachable', $brancherHypernode)
-            );
+            $this->sshPoller->sleep($checkInterval);
         }
 
-        $this->log->info(
-            sprintf(
-                'Brancher Hypernode %s became reachable!',
-                $brancherHypernode
-            )
-        );
+        return false;
     }
 
     /**

src/Brancher/SshPoller.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Hypernode\Deploy\Brancher;
+
+class SshPoller
+{
+    /**
+     * Check if SSH port is reachable on the given hostname.
+     *
+     * @param string $hostname The hostname to check (without .hypernode.io suffix)
+     * @return bool True if SSH port 22 is reachable
+     */
+    public function poll(string $hostname): bool
+    {
+        $connection = @fsockopen(sprintf('%s.hypernode.io', $hostname), 22);
+        if ($connection) {
+            fclose($connection);
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Sleep for the given number of seconds.
+     *
+     * @param int $seconds Number of seconds to sleep
+     */
+    public function sleep(int $seconds): void
+    {
+        sleep($seconds);
+    }
+
+    /**
+     * Get the current time in microseconds.
+     *
+     * @return float Current time as a float
+     */
+    public function microtime(): float
+    {
+        return microtime(true);
+    }
+}