the servers it goes through, in order */ public static function order(Connection $c): array { $master = (string)$c->cache->master(); $hosts = array_keys(array_filter($c->cache->hosts(), fn ($h, $ref) => Updates::pending($c, (string)$ref) > 0, ARRAY_FILTER_USE_BOTH)); usort($hosts, fn ($a, $b) => ($b === $master) <=> ($a === $master)); return array_values(array_map('strval', $hosts)); } public static function run(Session $session, Connection $c): Task { $hosts = self::order($c); $entry = $session->log("Rolling pool update of '" . $c->name() . "' (" . count($hosts) . ' servers)', $c->name(), $c->cache->poolRef(), LogEntry::RUNNING); $steps = []; $total = max(1, count($hosts) * 5); $at = 0; $progress = function (string $what) use ($entry, &$at, $total): void { $at++; $entry->progress = min(0.99, $at / $total); $entry->detail = $what; }; foreach ($hosts as $host) { $name = $c->cache->name($host); $reboot = false; $steps[] = function () use ($c, $host, $name, $progress): Task { $progress("maintenance mode on '$name'"); return $c->run("Entering maintenance mode on '$name'", 'host.disable', [$host], $host); }; $steps[] = function () use ($c, $host, $name, $progress): Task { $progress("migrating the VMs of '$name'"); return $c->run("Migrating VMs away from '$name'", 'host.evacuate', [$host], $host); }; $steps[] = function () use ($c, $host, $name, $progress, &$reboot): Task { $progress("updating '$name'"); $out = new Task(); Updates::install($c, $host)->then(function (mixed $needs) use (&$reboot, $out): void { $reboot = (bool)$needs; $out->settle(true); }, fn (\Throwable $e) => $out->settle(null, $e)); return $out; }; $steps[] = function () use ($c, $host, $name, $progress, &$reboot): ?Task { $progress("rebooting '$name'"); if (!$reboot) { return null; } $boot = (string)($c->cache->field($host, 'other_config', [])['boot_time'] ?? ''); return Chain::run([fn () => $c->run("Rebooting server '$name'", 'host.reboot', [$host], $host), fn () => self::back($c, $host, $boot)]); }; $steps[] = function () use ($c, $host, $name, $progress): Task { $progress("exiting maintenance mode on '$name'"); return $c->run("Exiting maintenance mode on '$name'", 'host.enable', [$host], $host); }; } $done = new Task(); Chain::run($steps)->then(function () use ($session, $entry, $done): void { $entry->status = LogEntry::DONE; $entry->progress = 1; $entry->detail = ''; $entry->finished = $session->clock; $done->settle(true); }, function (\Throwable $e) use ($session, $entry, $done): void { $entry->status = LogEntry::FAILED; $entry->detail = 'stopped: ' . $e->getMessage(); $entry->finished = $session->clock; $done->settle(null, $e); }); return $done; } /** A Task settled when the server is back from its reboot: a new boot_time, its metrics live (or failed after BACK_IN). */ public static function back(Connection $c, string $host, string $bootBefore): Task { $done = new Task(); if (self::isBack($c, $host, $bootBefore)) { $done->settle(true); return $done; } $waited = 0.0; $timer = null; $timer = GameLoop::addPeriodicTimer(1.0, function () use (&$timer, &$waited, $done, $c, $host, $bootBefore): void { $waited += 1.0; $back = self::isBack($c, $host, $bootBefore); if ($back || $waited > self::BACK_IN) { if ($timer !== null) { GameLoop::cancelTimer($timer); } $back ? $done->settle(true) : $done->settle(null, new \RuntimeException("'" . $c->cache->name($host) . "' did not come back after its reboot.")); } }); return $done; } private static function isBack(Connection $c, string $host, string $bootBefore): bool { $boot = (string)($c->cache->field($host, 'other_config', [])['boot_time'] ?? ''); $metrics = $c->cache->get((string)$c->cache->field($host, 'metrics', '')) ?? []; return $c->isConnected() && $boot !== $bootBefore && !empty($metrics['live']); } }