diff --git a/app/Actions/ReapplyInstanceAddress.php b/app/Actions/ReapplyInstanceAddress.php new file mode 100644 index 0000000..971bbb7 --- /dev/null +++ b/app/Actions/ReapplyInstanceAddress.php @@ -0,0 +1,138 @@ +isReapplyable($instance)) { + return null; + } + + // The lock closes the window between asking whether a run exists and + // creating one. Two requests can reach this in the same instant — the + // customer's own "check now" and the nightly command are the obvious + // pair — and two runs against one machine would write the same router + // twice and race each other's occ calls. Nobody waits for the lock: a + // re-apply that lost the race has nothing to add, because the run that + // won reads the same domain state it would have. + $lock = Cache::lock('instance-address:'.$instance->uuid, 30); + + if (! $lock->get()) { + return null; + } + + try { + return $this->start($instance); + } finally { + $lock->release(); + } + } + + private function start(Instance $instance): ?ProvisioningRun + { + $order = $instance->order; + + // A run already under way applies whatever the domain state says when + // it gets to the step, which is this run's state or newer. Checked + // against the ORDER, because that is the subject both pipelines share: + // starting an address run beside an unfinished customer run would have + // two runs writing one router and one Nextcloud config. + if ($this->hasRunInFlight($order)) { + return null; + } + + $run = ProvisioningRun::create([ + 'subject_type' => Order::class, + 'subject_id' => $order->id, + 'pipeline' => 'address', + 'status' => ProvisioningRun::STATUS_PENDING, + 'current_step' => 0, + // Everything the two steps read. instance_id is how CustomerStep + // finds the machine; node and vmid are how it reaches inside it. + 'context' => [ + 'instance_id' => $instance->id, + 'host_id' => $instance->host_id, + 'node' => $instance->host?->node, + 'vmid' => $instance->vmid, + 'subdomain' => $instance->subdomain, + ], + ]); + + AdvanceRunJob::dispatch($run->uuid); + + Log::info('Re-applying an instance address.', [ + 'instance' => $instance->uuid, + 'domain' => $instance->custom_domain, + 'verified' => $instance->domainIsVerified(), + 'run' => $run->uuid, + ]); + + return $run; + } + + /** + * Is there a machine here whose address can be re-applied at all? + * + * A reservation with no VM, a failed build and an instance whose order has + * gone all fail the same way: the steps would reach for a guest agent that + * is not there and burn the run's retries doing it. An instance that is + * still `provisioning` is excluded for a different reason — its own run + * will apply the address on its way past, and it holds the same lock this + * would. + */ + private function isReapplyable(Instance $instance): bool + { + return $instance->order !== null + && $instance->host !== null + && $instance->vmid !== null + && in_array($instance->status, ['active', 'cancellation_scheduled'], true); + } + + private function hasRunInFlight(Order $order): bool + { + return ProvisioningRun::query() + ->where('subject_type', Order::class) + ->where('subject_id', $order->id) + ->whereIn('status', [ + ProvisioningRun::STATUS_PENDING, + ProvisioningRun::STATUS_RUNNING, + ProvisioningRun::STATUS_WAITING, + ProvisioningRun::STATUS_PAUSED, + ]) + ->exists(); + } +} diff --git a/app/Console/Commands/VerifyCustomDomains.php b/app/Console/Commands/VerifyCustomDomains.php index 1346b20..be1d9ad 100644 --- a/app/Console/Commands/VerifyCustomDomains.php +++ b/app/Console/Commands/VerifyCustomDomains.php @@ -2,6 +2,7 @@ namespace App\Console\Commands; +use App\Actions\ReapplyInstanceAddress; use App\Models\Instance; use App\Services\Domains\DomainVerifier; use Illuminate\Console\Command; @@ -24,7 +25,7 @@ class VerifyCustomDomains extends Command protected $description = 'Re-read the DNS proof for every custom domain and withdraw the ones that lost it'; - public function handle(DomainVerifier $verifier): int + public function handle(DomainVerifier $verifier, ReapplyInstanceAddress $reapply): int { $query = Instance::query()->whereNotNull('custom_domain')->whereNotNull('domain_token'); @@ -39,6 +40,12 @@ class VerifyCustomDomains extends Command $checked++; $present = $verifier->proofPresent($instance); + // What was true before this check, so the CHANGE can be acted on + // rather than the state. Re-applying an address is remote work on a + // live machine; doing it nightly for every domain that is simply + // still fine would be a hundred pointless runs a night. + $wasVerified = $instance->domainIsVerified(); + if ($present) { $instance->forceFill([ 'domain_verified_at' => $instance->domain_verified_at ?? now(), @@ -47,6 +54,13 @@ class VerifyCustomDomains extends Command 'domain_failures' => 0, ])->save(); + // A domain that has just become provable is a domain nothing is + // serving yet: the router was written without it and Nextcloud + // does not trust it. This is where it becomes an address. + if (! $wasVerified) { + $reapply($instance); + } + continue; } @@ -64,10 +78,18 @@ class VerifyCustomDomains extends Command 'domain_error' => 'missing_txt', 'domain_failures' => $failures, 'domain_verified_at' => $withdraw ? null : $instance->domain_verified_at, + 'domain_cert_ok' => $withdraw ? false : $instance->domain_cert_ok, ])->save(); if ($withdraw) { $withdrawn++; + + // The other half of the flip, and the one that actually takes + // the domain away: the router stops carrying it and Nextcloud + // stops trusting it. Clearing verified_at alone only stops us + // TELLING people about it. + $reapply($instance); + $this->warn("Withdrew {$instance->custom_domain}: proof missing on {$failures} consecutive checks."); } } diff --git a/app/Livewire/CustomDomain.php b/app/Livewire/CustomDomain.php index cfb4d09..159904b 100644 --- a/app/Livewire/CustomDomain.php +++ b/app/Livewire/CustomDomain.php @@ -2,6 +2,7 @@ namespace App\Livewire; +use App\Actions\ReapplyInstanceAddress; use App\Livewire\Concerns\ResolvesCustomer; use App\Models\Instance; use App\Services\Billing\CustomDomainAccess; @@ -88,15 +89,24 @@ class CustomDomain extends Component return; } + // Whether anything is being SERVED under the old domain right now. A + // change here can only ever take a verified domain away — the new one + // starts unproven — so this is the one question that decides whether + // the proxy and Nextcloud have to be told. + $wasVerified = $instance->domainIsVerified(); + // Clearing the field takes the domain away entirely, including its // verification — the instance falls back to its platform address. if ($domain === '') { $instance->forceFill([ 'custom_domain' => null, 'domain_token' => null, - 'domain_verified_at' => null, 'domain_checked_at' => null, + 'domain_verified_at' => null, 'domain_cert_ok' => false, + 'domain_checked_at' => null, 'domain_error' => null, 'domain_failures' => 0, ])->save(); + $this->stopOrStartServing($instance, $wasVerified); + $this->domain = ''; $this->dispatch('notify', message: __('domain.removed')); @@ -108,10 +118,13 @@ class CustomDomain extends Component 'custom_domain' => $domain, 'domain_token' => $verifier->newToken(), 'domain_verified_at' => null, + 'domain_cert_ok' => false, 'domain_checked_at' => null, 'domain_error' => null, 'domain_failures' => 0, ])->save(); + + $this->stopOrStartServing($instance, $wasVerified); } $this->domain = $domain; @@ -133,19 +146,48 @@ class CustomDomain extends Component return; } + $wasVerified = $instance->domainIsVerified(); + $wasServed = $instance->domainIsServed(); + $present = $verifier->proofPresent($instance); $instance->forceFill([ 'domain_checked_at' => now(), 'domain_verified_at' => $present ? ($instance->domain_verified_at ?? now()) : null, + 'domain_cert_ok' => $present ? $instance->domain_cert_ok : false, 'domain_error' => $present ? null : 'missing_txt', 'domain_failures' => $present ? 0 : $instance->domain_failures + 1, ])->save(); + // This button flips verification both ways, so it is a place the + // address changes and has to be re-applied like any other. It also + // re-applies a domain that is proven but still answering nothing — + // which is what somebody is doing here when they add the A record + // after the TXT one and press it again, and it is the only way they + // have of asking us to look now rather than at the next change. + if ($present !== $wasVerified || ($present && ! $wasServed)) { + app(ReapplyInstanceAddress::class)($instance); + } + $this->justChecked = true; $this->dispatch('notify', message: __($present ? 'domain.verified' : 'domain.not_found')); } + /** + * Tell the proxy and Nextcloud that the address has moved. + * + * Only when something WAS being served: an unproven domain never reached a + * router or a trusted_domains entry, so replacing one with another changes + * nothing that has to be un-done, and starting a run for it would be remote + * work on a live machine for no reason at all. + */ + private function stopOrStartServing(Instance $instance, bool $wasVerified): void + { + if ($wasVerified) { + app(ReapplyInstanceAddress::class)($instance); + } + } + public function render() { $instance = $this->instance(); diff --git a/app/Livewire/CustomerProvisioning.php b/app/Livewire/CustomerProvisioning.php index d5dcd9a..bb4af33 100644 --- a/app/Livewire/CustomerProvisioning.php +++ b/app/Livewire/CustomerProvisioning.php @@ -29,8 +29,13 @@ class CustomerProvisioning extends Component return null; } + // The BUILD, not every run against the order. An address re-apply runs + // on the same subject and would light this card up with "Ihre Cloud + // wird bereitgestellt" over a router file being rewritten — on a cloud + // that has been running for months. return ProvisioningRun::query() ->where('subject_type', Order::class) + ->where('pipeline', 'customer') ->whereIn('subject_id', $customer->orders()->select('id')) ->latest('id') ->first(); diff --git a/app/Models/Host.php b/app/Models/Host.php index 0414aad..411b464 100644 --- a/app/Models/Host.php +++ b/app/Models/Host.php @@ -50,6 +50,11 @@ class Host extends Model implements ProvisioningSubject $this->update(['status' => 'error']); } + public function provisioningPipeline(): string + { + return 'host'; + } + public function instances(): HasMany { return $this->hasMany(Instance::class); diff --git a/app/Models/Instance.php b/app/Models/Instance.php index 1f4c1ac..d4b12df 100644 --- a/app/Models/Instance.php +++ b/app/Models/Instance.php @@ -3,6 +3,8 @@ namespace App\Models; use App\Models\Concerns\HasUuid; +use Database\Factories\InstanceFactory; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; @@ -11,14 +13,14 @@ use Illuminate\Database\Eloquent\Relations\HasOne; class Instance extends Model { - /** @use HasFactory<\Database\Factories\InstanceFactory> */ + /** @use HasFactory */ use HasFactory, HasUuid; protected $fillable = [ 'customer_id', 'order_id', 'host_id', 'vmid', 'guest_ip', 'plan', 'quota_gb', 'traffic_addons', 'disk_gb', 'ram_mb', 'cores', 'subdomain', 'custom_domain', 'nc_admin_ref', 'admin_password', 'credentials_acknowledged_at', - 'route_written', 'cert_ok', 'status', 'cancel_requested_at', 'service_ends_at', - 'domain_token', 'domain_verified_at', 'domain_checked_at', 'domain_error', 'domain_failures', + 'route_written', 'routed_hostnames', 'cert_ok', 'status', 'cancel_requested_at', 'service_ends_at', + 'domain_token', 'domain_verified_at', 'domain_cert_ok', 'domain_checked_at', 'domain_error', 'domain_failures', ]; protected $hidden = ['nc_admin_ref', 'admin_password']; @@ -36,7 +38,9 @@ class Instance extends Model 'domain_verified_at' => 'datetime', 'domain_checked_at' => 'datetime', 'domain_failures' => 'integer', + 'domain_cert_ok' => 'boolean', 'route_written' => 'boolean', + 'routed_hostnames' => 'array', 'cert_ok' => 'boolean', 'vmid' => 'integer', 'traffic_addons' => 'integer', @@ -59,7 +63,7 @@ class Instance extends Model * placement would call a host comfortable while orders were being refused * on it. * - * @param \Illuminate\Database\Eloquent\Builder $query + * @param Builder $query */ public function scopeOccupyingHost($query): void { @@ -78,6 +82,21 @@ class Instance extends Model return filled($this->custom_domain) && $this->domain_verified_at !== null; } + /** + * Is the customer's own domain not merely proven, but actually answering? + * + * Verification says the domain is theirs. It says nothing about whether + * they have pointed it at us — that is an A record in their zone, which we + * cannot create and (behind a CDN) cannot even read. So there is a real + * state between "proven" and "working", it can last forever, and the portal + * has to be able to say which one the customer is in rather than printing + * an address that answers nothing. + */ + public function domainIsServed(): bool + { + return $this->domainIsVerified() && (bool) $this->domain_cert_ok; + } + /** * The address this instance is actually served at. * diff --git a/app/Models/Order.php b/app/Models/Order.php index a14bd82..3a8d85f 100644 --- a/app/Models/Order.php +++ b/app/Models/Order.php @@ -133,4 +133,9 @@ class Order extends Model implements ProvisioningSubject $this->update(['status' => 'failed']); $this->instance()->update(['status' => 'failed']); } + + public function provisioningPipeline(): string + { + return 'customer'; + } } diff --git a/app/Provisioning/Contracts/ProvisioningSubject.php b/app/Provisioning/Contracts/ProvisioningSubject.php index ee372da..c20f56d 100644 --- a/app/Provisioning/Contracts/ProvisioningSubject.php +++ b/app/Provisioning/Contracts/ProvisioningSubject.php @@ -9,4 +9,17 @@ namespace App\Provisioning\Contracts; interface ProvisioningSubject { public function onProvisioningFailed(): void; + + /** + * The pipeline that BUILDS this subject, as opposed to one that maintains + * it afterwards. + * + * The failure hook writes the whole thing off — an order is marked failed + * and its instance released with it — which is the right answer when the + * machine was never finished and entirely the wrong one when a live + * customer's address re-apply could not reach their VM for ten minutes. So + * the runner asks which pipeline this subject exists to be built by, and + * only that one is allowed to condemn it. + */ + public function provisioningPipeline(): string; } diff --git a/app/Provisioning/RunRunner.php b/app/Provisioning/RunRunner.php index 1adb542..117042f 100644 --- a/app/Provisioning/RunRunner.php +++ b/app/Provisioning/RunRunner.php @@ -167,9 +167,14 @@ class RunRunner $run->save(); $this->record($run, $stepKey, 'failed', $reason); - // Let the subject react (e.g. a Host moves to the 'error' status). + // Let the subject react (e.g. a Host moves to the 'error' status) — but + // only when the run that failed is the one that builds it. A + // maintenance pipeline (`address`) shares the same subject, and letting + // it fire this hook would mark a paid, running customer's order failed + // and release their live instance because a router file could not be + // written. See ProvisioningSubject::provisioningPipeline(). $subject = $run->subject; - if ($subject instanceof ProvisioningSubject) { + if ($subject instanceof ProvisioningSubject && $run->pipeline === $subject->provisioningPipeline()) { $subject->onProvisioningFailed(); } } diff --git a/app/Provisioning/Steps/Customer/ConfigureDnsAndTls.php b/app/Provisioning/Steps/Customer/ConfigureDnsAndTls.php index b3d8095..13a5276 100644 --- a/app/Provisioning/Steps/Customer/ConfigureDnsAndTls.php +++ b/app/Provisioning/Steps/Customer/ConfigureDnsAndTls.php @@ -2,6 +2,7 @@ namespace App\Provisioning\Steps\Customer; +use App\Models\Instance; use App\Models\ProvisioningRun; use App\Provisioning\StepResult; use App\Services\Dns\HetznerDnsClient; @@ -10,6 +11,18 @@ use App\Support\ProvisioningSettings; class ConfigureDnsAndTls extends CustomerStep { + /** + * How long the step keeps looking for the CUSTOM domain's certificate + * before it moves on without it. + * + * Long enough that the ordinary case lands inside the run: the customer has + * already pointed their A record at us, Traefik sees the first request to + * the new hostname and finishes HTTP-01 in seconds. Short enough that it is + * never a wait — the domain may equally be pointed at us tomorrow, or + * never, and the run has no business hanging on that. + */ + private const CUSTOM_DOMAIN_GRACE = 120; + public function __construct( private HetznerDnsClient $dns, private TraefikWriter $traefik, @@ -32,8 +45,20 @@ class ConfigureDnsAndTls extends CustomerStep $host = $instance->host; $fqdn = $instance->subdomain.'.'.ProvisioningSettings::dnsZone(); + // The customer's own domain is served ONLY once it is proven to be + // theirs. Everything downstream reads this one flag, so an unverified + // hostname sitting in the column reaches neither the router nor a + // certificate — see Instance::domainIsVerified(). + $customDomain = $instance->domainIsVerified() ? (string) $instance->custom_domain : null; + + // The platform address first: it is ours, it always works, and it is + // what the instance falls back to the moment a custom domain goes away. + $hostnames = array_values(array_filter([$fqdn, $customDomain])); + // DNS — provider upsert is idempotent; the local row uses firstOrCreate on // the record id and is written BEFORE the breadcrumb (the short-circuit guard). + // Only OUR zone: the custom domain's A record lives in the customer's + // zone, which we have no access to and never will. if (! $this->hasResource($run, 'dns_record_id')) { $recordId = $this->dns->upsertRecord($fqdn, 'A', $host->public_ip); $instance->dnsRecords()->firstOrCreate( @@ -45,11 +70,19 @@ class ConfigureDnsAndTls extends CustomerStep // Traefik file-provider route, written on the serving host (DNS points at // it) and pointing at the guest VM. - if (! $instance->route_written) { + // + // Guarded on WHAT the router carries, not merely on whether one was ever + // written. `route_written` alone would short-circuit exactly the case + // this step is re-run for — an address that has changed — and the + // customer's domain would be announced but never routed, or stay routed + // after it was withdrawn. Comparing the hostname list keeps the retry + // cheap (a second attempt at the same address writes nothing) without + // making a re-apply a no-op. + if (! $instance->route_written || $instance->routed_hostnames !== $hostnames) { $trafficHost = $host->wg_ip ?? $host->public_ip; $backend = $instance->guest_ip ?: $host->public_ip; - $this->traefik->write($trafficHost, $instance->subdomain, $backend); - $instance->update(['route_written' => true]); + $this->traefik->write($trafficHost, $instance->subdomain, $hostnames, $backend); + $instance->update(['route_written' => true, 'routed_hostnames' => $hostnames]); } // TLS via HTTP-01 — poll until the certificate is served. @@ -63,6 +96,58 @@ class ConfigureDnsAndTls extends CustomerStep } } + return $this->settleCustomDomainCertificate($run, $instance, $customDomain); + } + + /** + * The custom domain's certificate: recorded, never required. + * + * The platform address above is ours — no certificate there means something + * is broken on our side and the run fails so somebody looks. This one is the + * opposite: it can only be issued once the customer has pointed their own A + * record at us, and whether they have is not ours to decide, not ours to + * check (behind a CDN the address is invisible from outside) and possibly + * never going to happen. Failing a run over it would mean a customer who + * typed a domain and went to lunch could not get their cloud built. + * + * So the outcome is written to `domain_cert_ok` and the run advances either + * way. The portal reads that flag to tell "proven" apart from "answering", + * because it is the portal that prints this address as the customer's. + */ + private function settleCustomDomainCertificate(ProvisioningRun $run, Instance $instance, ?string $customDomain): StepResult + { + if ($customDomain === null) { + // No verified domain: whatever was true before is not true now. + if ($instance->domain_cert_ok) { + $instance->update(['domain_cert_ok' => false]); + } + + return StepResult::advance(); + } + + if ($this->traefik->certReachable($customDomain)) { + $instance->update(['domain_cert_ok' => true]); + + return StepResult::advance(); + } + + // A short look, not a wait — see CUSTOM_DOMAIN_GRACE. + if ($run->started_at !== null && ! $run->started_at->copy()->addSeconds(self::CUSTOM_DOMAIN_GRACE)->isPast()) { + return StepResult::poll(15, 'waiting for the custom domain certificate'); + } + + $instance->update(['domain_cert_ok' => false]); + + // Said out loud rather than swallowed: from here the domain is proven, + // routed, and answering nothing, and that is a state an operator + // reading this run should be able to see without going looking. + $run->events()->create([ + 'step' => $this->key(), + 'attempt' => $run->attempt, + 'outcome' => 'info', + 'message' => "Eigene Domain {$customDomain}: noch kein Zertifikat — zeigt der A-Eintrag des Kunden schon auf uns?", + ]); + return StepResult::advance(); } } diff --git a/app/Provisioning/Steps/Customer/ConfigureNextcloud.php b/app/Provisioning/Steps/Customer/ConfigureNextcloud.php index bda02ae..3385f50 100644 --- a/app/Provisioning/Steps/Customer/ConfigureNextcloud.php +++ b/app/Provisioning/Steps/Customer/ConfigureNextcloud.php @@ -35,8 +35,19 @@ class ConfigureNextcloud extends CustomerStep // answer for anything listed here, so an unproven hostname added at // provisioning time would serve this customer's files to whoever // pointed that name at the proxy. + // + // And the other direction has to happen too. Writing the entry when + // there is a domain and doing nothing when there is not leaves a + // withdrawn domain trusted forever — the proof is gone, the router no + // longer carries it, and Nextcloud would still answer to it the moment + // anything reached it under that name. `config:system:delete` is the + // occ command that removes the entry rather than blanking it, and it + // exits 0 when the key is already absent, so the step stays idempotent + // for the overwhelmingly common case of an instance that never had one. if ($instance->domainIsVerified()) { $this->guest($pve, $run, $occ.'config:system:set trusted_domains 2 --value='.escapeshellarg($instance->custom_domain)); + } else { + $this->guest($pve, $run, $occ.'config:system:delete trusted_domains 2'); } $this->guest($pve, $run, $occ.'background:cron'); $this->guest($pve, $run, $occ.'config:system:set default_phone_region --value=DE'); diff --git a/app/Services/Billing/CustomDomainAccess.php b/app/Services/Billing/CustomDomainAccess.php index fc11a03..8ce612f 100644 --- a/app/Services/Billing/CustomDomainAccess.php +++ b/app/Services/Billing/CustomDomainAccess.php @@ -3,6 +3,7 @@ namespace App\Services\Billing; use App\Actions\BookAddon; +use App\Actions\ReapplyInstanceAddress; use App\Models\Customer; use App\Models\Instance; use App\Models\PlanFamily; @@ -288,6 +289,13 @@ final class CustomDomainAccess * Neither the verification nor the serving logic is reimplemented here: * Instance::address() already falls back to the subdomain the moment the * domain is gone, and that is the whole of the fallback in this codebase. + * + * Clearing the row is only half of it, though. A domain that was verified + * is a domain the proxy is routing and Nextcloud is trusting, and neither + * of those reads this table — so the address is re-applied, which is what + * actually stops the customer's old domain from answering. Without it a + * downgrade would take the domain off every screen while the machine + * carried on serving it. */ public function deactivate(?Instance $instance): bool { @@ -295,15 +303,22 @@ final class CustomDomainAccess return false; } + $wasVerified = $instance->domainIsVerified(); + $instance->forceFill([ 'custom_domain' => null, 'domain_token' => null, 'domain_verified_at' => null, + 'domain_cert_ok' => false, 'domain_checked_at' => null, 'domain_error' => null, 'domain_failures' => 0, ])->save(); + if ($wasVerified) { + app(ReapplyInstanceAddress::class)($instance); + } + return true; } diff --git a/app/Services/Traefik/FakeTraefikWriter.php b/app/Services/Traefik/FakeTraefikWriter.php index 79a059c..ccfe26e 100644 --- a/app/Services/Traefik/FakeTraefikWriter.php +++ b/app/Services/Traefik/FakeTraefikWriter.php @@ -7,24 +7,52 @@ class FakeTraefikWriter implements TraefikWriter /** @var array subdomain => backend */ public array $routes = []; + /** @var array> subdomain => hostnames the router serves */ + public array $hostnames = []; + /** @var array subdomain => traffic host */ public array $hosts = []; + /** How many times a router file was written, so a test can prove a rewrite happened. */ + public int $writes = 0; + public bool $certReady = true; - public function write(string $trafficHost, string $subdomain, string $backend): void + /** + * Hostnames that answer nothing however often they are probed — a + * customer's own domain whose A record does not point at us. Consulted + * before $certReady, so a test can have the platform address certified + * while the custom domain is not. + * + * @var array + */ + public array $certUnreachable = []; + + public function write(string $trafficHost, string $subdomain, array $hostnames, string $backend): void { $this->routes[$subdomain] = $backend; + $this->hostnames[$subdomain] = array_values($hostnames); $this->hosts[$subdomain] = $trafficHost; + $this->writes++; } public function remove(string $trafficHost, string $subdomain): void { - unset($this->routes[$subdomain], $this->hosts[$subdomain]); + unset($this->routes[$subdomain], $this->hostnames[$subdomain], $this->hosts[$subdomain]); } public function certReachable(string $fqdn): bool { + if (in_array($fqdn, $this->certUnreachable, true)) { + return false; + } + return $this->certReady; } + + /** Does the router for this instance serve that hostname right now? */ + public function serves(string $subdomain, string $hostname): bool + { + return in_array($hostname, $this->hostnames[$subdomain] ?? [], true); + } } diff --git a/app/Services/Traefik/SshTraefikWriter.php b/app/Services/Traefik/SshTraefikWriter.php index 950f5e2..56961c7 100644 --- a/app/Services/Traefik/SshTraefikWriter.php +++ b/app/Services/Traefik/SshTraefikWriter.php @@ -17,10 +17,13 @@ class SshTraefikWriter implements TraefikWriter { public function __construct(private RemoteShell $shell) {} - public function write(string $trafficHost, string $subdomain, string $backend): void + public function write(string $trafficHost, string $subdomain, array $hostnames, string $backend): void { - $zone = ProvisioningSettings::dnsZone(); - $yaml = $this->render($subdomain, "{$subdomain}.{$zone}", $backend); + // The file is rewritten wholesale rather than appended to, so the + // hostnames handed in are exactly the hostnames served afterwards — + // that is what makes withdrawing a domain a write rather than a + // separate deletion somebody has to remember. + $yaml = $this->render($subdomain, $hostnames, $backend); $this->shell->connectWithKey($trafficHost, 'root', $this->privateKey()); $this->shell->putFile($this->path($subdomain), $yaml); // putFile throws on failure @@ -59,13 +62,25 @@ class SshTraefikWriter implements TraefikWriter return rtrim(ProvisioningSettings::traefikDynamicPath(), '/')."/{$subdomain}.yml"; } - private function render(string $subdomain, string $fqdn, string $backend): string + /** + * One router, one rule, every hostname in it. + * + * Traefik's `Host()` matcher takes several names in one call, and a single + * rule keeps the certResolver, the service and the entryPoint stated once — + * a second router would have to repeat all three and could drift from the + * first. + * + * @param array $hostnames + */ + private function render(string $subdomain, array $hostnames, string $backend): string { + $names = implode(', ', array_map(fn (string $host) => "`{$host}`", $hostnames)); + return implode("\n", [ 'http:', ' routers:', " {$subdomain}:", - " rule: \"Host(`{$fqdn}`)\"", + " rule: \"Host({$names})\"", " service: \"{$subdomain}\"", ' entryPoints: ["websecure"]', ' tls:', diff --git a/app/Services/Traefik/TraefikWriter.php b/app/Services/Traefik/TraefikWriter.php index 1daa16e..87e60d6 100644 --- a/app/Services/Traefik/TraefikWriter.php +++ b/app/Services/Traefik/TraefikWriter.php @@ -5,10 +5,19 @@ namespace App\Services\Traefik; interface TraefikWriter { /** - * Write a file-provider router on the Traefik host ($trafficHost) that routes - * the subdomain to the guest ($backend). + * Write a file-provider router on the Traefik host ($trafficHost) that + * routes every hostname in $hostnames to the guest ($backend). + * + * A LIST, not one name: an instance is reachable under its platform + * address always, and under the customer's verified domain as well when + * there is one. They belong in ONE router under one name — the subdomain, + * which is stable and is what remove() deletes — because two routers for + * one backend are two things to keep in step, and the second one is what + * gets forgotten the day the domain is withdrawn. + * + * @param array $hostnames platform address first */ - public function write(string $trafficHost, string $subdomain, string $backend): void; + public function write(string $trafficHost, string $subdomain, array $hostnames, string $backend): void; public function remove(string $trafficHost, string $subdomain): void; diff --git a/config/provisioning.php b/config/provisioning.php index cbcb30b..e8ff5b6 100644 --- a/config/provisioning.php +++ b/config/provisioning.php @@ -61,6 +61,24 @@ return [ Customer\RunAcceptanceChecks::class, Customer\CompleteProvisioning::class, ], + + /* + | Making an existing instance's address real again, without rebuilding + | anything. Exactly the two steps an address consists of: the router, + | the DNS record and the certificate, then the hostname Nextcloud + | itself will answer to. + | + | Same subject as `customer` (the Order), because that is what + | CustomerStep::order()/instance() resolve — a run against any other + | subject would need every step in it rewritten. Started by + | App\Actions\ReapplyInstanceAddress whenever the address changes: + | a domain proven, a domain withdrawn, a package that no longer + | carries one. + */ + 'address' => [ + Customer\ConfigureDnsAndTls::class, + Customer\ConfigureNextcloud::class, + ], ], // The one currency the catalogue is priced in. Plan prices carry no currency diff --git a/database/migrations/2026_07_29_230000_add_address_state_to_instances.php b/database/migrations/2026_07_29_230000_add_address_state_to_instances.php new file mode 100644 index 0000000..a32dc8f --- /dev/null +++ b/database/migrations/2026_07_29_230000_add_address_state_to_instances.php @@ -0,0 +1,46 @@ +json('routed_hostnames')->nullable()->after('route_written'); + + // Whether the CUSTOM domain's certificate is being served, kept + // apart from `cert_ok` (the platform address) on purpose. The + // platform address is ours: if its certificate does not appear, the + // run has failed and somebody has to look. The custom domain's + // certificate depends on the customer pointing their DNS at us, + // which may not have happened and may never happen — so it is + // recorded, never waited on, and never a reason to fail anything. + $table->boolean('domain_cert_ok')->default(false)->after('domain_verified_at'); + }); + } + + public function down(): void + { + Schema::table('instances', function (Blueprint $table) { + $table->dropColumn(['routed_hostnames', 'domain_cert_ok']); + }); + } +}; diff --git a/lang/de/domain.php b/lang/de/domain.php index b9bcf3e..97e5301 100644 --- a/lang/de/domain.php +++ b/lang/de/domain.php @@ -28,10 +28,16 @@ return [ 'state' => [ 'verified' => 'Freigeschaltet', + // Nachgewiesen ist nicht dasselbe wie erreichbar: der TXT-Eintrag + // gehört uns, der A-Eintrag dem Kunden. Diese Seite hat die Adresse + // bisher als fertig angezeigt, obwohl darunter nichts geantwortet hat. + 'not_served' => 'Nachgewiesen, noch nicht erreichbar', 'pending' => 'Warten auf Nachweis', 'none' => 'Keine eigene Domain', ], + 'not_served_hint' => 'Der Besitz ist nachgewiesen und die Domain ist bei uns eingerichtet — unter dieser Adresse antwortet aber noch nichts. Meistens fehlt dann noch der A-Eintrag aus Schritt 1, oder er ist noch nicht überall bekannt. Bis dahin ist Ihre Cloud unverändert unter der Plattformadresse erreichbar.', + 'checked_at' => 'Zuletzt geprüft: :when', 'never_checked' => 'Noch nicht geprüft.', diff --git a/lang/en/domain.php b/lang/en/domain.php index b1fd9b4..57fafe5 100644 --- a/lang/en/domain.php +++ b/lang/en/domain.php @@ -28,10 +28,13 @@ return [ 'state' => [ 'verified' => 'Live', + 'not_served' => 'Proven, not reachable yet', 'pending' => 'Waiting for proof', 'none' => 'No custom domain', ], + 'not_served_hint' => 'Ownership is proven and the domain is set up on our side — but nothing answers at this address yet. Usually the A record from step 1 is still missing, or it has not propagated everywhere. Until then your cloud stays reachable at the platform address.', + 'checked_at' => 'Last checked: :when', 'never_checked' => 'Not checked yet.', diff --git a/resources/views/livewire/custom-domain.blade.php b/resources/views/livewire/custom-domain.blade.php index ed74ac7..4a42129 100644 --- a/resources/views/livewire/custom-domain.blade.php +++ b/resources/views/livewire/custom-domain.blade.php @@ -16,8 +16,14 @@

{{ $instance->address(\App\Support\ProvisioningSettings::dnsZone()) }}

- @if ($instance->domainIsVerified()) + {{-- Three states, not two. A domain can be proven and still + answer nothing, because pointing it at us is an A record + in the customer's own zone — and this card was printing + that address as if it worked. --}} + @if ($instance->domainIsServed()) {{ __('domain.state.verified') }} + @elseif ($instance->domainIsVerified()) + {{ __('domain.state.not_served') }} @elseif (filled($instance->custom_domain)) {{ __('domain.state.pending') }} @else @@ -30,6 +36,9 @@ {{ __('domain.platform_address') }}: {{ $platformAddress }}

@endif + @if ($instance->domainIsVerified() && ! $instance->domainIsServed()) + {{ __('domain.not_served_hint') }} + @endif diff --git a/tests/Feature/CustomDomainServingTest.php b/tests/Feature/CustomDomainServingTest.php new file mode 100644 index 0000000..e35ed99 --- /dev/null +++ b/tests/Feature/CustomDomainServingTest.php @@ -0,0 +1,372 @@ +active()->create(['datacenter' => 'fsn', 'node' => 'pve']); + $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']); + + $instance = Instance::factory()->create(array_merge([ + 'order_id' => $order->id, + 'customer_id' => $order->customer_id, + 'host_id' => $host->id, + 'vmid' => 101, + 'guest_ip' => '10.20.0.7', + 'subdomain' => $subdomain, + 'status' => 'active', + // The state a re-apply actually finds: built, routed, certified. + 'route_written' => true, + 'routed_hostnames' => [platformAddress($subdomain)], + 'cert_ok' => true, + ], $attributes)); + + return compact('host', 'order', 'instance'); +} + +/** A run of the `address` pipeline against that instance's order. */ +function addressRun(array $fixture): ProvisioningRun +{ + return ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, + 'subject_id' => $fixture['order']->id, + 'pipeline' => 'address', + 'context' => [ + 'instance_id' => $fixture['instance']->id, + 'host_id' => $fixture['host']->id, + 'node' => 'pve', + 'vmid' => 101, + 'subdomain' => $fixture['instance']->subdomain, + ], + ]); +} + +/** No real DNS: a suite that needs a nameserver fails on a train. */ +function bindDomainResolver(array $records): void +{ + app()->bind(DomainVerifier::class, fn () => new DomainVerifier( + fn (string $name) => $records[$name] ?? [], + )); +} + +function platformAddress(string $subdomain = 'berger'): string +{ + return $subdomain.'.'.ProvisioningSettings::dnsZone(); +} + +/** The portal account that owns the fixture's instance. */ +function signIntoPortal(array $fixture): User +{ + $user = User::factory()->create(['email_verified_at' => now()]); + + // customers.user_id, not users.customer_id — the link runs the other way. + $fixture['order']->customer->update(['user_id' => $user->id, 'email' => $user->email]); + + return $user; +} + +it('puts a verified custom domain in the router beside the platform address, and an unverified one nowhere', function () { + $s = fakeServices(); + + // Unverified: the hostname is in the column, so everything that reads the + // column would announce it — but it is a request, not an address. + $unproven = servedInstance(['custom_domain' => 'cloud.fremde-firma.at', 'domain_token' => 'tok'], 'fremde'); + expect(app(ConfigureDnsAndTls::class)->execute(addressRun($unproven))->type)->toBe('advance') + ->and($s['traefik']->writes)->toBe(0) // nothing changed, so nothing was written + ->and($unproven['instance']->fresh()->routed_hostnames)->toBe([platformAddress('fremde')]); + + // Proven: one router, both names, the platform address first — it is what + // the instance falls back to the moment the domain goes away. + $proven = servedInstance([ + 'custom_domain' => 'cloud.berger.at', + 'domain_token' => 'tok', + 'domain_verified_at' => now(), + ]); + + expect(app(ConfigureDnsAndTls::class)->execute(addressRun($proven))->type)->toBe('advance') + ->and($s['traefik']->hostnames['berger'])->toBe([platformAddress(), 'cloud.berger.at']) + ->and($proven['instance']->fresh()->routed_hostnames)->toBe([platformAddress(), 'cloud.berger.at']) + ->and($proven['instance']->fresh()->domain_cert_ok)->toBeTrue(); +}); + +it('writes one router file for both names rather than a second one', function () { + // Two routers for one backend are two things to keep in step, and the + // second is the one nobody remembers on the day the domain is withdrawn. + $s = fakeServices(); + $fixture = servedInstance([ + 'custom_domain' => 'cloud.berger.at', 'domain_token' => 'tok', 'domain_verified_at' => now(), + ]); + + app(ConfigureDnsAndTls::class)->execute(addressRun($fixture)); + + expect($s['traefik']->writes)->toBe(1) + ->and(array_keys($s['traefik']->hostnames))->toBe(['berger']); +}); + +it('stops serving a withdrawn domain and takes it out of trusted_domains', function () { + $s = fakeServices(); + + // Verified yesterday, served ever since, and withdrawn this morning: the + // proof is gone but the router and Nextcloud have never been told. + $fixture = servedInstance([ + 'custom_domain' => 'cloud.berger.at', + 'domain_token' => 'tok', + 'domain_verified_at' => null, + 'domain_cert_ok' => true, + 'routed_hostnames' => [platformAddress(), 'cloud.berger.at'], + ]); + + $run = addressRun($fixture); + + expect(app(ConfigureDnsAndTls::class)->execute($run)->type)->toBe('advance') + ->and(app(ConfigureNextcloud::class)->execute($run)->type)->toBe('advance'); + + expect($s['traefik']->hostnames['berger'])->toBe([platformAddress()]) + ->and($s['traefik']->serves('berger', 'cloud.berger.at'))->toBeFalse() + ->and($fixture['instance']->fresh()->routed_hostnames)->toBe([platformAddress()]) + ->and($fixture['instance']->fresh()->domain_cert_ok)->toBeFalse() + // Deleted, not blanked: an entry left in trusted_domains means Nextcloud + // still answers to that name whenever anything reaches it. + ->and($s['pve']->guestRan('config:system:delete trusted_domains 2'))->toBeTrue() + ->and($s['pve']->guestRan('config:system:set trusted_domains 2'))->toBeFalse(); +}); + +it('trusts a verified domain and leaves the delete alone', function () { + $s = fakeServices(); + $fixture = servedInstance([ + 'custom_domain' => 'cloud.berger.at', 'domain_token' => 'tok', 'domain_verified_at' => now(), + ]); + + app(ConfigureNextcloud::class)->execute(addressRun($fixture)); + + expect($s['pve']->guestRan("config:system:set trusted_domains 2 --value='cloud.berger.at'"))->toBeTrue() + ->and($s['pve']->guestRan('config:system:delete trusted_domains 2'))->toBeFalse(); +}); + +it('never fails a run over the custom domain’s certificate, but still fails over the platform’s', function () { + $s = fakeServices(); + + // The customer has proven the domain is theirs and has NOT pointed it at + // us — which is an A record in their own zone, may take days, and may never + // happen at all. Their cloud still has to be delivered. + $s['traefik']->certUnreachable = ['cloud.berger.at']; + $fixture = servedInstance([ + 'custom_domain' => 'cloud.berger.at', + 'domain_token' => 'tok', + 'domain_verified_at' => now(), + 'cert_ok' => false, + ]); + $run = addressRun($fixture); + $run->update(['started_at' => now()->subSeconds(300)]); // past the short look + + expect(app(ConfigureDnsAndTls::class)->execute($run)->type)->toBe('advance'); + + $instance = $fixture['instance']->fresh(); + expect($instance->cert_ok)->toBeTrue() // ours works + ->and($instance->domain_verified_at)->not->toBeNull() + ->and($instance->domain_cert_ok)->toBeFalse() // theirs does not, and is recorded as such + ->and($instance->domainIsVerified())->toBeTrue() + ->and($instance->domainIsServed())->toBeFalse() + // Visible to an operator reading the run, not swallowed. + ->and($run->events()->where('outcome', 'info')->where('step', 'configure_dns_and_tls')->exists())->toBeTrue(); + + // The platform address is ours: no certificate there is a broken run. + $s2 = fakeServices(); + $s2['traefik']->certReady = false; + $second = servedInstance(['cert_ok' => false], 'zweite'); + $failing = addressRun($second); + $failing->update(['started_at' => now()->subSeconds(900)]); + + $result = app(ConfigureDnsAndTls::class)->execute($failing); + expect($result->type)->toBe('fail') + ->and($result->reason)->toBe('cert_timeout'); +}); + +it('looks again for a short while before giving up on the custom certificate', function () { + // Traefik finishes HTTP-01 seconds after the first request to a new + // hostname, so advancing on the very first probe would mark a domain that + // is about to work as not working. + $s = fakeServices(); + $s['traefik']->certUnreachable = ['cloud.berger.at']; + $fixture = servedInstance([ + 'custom_domain' => 'cloud.berger.at', 'domain_token' => 'tok', 'domain_verified_at' => now(), + ]); + $run = addressRun($fixture); + $run->update(['started_at' => now()->subSeconds(5)]); + + expect(app(ConfigureDnsAndTls::class)->execute($run)->type)->toBe('poll'); +}); + +it('starts exactly one re-apply when verification flips, and none when nothing changed', function () { + Queue::fake(); + $fixture = servedInstance(['custom_domain' => 'cloud.berger.at', 'domain_token' => 'tok123']); + bindDomainResolver(['_clupilot-challenge.cloud.berger.at' => [['txt' => 'cp-verify=tok123']]]); + + // The flip: proof appears, so the domain has to become an address. + $this->artisan('clupilot:verify-domains')->assertSuccessful(); + + $runs = ProvisioningRun::query()->where('pipeline', 'address'); + expect($runs->count())->toBe(1) + ->and($fixture['instance']->fresh()->domainIsVerified())->toBeTrue(); + + // Finished, so nothing is in flight to suppress a second one — the only + // thing stopping it must be that nothing changed. + $runs->first()->update(['status' => ProvisioningRun::STATUS_COMPLETED]); + + $this->artisan('clupilot:verify-domains')->assertSuccessful(); + + expect(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1); + + // And the other flip: the proof is taken away for the third time running. + bindDomainResolver([]); + $this->artisan('clupilot:verify-domains'); + $this->artisan('clupilot:verify-domains'); + expect(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1); + + $this->artisan('clupilot:verify-domains'); + expect($fixture['instance']->fresh()->domainIsVerified())->toBeFalse() + ->and(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(2); +}); + +it('stops serving the domain when the package stops allowing one', function () { + $s = fakeServices(); + Queue::fake(); + + $fixture = servedInstance([ + 'custom_domain' => 'cloud.berger.at', + 'domain_token' => 'tok', + 'domain_verified_at' => now(), + 'domain_cert_ok' => true, + 'routed_hostnames' => [platformAddress(), 'cloud.berger.at'], + ]); + + // What PlanChange::settleCustomDomain() reaches through CustomDomainAccess + // when a downgrade lands on a package that carries no own domain. + expect(app(CustomDomainAccess::class)->deactivate($fixture['instance']))->toBeTrue(); + + $run = ProvisioningRun::query()->where('pipeline', 'address')->latest('id')->first(); + expect($run)->not->toBeNull(); + + app(ConfigureDnsAndTls::class)->execute($run); + app(ConfigureNextcloud::class)->execute($run); + + expect($fixture['instance']->fresh()->custom_domain)->toBeNull() + ->and($s['traefik']->serves('berger', 'cloud.berger.at'))->toBeFalse() + ->and($s['traefik']->hostnames['berger'])->toBe([platformAddress()]) + ->and($s['pve']->guestRan('config:system:delete trusted_domains 2'))->toBeTrue(); +}); + +it('does not start a second run while one is already going', function () { + Queue::fake(); + $fixture = servedInstance([ + 'custom_domain' => 'cloud.berger.at', 'domain_token' => 'tok', 'domain_verified_at' => now(), + ]); + + $reapply = app(ReapplyInstanceAddress::class); + + expect($reapply($fixture['instance']))->not->toBeNull() + ->and($reapply($fixture['instance']))->toBeNull() + ->and(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1); + + // Nor beside the build itself: the customer run applies the address on its + // way past, and two runs writing one router is the thing being avoided. + ProvisioningRun::query()->where('pipeline', 'address')->update(['status' => ProvisioningRun::STATUS_COMPLETED]); + ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, + 'subject_id' => $fixture['order']->id, + 'pipeline' => 'customer', + 'status' => ProvisioningRun::STATUS_RUNNING, + ]); + + expect($reapply($fixture['instance']))->toBeNull() + ->and(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1); +}); + +it('stops serving the old domain the moment the customer changes or clears it', function () { + Queue::fake(); + + $fixture = servedInstance([ + 'custom_domain' => 'cloud.berger.at', + 'domain_token' => 'tok', + 'domain_verified_at' => now(), + 'domain_cert_ok' => true, + 'routed_hostnames' => [platformAddress(), 'cloud.berger.at'], + ]); + $user = signIntoPortal($fixture); + + // Moving to another domain leaves the old one routed and trusted until + // something says otherwise — the new one is unproven and serves nothing. + Livewire::actingAs($user)->test(CustomDomain::class) + ->set('domain', 'neu.berger.at') + ->call('save') + ->assertHasNoErrors(); + + expect(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1) + ->and($fixture['instance']->fresh()->domain_cert_ok)->toBeFalse(); + + ProvisioningRun::query()->where('pipeline', 'address')->update(['status' => ProvisioningRun::STATUS_COMPLETED]); + + // Clearing an unproven domain changes nothing that is being served, so it + // is not worth a run on a live machine. + Livewire::actingAs($user)->test(CustomDomain::class) + ->set('domain', '') + ->call('save'); + + expect($fixture['instance']->fresh()->custom_domain)->toBeNull() + ->and(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1); +}); + +it('makes the domain an address as soon as the customer’s own check finds the proof', function () { + Queue::fake(); + + $fixture = servedInstance(['custom_domain' => 'cloud.berger.at', 'domain_token' => 'tok123']); + $user = signIntoPortal($fixture); + bindDomainResolver(['_clupilot-challenge.cloud.berger.at' => [['txt' => 'cp-verify=tok123']]]); + + Livewire::actingAs($user)->test(CustomDomain::class)->call('checkNow'); + + expect($fixture['instance']->fresh()->domainIsVerified())->toBeTrue() + ->and(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1); +}); + +it('has nothing to re-apply for an instance with no machine', function () { + Queue::fake(); + + // A reservation that never became a VM, and a build that failed: the steps + // would reach for a guest agent that is not there and spend the run's + // retries finding out. + $reserved = Instance::factory()->create(['status' => 'reserving', 'vmid' => null]); + + expect(app(ReapplyInstanceAddress::class)($reserved))->toBeNull() + ->and(app(ReapplyInstanceAddress::class)(null))->toBeNull() + ->and(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(0); +}); diff --git a/tests/Feature/Provisioning/RunRunnerTest.php b/tests/Feature/Provisioning/RunRunnerTest.php index f3efaeb..f364741 100644 --- a/tests/Feature/Provisioning/RunRunnerTest.php +++ b/tests/Feature/Provisioning/RunRunnerTest.php @@ -106,9 +106,11 @@ it('keeps started_at stable across polls so step deadlines can accumulate', func }); it('marks a Host subject as error when the run fails', function () { - bindPipeline(['test' => [FakeFailStep::class]]); + // Named 'host' rather than 'test': the failure hook belongs to the pipeline + // that BUILDS the subject, and the runner now asks which one that is. + bindPipeline(['host' => [FakeFailStep::class]]); $host = Host::factory()->create(['status' => 'onboarding']); - $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'test']); + $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); app(RunRunner::class)->advance($run); @@ -116,6 +118,21 @@ it('marks a Host subject as error when the run fails', function () { ->and($host->fresh()->status)->toBe('error'); }); +it('does not condemn the subject when a maintenance run fails', function () { + // A run that maintains something already built shares its subject. If its + // failure fired the same hook, a router file that could not be written + // would put a live, paid-for host into 'error' — and, on the customer side, + // mark the order failed and release the running instance with it. + bindPipeline(['address' => [FakeFailStep::class]]); + $host = Host::factory()->create(['status' => 'active']); + $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'address']); + + app(RunRunner::class)->advance($run); + + expect($run->fresh()->status)->toBe('failed') + ->and($host->fresh()->status)->toBe('active'); +}); + it('treats a thrown exception as a retry', function () { bindPipeline(['test' => [FakeThrowStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 5]);