From 36526928df6c896749c97a8e7abb63ab06fe0b4a Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 22 May 2026 12:44:41 +0200 Subject: [PATCH] feat(sentinel): deduplicate metrics push processing Move Sentinel push handling into a controller and dispatch server update jobs only when container state changes or the force interval elapses. Add opt-in PostgreSQL read/write replica configuration and tune periodic proxy network and storage checks to reduce unnecessary work. Add feature coverage for replica config, Sentinel push deduplication, deployment log scrolling, and server update job optimizations. --- .env.development.example | 12 ++ .../Controllers/Api/SentinelController.php | 146 ++++++++++++++++++ app/Jobs/PushServerUpdateJob.php | 17 +- config/constants.php | 15 ++ config/database.php | 58 +++++-- routes/api.php | 73 +-------- tests/Feature/DatabaseReplicaConfigTest.php | 74 +++++++++ tests/Feature/DeploymentLogScrollTest.php | 99 ++++++++++++ .../PushServerUpdateJobOptimizationTest.php | 69 +++++++-- .../Feature/SentinelPushDeduplicationTest.php | 119 ++++++++++++++ 10 files changed, 577 insertions(+), 105 deletions(-) create mode 100644 app/Http/Controllers/Api/SentinelController.php create mode 100644 tests/Feature/DatabaseReplicaConfigTest.php create mode 100644 tests/Feature/DeploymentLogScrollTest.php create mode 100644 tests/Feature/SentinelPushDeduplicationTest.php diff --git a/.env.development.example b/.env.development.example index 594b89201..d02b8ba59 100644 --- a/.env.development.example +++ b/.env.development.example @@ -15,6 +15,18 @@ DB_PASSWORD=password DB_HOST=host.docker.internal DB_PORT=5432 +# Read/write replicas (optional). Set DB_READ_HOST to enable the read/write split. +# Hosts may be comma-separated. Port/username/password fall back to DB_* when unset. +# DB_READ_HOST=replica1,replica2 +# DB_READ_PORT=5432 +# DB_READ_USERNAME=coolify +# DB_READ_PASSWORD= +# DB_WRITE_HOST= +# DB_WRITE_PORT=5432 +# DB_WRITE_USERNAME=coolify +# DB_WRITE_PASSWORD= +# DB_STICKY=true + # Ray Configuration # Set to true to enable Ray RAY_ENABLED=false diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php new file mode 100644 index 000000000..4a469f09c --- /dev/null +++ b/app/Http/Controllers/Api/SentinelController.php @@ -0,0 +1,146 @@ +header('Authorization'); + if (! $token) { + auditLogWebhookFailure('sentinel', 'token_missing'); + + return response()->json(['message' => 'Unauthorized'], 401); + } + $naked_token = str_replace('Bearer ', '', $token); + try { + $decrypted = decrypt($naked_token); + $decrypted_token = json_decode($decrypted, true); + } catch (Exception $e) { + auditLogWebhookFailure('sentinel', 'decrypt_failed'); + + return response()->json(['message' => 'Invalid token'], 401); + } + $server_uuid = data_get($decrypted_token, 'server_uuid'); + if (! $server_uuid) { + auditLogWebhookFailure('sentinel', 'invalid_token_payload'); + + return response()->json(['message' => 'Invalid token'], 401); + } + $server = Server::where('uuid', $server_uuid)->first(); + if (! $server) { + auditLogWebhookFailure('sentinel', 'server_not_found', [ + 'server_uuid' => $server_uuid, + ]); + + return response()->json(['message' => 'Server not found'], 404); + } + + if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { + auditLogWebhookFailure('sentinel', 'subscription_unpaid', [ + 'server_uuid' => $server->uuid, + 'team_id' => $server->team_id, + ]); + + return response()->json(['message' => 'Unauthorized'], 401); + } + + if ($server->isFunctional() === false) { + auditLogWebhookFailure('sentinel', 'server_not_functional', [ + 'server_uuid' => $server->uuid, + 'team_id' => $server->team_id, + ]); + + return response()->json(['message' => 'Server is not functional'], 401); + } + + if ($server->settings->sentinel_token !== $naked_token) { + auditLogWebhookFailure('sentinel', 'token_mismatch', [ + 'server_uuid' => $server->uuid, + 'team_id' => $server->team_id, + ]); + + return response()->json(['message' => 'Unauthorized'], 401); + } + $data = $request->all(); + + // Heartbeat MUST update on every push — drives isSentinelLive() and SSH-check skipping. + $server->sentinelHeartbeat(); + + if ($this->shouldDispatchUpdate($server, $data)) { + PushServerUpdateJob::dispatch($server, $data); + } + + auditLog('sentinel.metrics_pushed', [ + 'server_uuid' => $server->uuid, + 'team_id' => $server->team_id, + ]); + + return response()->json(['message' => 'ok'], 200); + } + + /** + * Decide whether PushServerUpdateJob should be dispatched for this push. + * + * Dispatches when: first push (no cached hash), the container state changed, + * or the force window elapsed. + */ + private function shouldDispatchUpdate(Server $server, array $data): bool + { + $hash = $this->containerStateHash($data); + $hashKey = "sentinel:push-hash:{$server->id}"; + $forceKey = "sentinel:push-force:{$server->id}"; + + $cachedHash = Cache::get($hashKey); + $forceActive = Cache::has($forceKey); + + $shouldDispatch = $cachedHash === null || $cachedHash !== $hash || ! $forceActive; + + if ($shouldDispatch) { + // Day-long TTL bounds memory if a server stops pushing entirely. + Cache::put($hashKey, $hash, now()->addDay()); + Cache::put($forceKey, true, config('constants.sentinel.push_force_interval_seconds', 300)); + } + + return $shouldDispatch; + } + + /** + * Build a stable hash of container state. + * + * Covers [name, state, health_status] only — metrics and + * filesystem_usage_root are excluded on purpose (disk % churns constantly + * and would defeat the hash; the storage check is separately cache-gated + * inside PushServerUpdateJob). Sorted by name so container ordering from + * Sentinel does not affect the hash. + */ + private function containerStateHash(array $data): string + { + $containers = collect(data_get($data, 'containers', [])) + ->map(fn ($c) => [ + 'name' => data_get($c, 'name'), + 'state' => data_get($c, 'state'), + 'health_status' => data_get($c, 'health_status'), + ]) + ->sortBy('name') + ->values() + ->all(); + + return hash('xxh128', json_encode($containers)); + } +} diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index b1a12ae2a..cdfa174ed 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -127,15 +127,20 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced } $data = collect($this->data); - $this->server->sentinelHeartbeat(); - + // Heartbeat is updated by SentinelController on every push, before dispatch. $this->containers = collect(data_get($data, 'containers')); $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage'); - // Only dispatch storage check when disk percentage actually changes + // Only dispatch the storage check when disk usage is at/above the notification + // threshold AND the value changed. Below the threshold ServerStorageCheckJob + // has nothing to do (it only sends a HighDiskUsage notification), so dispatching + // it is wasted work — and most servers sit well below the threshold. + $diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80); $storageCacheKey = 'storage-check:'.$this->server->id; $lastPercentage = Cache::get($storageCacheKey); - if ($lastPercentage === null || (string) $lastPercentage !== (string) $filesystemUsageRoot) { + if ($filesystemUsageRoot !== null + && $filesystemUsageRoot >= $diskThreshold + && (string) $lastPercentage !== (string) $filesystemUsageRoot) { Cache::put($storageCacheKey, $filesystemUsageRoot, 600); ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot); } @@ -500,11 +505,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced } catch (\Throwable $e) { } } else { - // Connect proxy to networks periodically (every 10 min) to avoid excessive job dispatches. + // Connect proxy to networks periodically as a safety net to avoid excessive job dispatches. // On-demand triggers (new network, service deploy) use dispatchSync() and bypass this. $proxyCacheKey = 'connect-proxy:'.$this->server->id; if (! Cache::has($proxyCacheKey)) { - Cache::put($proxyCacheKey, true, 600); + Cache::put($proxyCacheKey, true, config('constants.proxy.connect_networks_interval_seconds', 3600)); ConnectProxyToNetworksJob::dispatch($this->server); } } diff --git a/config/constants.php b/config/constants.php index bd3e5b2aa..f4a32be6a 100644 --- a/config/constants.php +++ b/config/constants.php @@ -94,6 +94,21 @@ return [ 'sentry_dsn' => env('SENTRY_DSN'), ], + 'sentinel' => [ + // How often (seconds) PushServerUpdateJob is force-dispatched even when + // the container state hash is unchanged. Keeps last_online_at, + // exited-detection and storage checks from going stale. + 'push_force_interval_seconds' => env('SENTINEL_PUSH_FORCE_INTERVAL_SECONDS', 300), + ], + + 'proxy' => [ + // How often (seconds) PushServerUpdateJob periodically re-connects the + // proxy to Docker networks as a safety net. Real network-layout changes + // already connect the proxy on-demand; this only covers gaps (Swarm + // networks added via UI, proxy crash recovery). + 'connect_networks_interval_seconds' => env('PROXY_CONNECT_NETWORKS_INTERVAL_SECONDS', 3600), + ], + 'webhooks' => [ 'feedback_discord_webhook' => env('FEEDBACK_DISCORD_WEBHOOK'), 'dev_webhook' => env('SERVEO_URL'), diff --git a/config/database.php b/config/database.php index a5e0ba703..94c27f038 100644 --- a/config/database.php +++ b/config/database.php @@ -1,6 +1,46 @@ 'pgsql', + 'url' => env('DATABASE_URL'), + 'host' => env('DB_HOST', 'coolify-db'), + 'port' => env('DB_PORT', '5432'), + 'database' => env('DB_DATABASE', 'coolify'), + 'username' => env('DB_USERNAME', 'coolify'), + 'password' => env('DB_PASSWORD', ''), + 'charset' => 'utf8', + 'prefix' => '', + 'prefix_indexes' => true, + 'search_path' => 'public', + 'sslmode' => 'prefer', + 'options' => [ + (defined('Pdo\Pgsql::ATTR_DISABLE_PREPARES') ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => env('DB_DISABLE_PREPARES', false), + ], +]; + +/* + * Opt-in read/write replica split. Activates only when DB_READ_HOST is set. + * When unset, the pgsql connection is identical to a single-primary setup. + * Hosts may be comma-separated; Laravel random-picks one per connection. + */ +if (env('DB_READ_HOST')) { + $pgsql['read'] = [ + 'host' => array_map('trim', explode(',', (string) env('DB_READ_HOST'))), + 'port' => env('DB_READ_PORT', env('DB_PORT', '5432')), + 'username' => env('DB_READ_USERNAME', env('DB_USERNAME', 'coolify')), + 'password' => env('DB_READ_PASSWORD', env('DB_PASSWORD', '')), + ]; + $pgsql['write'] = [ + 'host' => array_map('trim', explode(',', (string) env('DB_WRITE_HOST', env('DB_HOST', 'coolify-db')))), + 'port' => env('DB_WRITE_PORT', env('DB_PORT', '5432')), + 'username' => env('DB_WRITE_USERNAME', env('DB_USERNAME', 'coolify')), + 'password' => env('DB_WRITE_PASSWORD', env('DB_PASSWORD', '')), + ]; + $pgsql['sticky'] = (bool) env('DB_STICKY', true); +} return [ @@ -35,23 +75,7 @@ return [ 'connections' => [ - 'pgsql' => [ - 'driver' => 'pgsql', - 'url' => env('DATABASE_URL'), - 'host' => env('DB_HOST', 'coolify-db'), - 'port' => env('DB_PORT', '5432'), - 'database' => env('DB_DATABASE', 'coolify'), - 'username' => env('DB_USERNAME', 'coolify'), - 'password' => env('DB_PASSWORD', ''), - 'charset' => 'utf8', - 'prefix' => '', - 'prefix_indexes' => true, - 'search_path' => 'public', - 'sslmode' => 'prefer', - 'options' => [ - (defined('Pdo\Pgsql::ATTR_DISABLE_PREPARES') ? \Pdo\Pgsql::ATTR_DISABLE_PREPARES : \PDO::PGSQL_ATTR_DISABLE_PREPARES) => env('DB_DISABLE_PREPARES', false), - ], - ], + 'pgsql' => $pgsql, 'testing' => [ 'driver' => 'sqlite', diff --git a/routes/api.php b/routes/api.php index cc380b2be..fb3b4bad6 100644 --- a/routes/api.php +++ b/routes/api.php @@ -11,12 +11,11 @@ use App\Http\Controllers\Api\ProjectController; use App\Http\Controllers\Api\ResourcesController; use App\Http\Controllers\Api\ScheduledTasksController; use App\Http\Controllers\Api\SecurityController; +use App\Http\Controllers\Api\SentinelController; use App\Http\Controllers\Api\ServersController; use App\Http\Controllers\Api\ServicesController; use App\Http\Controllers\Api\TeamController; use App\Http\Middleware\ApiAllowed; -use App\Jobs\PushServerUpdateJob; -use App\Models\Server; use Illuminate\Support\Facades\Route; Route::get('/health', [OtherController::class, 'healthcheck']); @@ -209,75 +208,7 @@ Route::group([ Route::group([ 'prefix' => 'v1', ], function () { - Route::post('/sentinel/push', function () { - $token = request()->header('Authorization'); - if (! $token) { - auditLogWebhookFailure('sentinel', 'token_missing'); - - return response()->json(['message' => 'Unauthorized'], 401); - } - $naked_token = str_replace('Bearer ', '', $token); - try { - $decrypted = decrypt($naked_token); - $decrypted_token = json_decode($decrypted, true); - } catch (Exception $e) { - auditLogWebhookFailure('sentinel', 'decrypt_failed'); - - return response()->json(['message' => 'Invalid token'], 401); - } - $server_uuid = data_get($decrypted_token, 'server_uuid'); - if (! $server_uuid) { - auditLogWebhookFailure('sentinel', 'invalid_token_payload'); - - return response()->json(['message' => 'Invalid token'], 401); - } - $server = Server::where('uuid', $server_uuid)->first(); - if (! $server) { - auditLogWebhookFailure('sentinel', 'server_not_found', [ - 'server_uuid' => $server_uuid, - ]); - - return response()->json(['message' => 'Server not found'], 404); - } - - if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { - auditLogWebhookFailure('sentinel', 'subscription_unpaid', [ - 'server_uuid' => $server->uuid, - 'team_id' => $server->team_id, - ]); - - return response()->json(['message' => 'Unauthorized'], 401); - } - - if ($server->isFunctional() === false) { - auditLogWebhookFailure('sentinel', 'server_not_functional', [ - 'server_uuid' => $server->uuid, - 'team_id' => $server->team_id, - ]); - - return response()->json(['message' => 'Server is not functional'], 401); - } - - if ($server->settings->sentinel_token !== $naked_token) { - auditLogWebhookFailure('sentinel', 'token_mismatch', [ - 'server_uuid' => $server->uuid, - 'team_id' => $server->team_id, - ]); - - return response()->json(['message' => 'Unauthorized'], 401); - } - $data = request()->all(); - - // \App\Jobs\ServerCheckNewJob::dispatch($server, $data); - PushServerUpdateJob::dispatch($server, $data); - - auditLog('sentinel.metrics_pushed', [ - 'server_uuid' => $server->uuid, - 'team_id' => $server->team_id, - ]); - - return response()->json(['message' => 'ok'], 200); - }); + Route::post('/sentinel/push', [SentinelController::class, 'push']); }); Route::any('/{any}', function () { diff --git a/tests/Feature/DatabaseReplicaConfigTest.php b/tests/Feature/DatabaseReplicaConfigTest.php new file mode 100644 index 000000000..8a9c58a38 --- /dev/null +++ b/tests/Feature/DatabaseReplicaConfigTest.php @@ -0,0 +1,74 @@ +not->toHaveKey('read') + ->not->toHaveKey('write') + ->not->toHaveKey('sticky') + ->and($pgsql['driver'])->toBe('pgsql'); +}); + +it('enables the read/write split when DB_READ_HOST is set', function () { + putenv('DB_READ_HOST=replica1, replica2'); + + $pgsql = loadDbConfig()['connections']['pgsql']; + + expect($pgsql) + ->toHaveKey('read') + ->toHaveKey('write') + ->and($pgsql['read']['host'])->toBe(['replica1', 'replica2']) + ->and($pgsql['sticky'])->toBeTrue(); +}); + +it('falls back to DB_* values for unset replica options', function () { + putenv('DB_READ_HOST=replica1'); + + $pgsql = loadDbConfig()['connections']['pgsql']; + + expect($pgsql['read']['port'])->toBe(env('DB_PORT', '5432')) + ->and($pgsql['read']['username'])->toBe(env('DB_USERNAME', 'coolify')) + ->and($pgsql['write']['host'])->toBe([env('DB_HOST', 'coolify-db')]); +}); + +it('respects discrete replica overrides', function () { + putenv('DB_READ_HOST=replica1'); + putenv('DB_READ_PORT=6432'); + putenv('DB_READ_USERNAME=reader'); + + $pgsql = loadDbConfig()['connections']['pgsql']; + + expect($pgsql['read']['port'])->toBe('6432') + ->and($pgsql['read']['username'])->toBe('reader'); +}); + +it('disables sticky reads when DB_STICKY is false', function () { + putenv('DB_READ_HOST=replica1'); + putenv('DB_STICKY=false'); + + $pgsql = loadDbConfig()['connections']['pgsql']; + + expect($pgsql['sticky'])->toBeFalse(); +}); diff --git a/tests/Feature/DeploymentLogScrollTest.php b/tests/Feature/DeploymentLogScrollTest.php new file mode 100644 index 000000000..c40752542 --- /dev/null +++ b/tests/Feature/DeploymentLogScrollTest.php @@ -0,0 +1,99 @@ +user = User::factory()->create(); + $this->team = Team::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + + InstanceSettings::unguarded(function () { + InstanceSettings::query()->create([ + 'id' => 0, + 'is_registration_enabled' => true, + ]); + }); + + $this->actingAs($this->user); + session(['currentTeam' => $this->team]); + + $this->server = Server::factory()->create(['team_id' => $this->team->id]); + $this->destination = StandaloneDocker::query()->where('server_id', $this->server->id)->firstOrFail(); + $this->project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $this->project->id]); + $this->application = Application::factory()->create([ + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + 'status' => 'running', + ]); +}); + +function showDeployment(string $status): TestResponse +{ + $deployment = ApplicationDeploymentQueue::create([ + 'application_id' => test()->application->id, + 'deployment_uuid' => 'deploy-scroll-'.$status, + 'server_id' => test()->server->id, + 'status' => $status, + 'logs' => json_encode([[ + 'command' => null, + 'output' => 'log line for '.$status, + 'type' => 'stdout', + 'timestamp' => now()->toISOString(), + 'hidden' => false, + 'batch' => 1, + 'order' => 1, + ]], JSON_THROW_ON_ERROR), + ]); + + return test()->get(route('project.application.deployment.show', [ + 'project_uuid' => test()->project->uuid, + 'environment_uuid' => test()->environment->uuid, + 'application_uuid' => test()->application->uuid, + 'deployment_uuid' => $deployment->deployment_uuid, + ])); +} + +it('does not enable follow mode for a finished deployment', function () { + $response = showDeployment(ApplicationDeploymentStatus::FINISHED->value); + + $response->assertSuccessful(); + $response->assertSee('alwaysScroll: false', false); + $response->assertDontSee('alwaysScroll: true', false); +}); + +it('enables follow mode for an in-progress deployment', function () { + $response = showDeployment(ApplicationDeploymentStatus::IN_PROGRESS->value); + + $response->assertSuccessful(); + $response->assertSee('alwaysScroll: true', false); +}); + +it('scopes scroll teardown to the component so a stale loop cannot leak across deployments', function () { + $content = showDeployment(ApplicationDeploymentStatus::FINISHED->value)->getContent(); + + // Alpine destroy() tears the scroll loop down on wire:navigate away. + expect($content)->toContain('destroy()') + ->toContain('cancelScrollLoop()') + // Container lookup is component-scoped, not a global getElementById. + ->toContain("this.\$root.querySelector('#logsContainer')") + ->not->toContain("document.getElementById('logsContainer')") + // morph.updated hook only acts on this component's own DOM. + ->toContain('this.$root.contains(el)') + // Continuation timeout is tracked so it can be cancelled. + ->toContain('scrollTimeout'); +}); diff --git a/tests/Feature/PushServerUpdateJobOptimizationTest.php b/tests/Feature/PushServerUpdateJobOptimizationTest.php index eb51059db..92c98a2e1 100644 --- a/tests/Feature/PushServerUpdateJobOptimizationTest.php +++ b/tests/Feature/PushServerUpdateJobOptimizationTest.php @@ -16,10 +16,29 @@ beforeEach(function () { Cache::flush(); }); -it('dispatches storage check when disk percentage changes', function () { +it('dispatches storage check when disk percentage changes above threshold', function () { $team = Team::factory()->create(); $server = Server::factory()->create(['team_id' => $team->id]); + // Default notification threshold is 80%. + $data = [ + 'containers' => [], + 'filesystem_usage_root' => ['used_percentage' => 85], + ]; + + $job = new PushServerUpdateJob($server, $data); + $job->handle(); + + Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { + return $job->server->id === $server->id && $job->percentage === 85; + }); +}); + +it('does not dispatch storage check when disk usage is below threshold', function () { + $team = Team::factory()->create(); + $server = Server::factory()->create(['team_id' => $team->id]); + + // 45% is well below the default 80% notification threshold — nothing to do. $data = [ 'containers' => [], 'filesystem_usage_root' => ['used_percentage' => 45], @@ -28,21 +47,19 @@ it('dispatches storage check when disk percentage changes', function () { $job = new PushServerUpdateJob($server, $data); $job->handle(); - Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { - return $job->server->id === $server->id && $job->percentage === 45; - }); + Queue::assertNotPushed(ServerStorageCheckJob::class); }); it('does not dispatch storage check when disk percentage is unchanged', function () { $team = Team::factory()->create(); $server = Server::factory()->create(['team_id' => $team->id]); - // Simulate a previous push that cached the percentage - Cache::put('storage-check:'.$server->id, 45, 600); + // Simulate a previous push that cached the percentage (above threshold). + Cache::put('storage-check:'.$server->id, 85, 600); $data = [ 'containers' => [], - 'filesystem_usage_root' => ['used_percentage' => 45], + 'filesystem_usage_root' => ['used_percentage' => 85], ]; $job = new PushServerUpdateJob($server, $data); @@ -55,19 +72,19 @@ it('dispatches storage check when disk percentage changes from cached value', fu $team = Team::factory()->create(); $server = Server::factory()->create(['team_id' => $team->id]); - // Simulate a previous push that cached 45% - Cache::put('storage-check:'.$server->id, 45, 600); + // Simulate a previous push that cached 85% (above threshold). + Cache::put('storage-check:'.$server->id, 85, 600); $data = [ 'containers' => [], - 'filesystem_usage_root' => ['used_percentage' => 50], + 'filesystem_usage_root' => ['used_percentage' => 90], ]; $job = new PushServerUpdateJob($server, $data); $job->handle(); Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { - return $job->server->id === $server->id && $job->percentage === 50; + return $job->server->id === $server->id && $job->percentage === 90; }); }); @@ -140,6 +157,36 @@ it('dispatches ConnectProxyToNetworksJob again after cache expires', function () Queue::assertPushed(ConnectProxyToNetworksJob::class, 1); }); +it('respects the configured proxy connect interval', function () { + // Interval 0 → the connect-proxy gate key expires immediately, so every + // push re-dispatches without a manual Cache::forget. Proves the TTL is + // driven by config('constants.proxy.connect_networks_interval_seconds'). + config(['constants.proxy.connect_networks_interval_seconds' => 0]); + + $team = Team::factory()->create(); + $server = Server::factory()->create(['team_id' => $team->id]); + $server->settings->update(['is_reachable' => true, 'is_usable' => true]); + + $data = [ + 'containers' => [ + [ + 'name' => 'coolify-proxy', + 'state' => 'running', + 'health_status' => 'healthy', + 'labels' => ['coolify.managed' => true], + ], + ], + 'filesystem_usage_root' => ['used_percentage' => 10], + ]; + + (new PushServerUpdateJob($server, $data))->handle(); + Queue::assertPushed(ConnectProxyToNetworksJob::class, 1); + + Queue::fake(); + (new PushServerUpdateJob($server, $data))->handle(); + Queue::assertPushed(ConnectProxyToNetworksJob::class, 1); +}); + it('uses default queue for PushServerUpdateJob', function () { $team = Team::factory()->create(); $server = Server::factory()->create(['team_id' => $team->id]); diff --git a/tests/Feature/SentinelPushDeduplicationTest.php b/tests/Feature/SentinelPushDeduplicationTest.php new file mode 100644 index 000000000..e5995a687 --- /dev/null +++ b/tests/Feature/SentinelPushDeduplicationTest.php @@ -0,0 +1,119 @@ +create(); + $this->team = $user->teams()->first(); + + $this->server = Server::factory()->create([ + 'team_id' => $this->team->id, + ]); + $this->server->settings->update([ + 'is_reachable' => true, + 'is_usable' => true, + ]); + + $this->token = $this->server->settings->sentinel_token; +}); + +function pushSentinel(string $token, array $payload) +{ + return test()->postJson('/api/v1/sentinel/push', $payload, [ + 'Authorization' => 'Bearer '.$token, + ]); +} + +function sentinelPayload(array $containers, ?float $diskPercentage = 42.0): array +{ + return [ + 'containers' => $containers, + 'filesystem_usage_root' => ['used_percentage' => $diskPercentage], + ]; +} + +$running = fn () => [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']]; + +it('dispatches the job on the first push', function () use ($running) { + pushSentinel($this->token, sentinelPayload($running()))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 1); +}); + +it('skips the job when the second push is identical', function () use ($running) { + pushSentinel($this->token, sentinelPayload($running()))->assertOk(); + pushSentinel($this->token, sentinelPayload($running()))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 1); +}); + +it('updates the heartbeat even when the job is skipped', function () use ($running) { + pushSentinel($this->token, sentinelPayload($running()))->assertOk(); + + $this->server->update(['sentinel_updated_at' => now()->subHour()]); + + pushSentinel($this->token, sentinelPayload($running()))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 1); + expect(Carbon::parse($this->server->fresh()->sentinel_updated_at)->diffInSeconds(now()))->toBeLessThan(5); +}); + +it('dispatches the job when container state changes', function () use ($running) { + pushSentinel($this->token, sentinelPayload($running()))->assertOk(); + + $exited = [['name' => 'app-1', 'state' => 'exited', 'health_status' => 'unhealthy']]; + pushSentinel($this->token, sentinelPayload($exited))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 2); +}); + +it('ignores disk percentage changes (excluded from the hash)', function () use ($running) { + pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk(); + pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 1); +}); + +it('ignores container reordering (hash is sorted by name)', function () { + $order1 = [ + ['name' => 'app-a', 'state' => 'running', 'health_status' => 'healthy'], + ['name' => 'app-b', 'state' => 'running', 'health_status' => 'healthy'], + ]; + $order2 = [ + ['name' => 'app-b', 'state' => 'running', 'health_status' => 'healthy'], + ['name' => 'app-a', 'state' => 'running', 'health_status' => 'healthy'], + ]; + + pushSentinel($this->token, sentinelPayload($order1))->assertOk(); + pushSentinel($this->token, sentinelPayload($order2))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 1); +}); + +it('force-dispatches an identical push after the force window expires', function () use ($running) { + pushSentinel($this->token, sentinelPayload($running()))->assertOk(); + + // Simulate the force key TTL elapsing. + Cache::forget('sentinel:push-force:'.$this->server->id); + + pushSentinel($this->token, sentinelPayload($running()))->assertOk(); + + Queue::assertPushed(PushServerUpdateJob::class, 2); +}); + +it('rejects an invalid token without dispatching', function () use ($running) { + pushSentinel('not-a-real-token', sentinelPayload($running()))->assertUnauthorized(); + + Queue::assertNotPushed(PushServerUpdateJob::class); +});