feat(sentinel): deduplicate metrics push processing

Move Sentinel push handling into a controller and dispatch server update jobs only when container state changes or the force interval elapses. Add opt-in PostgreSQL read/write replica configuration and tune periodic proxy network and storage checks to reduce unnecessary work.

Add feature coverage for replica config, Sentinel push deduplication, deployment log scrolling, and server update job optimizations.
This commit is contained in:
Andras Bacsai
2026-05-22 12:44:41 +02:00
parent 5e0e6772d5
commit 36526928df
10 changed files with 577 additions and 105 deletions
+12
View File
@@ -15,6 +15,18 @@ DB_PASSWORD=password
DB_HOST=host.docker.internal DB_HOST=host.docker.internal
DB_PORT=5432 DB_PORT=5432
# Read/write replicas (optional). Set DB_READ_HOST to enable the read/write split.
# Hosts may be comma-separated. Port/username/password fall back to DB_* when unset.
# DB_READ_HOST=replica1,replica2
# DB_READ_PORT=5432
# DB_READ_USERNAME=coolify
# DB_READ_PASSWORD=
# DB_WRITE_HOST=
# DB_WRITE_PORT=5432
# DB_WRITE_USERNAME=coolify
# DB_WRITE_PASSWORD=
# DB_STICKY=true
# Ray Configuration # Ray Configuration
# Set to true to enable Ray # Set to true to enable Ray
RAY_ENABLED=false RAY_ENABLED=false
@@ -0,0 +1,146 @@
<?php
namespace App\Http\Controllers\Api;
use App\Http\Controllers\Controller;
use App\Jobs\PushServerUpdateJob;
use App\Models\Server;
use Exception;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Cache;
class SentinelController extends Controller
{
/**
* Handle a Sentinel agent metrics push.
*
* Sentinel pushes its full container list on a fixed interval (default 60s),
* even when nothing changed. To avoid dispatching one PushServerUpdateJob per
* server per minute, the job is only dispatched when the container state hash
* changes, or when the force window has elapsed.
*/
public function push(Request $request)
{
$token = $request->header('Authorization');
if (! $token) {
auditLogWebhookFailure('sentinel', 'token_missing');
return response()->json(['message' => 'Unauthorized'], 401);
}
$naked_token = str_replace('Bearer ', '', $token);
try {
$decrypted = decrypt($naked_token);
$decrypted_token = json_decode($decrypted, true);
} catch (Exception $e) {
auditLogWebhookFailure('sentinel', 'decrypt_failed');
return response()->json(['message' => 'Invalid token'], 401);
}
$server_uuid = data_get($decrypted_token, 'server_uuid');
if (! $server_uuid) {
auditLogWebhookFailure('sentinel', 'invalid_token_payload');
return response()->json(['message' => 'Invalid token'], 401);
}
$server = Server::where('uuid', $server_uuid)->first();
if (! $server) {
auditLogWebhookFailure('sentinel', 'server_not_found', [
'server_uuid' => $server_uuid,
]);
return response()->json(['message' => 'Server not found'], 404);
}
if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
auditLogWebhookFailure('sentinel', 'subscription_unpaid', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Unauthorized'], 401);
}
if ($server->isFunctional() === false) {
auditLogWebhookFailure('sentinel', 'server_not_functional', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Server is not functional'], 401);
}
if ($server->settings->sentinel_token !== $naked_token) {
auditLogWebhookFailure('sentinel', 'token_mismatch', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Unauthorized'], 401);
}
$data = $request->all();
// Heartbeat MUST update on every push — drives isSentinelLive() and SSH-check skipping.
$server->sentinelHeartbeat();
if ($this->shouldDispatchUpdate($server, $data)) {
PushServerUpdateJob::dispatch($server, $data);
}
auditLog('sentinel.metrics_pushed', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'ok'], 200);
}
/**
* Decide whether PushServerUpdateJob should be dispatched for this push.
*
* Dispatches when: first push (no cached hash), the container state changed,
* or the force window elapsed.
*/
private function shouldDispatchUpdate(Server $server, array $data): bool
{
$hash = $this->containerStateHash($data);
$hashKey = "sentinel:push-hash:{$server->id}";
$forceKey = "sentinel:push-force:{$server->id}";
$cachedHash = Cache::get($hashKey);
$forceActive = Cache::has($forceKey);
$shouldDispatch = $cachedHash === null || $cachedHash !== $hash || ! $forceActive;
if ($shouldDispatch) {
// Day-long TTL bounds memory if a server stops pushing entirely.
Cache::put($hashKey, $hash, now()->addDay());
Cache::put($forceKey, true, config('constants.sentinel.push_force_interval_seconds', 300));
}
return $shouldDispatch;
}
/**
* Build a stable hash of container state.
*
* Covers [name, state, health_status] only metrics and
* filesystem_usage_root are excluded on purpose (disk % churns constantly
* and would defeat the hash; the storage check is separately cache-gated
* inside PushServerUpdateJob). Sorted by name so container ordering from
* Sentinel does not affect the hash.
*/
private function containerStateHash(array $data): string
{
$containers = collect(data_get($data, 'containers', []))
->map(fn ($c) => [
'name' => data_get($c, 'name'),
'state' => data_get($c, 'state'),
'health_status' => data_get($c, 'health_status'),
])
->sortBy('name')
->values()
->all();
return hash('xxh128', json_encode($containers));
}
}
+11 -6
View File
@@ -127,15 +127,20 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
} }
$data = collect($this->data); $data = collect($this->data);
$this->server->sentinelHeartbeat(); // Heartbeat is updated by SentinelController on every push, before dispatch.
$this->containers = collect(data_get($data, 'containers')); $this->containers = collect(data_get($data, 'containers'));
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage'); $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
// Only dispatch storage check when disk percentage actually changes // Only dispatch the storage check when disk usage is at/above the notification
// threshold AND the value changed. Below the threshold ServerStorageCheckJob
// has nothing to do (it only sends a HighDiskUsage notification), so dispatching
// it is wasted work — and most servers sit well below the threshold.
$diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80);
$storageCacheKey = 'storage-check:'.$this->server->id; $storageCacheKey = 'storage-check:'.$this->server->id;
$lastPercentage = Cache::get($storageCacheKey); $lastPercentage = Cache::get($storageCacheKey);
if ($lastPercentage === null || (string) $lastPercentage !== (string) $filesystemUsageRoot) { if ($filesystemUsageRoot !== null
&& $filesystemUsageRoot >= $diskThreshold
&& (string) $lastPercentage !== (string) $filesystemUsageRoot) {
Cache::put($storageCacheKey, $filesystemUsageRoot, 600); Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot); ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
} }
@@ -500,11 +505,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
} catch (\Throwable $e) { } catch (\Throwable $e) {
} }
} else { } else {
// Connect proxy to networks periodically (every 10 min) to avoid excessive job dispatches. // Connect proxy to networks periodically as a safety net to avoid excessive job dispatches.
// On-demand triggers (new network, service deploy) use dispatchSync() and bypass this. // On-demand triggers (new network, service deploy) use dispatchSync() and bypass this.
$proxyCacheKey = 'connect-proxy:'.$this->server->id; $proxyCacheKey = 'connect-proxy:'.$this->server->id;
if (! Cache::has($proxyCacheKey)) { if (! Cache::has($proxyCacheKey)) {
Cache::put($proxyCacheKey, true, 600); Cache::put($proxyCacheKey, true, config('constants.proxy.connect_networks_interval_seconds', 3600));
ConnectProxyToNetworksJob::dispatch($this->server); ConnectProxyToNetworksJob::dispatch($this->server);
} }
} }
+15
View File
@@ -94,6 +94,21 @@ return [
'sentry_dsn' => env('SENTRY_DSN'), 'sentry_dsn' => env('SENTRY_DSN'),
], ],
'sentinel' => [
// How often (seconds) PushServerUpdateJob is force-dispatched even when
// the container state hash is unchanged. Keeps last_online_at,
// exited-detection and storage checks from going stale.
'push_force_interval_seconds' => env('SENTINEL_PUSH_FORCE_INTERVAL_SECONDS', 300),
],
'proxy' => [
// How often (seconds) PushServerUpdateJob periodically re-connects the
// proxy to Docker networks as a safety net. Real network-layout changes
// already connect the proxy on-demand; this only covers gaps (Swarm
// networks added via UI, proxy crash recovery).
'connect_networks_interval_seconds' => env('PROXY_CONNECT_NETWORKS_INTERVAL_SECONDS', 3600),
],
'webhooks' => [ 'webhooks' => [
'feedback_discord_webhook' => env('FEEDBACK_DISCORD_WEBHOOK'), 'feedback_discord_webhook' => env('FEEDBACK_DISCORD_WEBHOOK'),
'dev_webhook' => env('SERVEO_URL'), 'dev_webhook' => env('SERVEO_URL'),
+41 -17
View File
@@ -1,6 +1,46 @@
<?php <?php
use Illuminate\Support\Str; use Illuminate\Support\Str;
use Pdo\Pgsql;
$pgsql = [
'driver' => 'pgsql',
'url' => env('DATABASE_URL'),
'host' => env('DB_HOST', 'coolify-db'),
'port' => env('DB_PORT', '5432'),
'database' => env('DB_DATABASE', 'coolify'),
'username' => env('DB_USERNAME', 'coolify'),
'password' => env('DB_PASSWORD', ''),
'charset' => 'utf8',
'prefix' => '',
'prefix_indexes' => true,
'search_path' => 'public',
'sslmode' => 'prefer',
'options' => [
(defined('Pdo\Pgsql::ATTR_DISABLE_PREPARES') ? Pgsql::ATTR_DISABLE_PREPARES : PDO::PGSQL_ATTR_DISABLE_PREPARES) => env('DB_DISABLE_PREPARES', false),
],
];
/*
* Opt-in read/write replica split. Activates only when DB_READ_HOST is set.
* When unset, the pgsql connection is identical to a single-primary setup.
* Hosts may be comma-separated; Laravel random-picks one per connection.
*/
if (env('DB_READ_HOST')) {
$pgsql['read'] = [
'host' => array_map('trim', explode(',', (string) env('DB_READ_HOST'))),
'port' => env('DB_READ_PORT', env('DB_PORT', '5432')),
'username' => env('DB_READ_USERNAME', env('DB_USERNAME', 'coolify')),
'password' => env('DB_READ_PASSWORD', env('DB_PASSWORD', '')),
];
$pgsql['write'] = [
'host' => array_map('trim', explode(',', (string) env('DB_WRITE_HOST', env('DB_HOST', 'coolify-db')))),
'port' => env('DB_WRITE_PORT', env('DB_PORT', '5432')),
'username' => env('DB_WRITE_USERNAME', env('DB_USERNAME', 'coolify')),
'password' => env('DB_WRITE_PASSWORD', env('DB_PASSWORD', '')),
];
$pgsql['sticky'] = (bool) env('DB_STICKY', true);
}
return [ return [
@@ -35,23 +75,7 @@ return [
'connections' => [ 'connections' => [
'pgsql' => [ 'pgsql' => $pgsql,
'driver' => 'pgsql',
'url' => env('DATABASE_URL'),
'host' => env('DB_HOST', 'coolify-db'),
'port' => env('DB_PORT', '5432'),
'database' => env('DB_DATABASE', 'coolify'),
'username' => env('DB_USERNAME', 'coolify'),
'password' => env('DB_PASSWORD', ''),
'charset' => 'utf8',
'prefix' => '',
'prefix_indexes' => true,
'search_path' => 'public',
'sslmode' => 'prefer',
'options' => [
(defined('Pdo\Pgsql::ATTR_DISABLE_PREPARES') ? \Pdo\Pgsql::ATTR_DISABLE_PREPARES : \PDO::PGSQL_ATTR_DISABLE_PREPARES) => env('DB_DISABLE_PREPARES', false),
],
],
'testing' => [ 'testing' => [
'driver' => 'sqlite', 'driver' => 'sqlite',
+2 -71
View File
@@ -11,12 +11,11 @@ use App\Http\Controllers\Api\ProjectController;
use App\Http\Controllers\Api\ResourcesController; use App\Http\Controllers\Api\ResourcesController;
use App\Http\Controllers\Api\ScheduledTasksController; use App\Http\Controllers\Api\ScheduledTasksController;
use App\Http\Controllers\Api\SecurityController; use App\Http\Controllers\Api\SecurityController;
use App\Http\Controllers\Api\SentinelController;
use App\Http\Controllers\Api\ServersController; use App\Http\Controllers\Api\ServersController;
use App\Http\Controllers\Api\ServicesController; use App\Http\Controllers\Api\ServicesController;
use App\Http\Controllers\Api\TeamController; use App\Http\Controllers\Api\TeamController;
use App\Http\Middleware\ApiAllowed; use App\Http\Middleware\ApiAllowed;
use App\Jobs\PushServerUpdateJob;
use App\Models\Server;
use Illuminate\Support\Facades\Route; use Illuminate\Support\Facades\Route;
Route::get('/health', [OtherController::class, 'healthcheck']); Route::get('/health', [OtherController::class, 'healthcheck']);
@@ -209,75 +208,7 @@ Route::group([
Route::group([ Route::group([
'prefix' => 'v1', 'prefix' => 'v1',
], function () { ], function () {
Route::post('/sentinel/push', function () { Route::post('/sentinel/push', [SentinelController::class, 'push']);
$token = request()->header('Authorization');
if (! $token) {
auditLogWebhookFailure('sentinel', 'token_missing');
return response()->json(['message' => 'Unauthorized'], 401);
}
$naked_token = str_replace('Bearer ', '', $token);
try {
$decrypted = decrypt($naked_token);
$decrypted_token = json_decode($decrypted, true);
} catch (Exception $e) {
auditLogWebhookFailure('sentinel', 'decrypt_failed');
return response()->json(['message' => 'Invalid token'], 401);
}
$server_uuid = data_get($decrypted_token, 'server_uuid');
if (! $server_uuid) {
auditLogWebhookFailure('sentinel', 'invalid_token_payload');
return response()->json(['message' => 'Invalid token'], 401);
}
$server = Server::where('uuid', $server_uuid)->first();
if (! $server) {
auditLogWebhookFailure('sentinel', 'server_not_found', [
'server_uuid' => $server_uuid,
]);
return response()->json(['message' => 'Server not found'], 404);
}
if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
auditLogWebhookFailure('sentinel', 'subscription_unpaid', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Unauthorized'], 401);
}
if ($server->isFunctional() === false) {
auditLogWebhookFailure('sentinel', 'server_not_functional', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Server is not functional'], 401);
}
if ($server->settings->sentinel_token !== $naked_token) {
auditLogWebhookFailure('sentinel', 'token_mismatch', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'Unauthorized'], 401);
}
$data = request()->all();
// \App\Jobs\ServerCheckNewJob::dispatch($server, $data);
PushServerUpdateJob::dispatch($server, $data);
auditLog('sentinel.metrics_pushed', [
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
return response()->json(['message' => 'ok'], 200);
});
}); });
Route::any('/{any}', function () { Route::any('/{any}', function () {
@@ -0,0 +1,74 @@
<?php
/*
* Verifies the opt-in read/write replica split in config/database.php.
* The config file is re-required under different putenv() states so the
* env() calls re-evaluate, then the resulting pgsql array shape is asserted.
*/
function loadDbConfig(): array
{
return require base_path('config/database.php');
}
afterEach(function () {
foreach ([
'DB_READ_HOST', 'DB_READ_PORT', 'DB_READ_USERNAME', 'DB_READ_PASSWORD',
'DB_WRITE_HOST', 'DB_WRITE_PORT', 'DB_WRITE_USERNAME', 'DB_WRITE_PASSWORD',
'DB_STICKY',
] as $key) {
putenv($key);
}
});
it('has no replica keys when DB_READ_HOST is unset', function () {
$pgsql = loadDbConfig()['connections']['pgsql'];
expect($pgsql)
->not->toHaveKey('read')
->not->toHaveKey('write')
->not->toHaveKey('sticky')
->and($pgsql['driver'])->toBe('pgsql');
});
it('enables the read/write split when DB_READ_HOST is set', function () {
putenv('DB_READ_HOST=replica1, replica2');
$pgsql = loadDbConfig()['connections']['pgsql'];
expect($pgsql)
->toHaveKey('read')
->toHaveKey('write')
->and($pgsql['read']['host'])->toBe(['replica1', 'replica2'])
->and($pgsql['sticky'])->toBeTrue();
});
it('falls back to DB_* values for unset replica options', function () {
putenv('DB_READ_HOST=replica1');
$pgsql = loadDbConfig()['connections']['pgsql'];
expect($pgsql['read']['port'])->toBe(env('DB_PORT', '5432'))
->and($pgsql['read']['username'])->toBe(env('DB_USERNAME', 'coolify'))
->and($pgsql['write']['host'])->toBe([env('DB_HOST', 'coolify-db')]);
});
it('respects discrete replica overrides', function () {
putenv('DB_READ_HOST=replica1');
putenv('DB_READ_PORT=6432');
putenv('DB_READ_USERNAME=reader');
$pgsql = loadDbConfig()['connections']['pgsql'];
expect($pgsql['read']['port'])->toBe('6432')
->and($pgsql['read']['username'])->toBe('reader');
});
it('disables sticky reads when DB_STICKY is false', function () {
putenv('DB_READ_HOST=replica1');
putenv('DB_STICKY=false');
$pgsql = loadDbConfig()['connections']['pgsql'];
expect($pgsql['sticky'])->toBeFalse();
});
+99
View File
@@ -0,0 +1,99 @@
<?php
use App\Enums\ApplicationDeploymentStatus;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\Project;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Testing\TestResponse;
uses(RefreshDatabase::class);
beforeEach(function () {
$this->user = User::factory()->create();
$this->team = Team::factory()->create();
$this->team->members()->attach($this->user->id, ['role' => 'owner']);
InstanceSettings::unguarded(function () {
InstanceSettings::query()->create([
'id' => 0,
'is_registration_enabled' => true,
]);
});
$this->actingAs($this->user);
session(['currentTeam' => $this->team]);
$this->server = Server::factory()->create(['team_id' => $this->team->id]);
$this->destination = StandaloneDocker::query()->where('server_id', $this->server->id)->firstOrFail();
$this->project = Project::factory()->create(['team_id' => $this->team->id]);
$this->environment = Environment::factory()->create(['project_id' => $this->project->id]);
$this->application = Application::factory()->create([
'environment_id' => $this->environment->id,
'destination_id' => $this->destination->id,
'destination_type' => $this->destination->getMorphClass(),
'status' => 'running',
]);
});
function showDeployment(string $status): TestResponse
{
$deployment = ApplicationDeploymentQueue::create([
'application_id' => test()->application->id,
'deployment_uuid' => 'deploy-scroll-'.$status,
'server_id' => test()->server->id,
'status' => $status,
'logs' => json_encode([[
'command' => null,
'output' => 'log line for '.$status,
'type' => 'stdout',
'timestamp' => now()->toISOString(),
'hidden' => false,
'batch' => 1,
'order' => 1,
]], JSON_THROW_ON_ERROR),
]);
return test()->get(route('project.application.deployment.show', [
'project_uuid' => test()->project->uuid,
'environment_uuid' => test()->environment->uuid,
'application_uuid' => test()->application->uuid,
'deployment_uuid' => $deployment->deployment_uuid,
]));
}
it('does not enable follow mode for a finished deployment', function () {
$response = showDeployment(ApplicationDeploymentStatus::FINISHED->value);
$response->assertSuccessful();
$response->assertSee('alwaysScroll: false', false);
$response->assertDontSee('alwaysScroll: true', false);
});
it('enables follow mode for an in-progress deployment', function () {
$response = showDeployment(ApplicationDeploymentStatus::IN_PROGRESS->value);
$response->assertSuccessful();
$response->assertSee('alwaysScroll: true', false);
});
it('scopes scroll teardown to the component so a stale loop cannot leak across deployments', function () {
$content = showDeployment(ApplicationDeploymentStatus::FINISHED->value)->getContent();
// Alpine destroy() tears the scroll loop down on wire:navigate away.
expect($content)->toContain('destroy()')
->toContain('cancelScrollLoop()')
// Container lookup is component-scoped, not a global getElementById.
->toContain("this.\$root.querySelector('#logsContainer')")
->not->toContain("document.getElementById('logsContainer')")
// morph.updated hook only acts on this component's own DOM.
->toContain('this.$root.contains(el)')
// Continuation timeout is tracked so it can be cancelled.
->toContain('scrollTimeout');
});
@@ -16,10 +16,29 @@ beforeEach(function () {
Cache::flush(); Cache::flush();
}); });
it('dispatches storage check when disk percentage changes', function () { it('dispatches storage check when disk percentage changes above threshold', function () {
$team = Team::factory()->create(); $team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]); $server = Server::factory()->create(['team_id' => $team->id]);
// Default notification threshold is 80%.
$data = [
'containers' => [],
'filesystem_usage_root' => ['used_percentage' => 85],
];
$job = new PushServerUpdateJob($server, $data);
$job->handle();
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id && $job->percentage === 85;
});
});
it('does not dispatch storage check when disk usage is below threshold', function () {
$team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]);
// 45% is well below the default 80% notification threshold — nothing to do.
$data = [ $data = [
'containers' => [], 'containers' => [],
'filesystem_usage_root' => ['used_percentage' => 45], 'filesystem_usage_root' => ['used_percentage' => 45],
@@ -28,21 +47,19 @@ it('dispatches storage check when disk percentage changes', function () {
$job = new PushServerUpdateJob($server, $data); $job = new PushServerUpdateJob($server, $data);
$job->handle(); $job->handle();
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { Queue::assertNotPushed(ServerStorageCheckJob::class);
return $job->server->id === $server->id && $job->percentage === 45;
});
}); });
it('does not dispatch storage check when disk percentage is unchanged', function () { it('does not dispatch storage check when disk percentage is unchanged', function () {
$team = Team::factory()->create(); $team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]); $server = Server::factory()->create(['team_id' => $team->id]);
// Simulate a previous push that cached the percentage // Simulate a previous push that cached the percentage (above threshold).
Cache::put('storage-check:'.$server->id, 45, 600); Cache::put('storage-check:'.$server->id, 85, 600);
$data = [ $data = [
'containers' => [], 'containers' => [],
'filesystem_usage_root' => ['used_percentage' => 45], 'filesystem_usage_root' => ['used_percentage' => 85],
]; ];
$job = new PushServerUpdateJob($server, $data); $job = new PushServerUpdateJob($server, $data);
@@ -55,19 +72,19 @@ it('dispatches storage check when disk percentage changes from cached value', fu
$team = Team::factory()->create(); $team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]); $server = Server::factory()->create(['team_id' => $team->id]);
// Simulate a previous push that cached 45% // Simulate a previous push that cached 85% (above threshold).
Cache::put('storage-check:'.$server->id, 45, 600); Cache::put('storage-check:'.$server->id, 85, 600);
$data = [ $data = [
'containers' => [], 'containers' => [],
'filesystem_usage_root' => ['used_percentage' => 50], 'filesystem_usage_root' => ['used_percentage' => 90],
]; ];
$job = new PushServerUpdateJob($server, $data); $job = new PushServerUpdateJob($server, $data);
$job->handle(); $job->handle();
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) { Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id && $job->percentage === 50; return $job->server->id === $server->id && $job->percentage === 90;
}); });
}); });
@@ -140,6 +157,36 @@ it('dispatches ConnectProxyToNetworksJob again after cache expires', function ()
Queue::assertPushed(ConnectProxyToNetworksJob::class, 1); Queue::assertPushed(ConnectProxyToNetworksJob::class, 1);
}); });
it('respects the configured proxy connect interval', function () {
// Interval 0 → the connect-proxy gate key expires immediately, so every
// push re-dispatches without a manual Cache::forget. Proves the TTL is
// driven by config('constants.proxy.connect_networks_interval_seconds').
config(['constants.proxy.connect_networks_interval_seconds' => 0]);
$team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]);
$server->settings->update(['is_reachable' => true, 'is_usable' => true]);
$data = [
'containers' => [
[
'name' => 'coolify-proxy',
'state' => 'running',
'health_status' => 'healthy',
'labels' => ['coolify.managed' => true],
],
],
'filesystem_usage_root' => ['used_percentage' => 10],
];
(new PushServerUpdateJob($server, $data))->handle();
Queue::assertPushed(ConnectProxyToNetworksJob::class, 1);
Queue::fake();
(new PushServerUpdateJob($server, $data))->handle();
Queue::assertPushed(ConnectProxyToNetworksJob::class, 1);
});
it('uses default queue for PushServerUpdateJob', function () { it('uses default queue for PushServerUpdateJob', function () {
$team = Team::factory()->create(); $team = Team::factory()->create();
$server = Server::factory()->create(['team_id' => $team->id]); $server = Server::factory()->create(['team_id' => $team->id]);
@@ -0,0 +1,119 @@
<?php
use App\Jobs\PushServerUpdateJob;
use App\Models\Server;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
beforeEach(function () {
Queue::fake();
Cache::flush();
$user = User::factory()->create();
$this->team = $user->teams()->first();
$this->server = Server::factory()->create([
'team_id' => $this->team->id,
]);
$this->server->settings->update([
'is_reachable' => true,
'is_usable' => true,
]);
$this->token = $this->server->settings->sentinel_token;
});
function pushSentinel(string $token, array $payload)
{
return test()->postJson('/api/v1/sentinel/push', $payload, [
'Authorization' => 'Bearer '.$token,
]);
}
function sentinelPayload(array $containers, ?float $diskPercentage = 42.0): array
{
return [
'containers' => $containers,
'filesystem_usage_root' => ['used_percentage' => $diskPercentage],
];
}
$running = fn () => [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']];
it('dispatches the job on the first push', function () use ($running) {
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 1);
});
it('skips the job when the second push is identical', function () use ($running) {
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 1);
});
it('updates the heartbeat even when the job is skipped', function () use ($running) {
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
$this->server->update(['sentinel_updated_at' => now()->subHour()]);
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 1);
expect(Carbon::parse($this->server->fresh()->sentinel_updated_at)->diffInSeconds(now()))->toBeLessThan(5);
});
it('dispatches the job when container state changes', function () use ($running) {
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
$exited = [['name' => 'app-1', 'state' => 'exited', 'health_status' => 'unhealthy']];
pushSentinel($this->token, sentinelPayload($exited))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
it('ignores disk percentage changes (excluded from the hash)', function () use ($running) {
pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk();
pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 1);
});
it('ignores container reordering (hash is sorted by name)', function () {
$order1 = [
['name' => 'app-a', 'state' => 'running', 'health_status' => 'healthy'],
['name' => 'app-b', 'state' => 'running', 'health_status' => 'healthy'],
];
$order2 = [
['name' => 'app-b', 'state' => 'running', 'health_status' => 'healthy'],
['name' => 'app-a', 'state' => 'running', 'health_status' => 'healthy'],
];
pushSentinel($this->token, sentinelPayload($order1))->assertOk();
pushSentinel($this->token, sentinelPayload($order2))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 1);
});
it('force-dispatches an identical push after the force window expires', function () use ($running) {
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
// Simulate the force key TTL elapsing.
Cache::forget('sentinel:push-force:'.$this->server->id);
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
Queue::assertPushed(PushServerUpdateJob::class, 2);
});
it('rejects an invalid token without dispatching', function () use ($running) {
pushSentinel('not-a-real-token', sentinelPayload($running()))->assertUnauthorized();
Queue::assertNotPushed(PushServerUpdateJob::class);
});