Skill: Uptime Monitoring

# Skill: Uptime Monitoring

## Overview

Pattern for scheduled HTTP health checks using `node-cron` and `axios`, including concurrent execution control, status normalization, DB write patterns, and cron job lifecycle management tied to service CRUD operations.

## Health Check Implementation

```ts
// api/src/lib/checker.ts
import axios, { type AxiosError } from 'axios';
import type { Database } from 'better-sqlite3';
import type { Service } from '../types';

export interface CheckResult {
  status: 'up' | 'down' | 'timeout' | 'error';
  statusCode: number | null;
  responseMs: number | null;
  errorMessage: string | null;
}

export async function checkService(service: Service): Promise<CheckResult> {
  const start = Date.now();
  try {
    const res = await axios.get(service.url, {
      timeout: service.timeout_seconds * 1000,
      validateStatus: () => true,   // never throw on non-2xx
      maxRedirects: 3,
    });
    const responseMs = Date.now() - start;
    const status = res.status === service.expected_status ? 'up' : 'down';
    return { status, statusCode: res.status, responseMs, errorMessage: null };
  } catch (err: unknown) {
    const responseMs = Date.now() - start;
    const e = err as AxiosError;
    if (e.code === 'ECONNABORTED' || e.message?.includes('timeout')) {
      return { status: 'timeout', statusCode: null, responseMs: null, errorMessage: 'Request timed out' };
    }
    return { status: 'error', statusCode: null, responseMs: null, errorMessage: e.message ?? 'Unknown error' };
  }
}

export function writeCheck(db: Database, serviceId: number, result: CheckResult): void {
  db.prepare(`
    INSERT INTO checks (service_id, status, status_code, response_ms, error_message)
    VALUES (?, ?, ?, ?, ?)
  `).run(serviceId, result.status, result.statusCode, result.responseMs, result.errorMessage);
}
```

## Scheduler with Concurrency Control

```ts
// api/src/lib/scheduler.ts
import cron, { type ScheduledTask } from 'node-cron';
import type { Database } from 'better-sqlite3';
import { checkService, writeCheck } from './checker';
import { maybeOpenAutoIncident } from './incidents';
import type { Service } from '../types';

// Map from service ID to cron task
const tasks = new Map<number, ScheduledTask>();

// Semaphore: limit concurrent checks
let running = 0;
const MAX_CONCURRENT = Number(process.env.CHECK_CONCURRENCY ?? 5);

async function runCheck(db: Database, service: Service): Promise<void> {
  if (running >= MAX_CONCURRENT) return;  // skip if at capacity
  running++;
  try {
    const result = await checkService(service);
    writeCheck(db, service.id, result);
    if (result.status !== 'up' && service.auto_incident) {
      maybeOpenAutoIncident(db, service.id);
    }
  } finally {
    running--;
  }
}

function cronExpression(intervalMinutes: number): string {
  if (intervalMinutes === 1) return '* * * * *';
  return `*/${intervalMinutes} * * * *`;
}

export function scheduleService(db: Database, service: Service): void {
  // Stop existing task if any
  unscheduleService(service.id);

  if (!service.is_active) return;

  const task = cron.schedule(cronExpression(service.check_interval), () => {
    runCheck(db, service);
  });

  tasks.set(service.id, task);
}

export function unscheduleService(serviceId: number): void {
  const task = tasks.get(serviceId);
  if (task) {
    task.stop();
    tasks.delete(serviceId);
  }
}

export function initScheduler(db: Database): void {
  const services = db.prepare('SELECT * FROM services WHERE is_active = 1').all() as Service[];
  for (const s of services) {
    scheduleService(db, s);
  }
}
```

Call `initScheduler(db)` once at server startup after DB migrations.

## Scheduler Integration with Service Routes

```ts
// api/src/routes/services.ts (excerpt)
import { scheduleService, unscheduleService } from '../lib/scheduler';

// After creating a service:
router.post('/', requireAuth, (req, res) => {
  // ... insert service ...
  const service = db.prepare('SELECT * FROM services WHERE id = ?').get(newId) as Service;
  scheduleService(db, service);
  res.status(201).json(service);
});

// After updating a service:
router.put('/:id', requireAuth, (req, res) => {
  // ... update service ...
  const service = db.prepare('SELECT * FROM services WHERE id = ?').get(req.params.id) as Service;
  scheduleService(db, service);  // reschedule with new interval
  res.json(service);
});

// After deleting a service:
router.delete('/:id', requireAuth, (req, res) => {
  unscheduleService(Number(req.params.id));
  db.prepare('DELETE FROM services WHERE id = ?').run(req.params.id);
  res.json({ ok: true });
});
```

## Manual Check Endpoint

```ts
// POST /api/services/:id/check
router.post('/:id/check', requireAuth, async (req, res) => {
  const service = db.prepare('SELECT * FROM services WHERE id = ?').get(req.params.id) as Service | undefined;
  if (!service) { res.status(404).json({ error: 'Not found' }); return; }

  const result = await checkService(service);
  writeCheck(db, service.id, result);

  res.json({ status: result.status, responseMs: result.responseMs, statusCode: result.statusCode });
});
```

## Cron Expression Reference

| Interval | Expression | Notes |
|---|---|---|
| 1 minute | `* * * * *` | Every minute |
| 2 minutes | `*/2 * * * *` | Every 2 minutes |
| 5 minutes | `*/5 * * * *` | Every 5 minutes |
| 10 minutes | `*/10 * * * *` | Every 10 minutes |
| 30 minutes | `*/30 * * * *` | Every 30 minutes |
| 1 hour | `0 * * * *` | Top of every hour |

## Check Status Normalization

| Scenario | Status |
|---|---|
| Response received, correct status code | `up` |
| Response received, wrong status code | `down` |
| `axios` timeout (ECONNABORTED) | `timeout` |
| Network error (ECONNREFUSED, ENOTFOUND) | `error` |
| DNS resolution failure | `error` |

## Uptime Color Scale

| Uptime | Color | CSS var |
|---|---|---|
| >= 99% | Green | `--status-up` (#16a34a) |
| 95-99% | Amber | `--status-degraded` (#d97706) |
| < 95% | Red | `--status-down` (#dc2626) |

## Testing the Checker

```ts
// api/src/lib/__tests__/checker.test.ts
import { describe, test, expect, vi } from 'vitest';
import axios from 'axios';

vi.mock('axios');

test('returns up when status code matches expected', async () => {
  vi.mocked(axios.get).mockResolvedValueOnce({ status: 200 });
  const result = await checkService({ url: 'https://example.com', expected_status: 200, timeout_seconds: 10, ...rest });
  expect(result.status).toBe('up');
  expect(result.statusCode).toBe(200);
});

test('returns down when status code does not match', async () => {
  vi.mocked(axios.get).mockResolvedValueOnce({ status: 503 });
  const result = await checkService({ url: 'https://example.com', expected_status: 200, timeout_seconds: 10, ...rest });
  expect(result.status).toBe('down');
});

test('returns timeout on ECONNABORTED', async () => {
  const err = new Error('timeout') as Error & { code: string };
  err.code = 'ECONNABORTED';
  vi.mocked(axios.get).mockRejectedValueOnce(err);
  const result = await checkService({ url: 'https://example.com', expected_status: 200, timeout_seconds: 10, ...rest });
  expect(result.status).toBe('timeout');
});
```