Node.js App
  └── prom-client (exposes /metrics endpoint)
        │
        ▼
Metrics Scraper (pulls every 15 seconds)
  ├── Prometheus (stores locally, 15-day default retention)
  ├── VictoriaMetrics (stores locally, excellent compression, long-term)
  └── Grafana Agent → Grafana Mimir (stores in S3, unlimited, multi-tenant)
        │
        ▼
Grafana (visualization — queries PromQL from any backend)

npm install prom-client

import { Registry, Counter, Histogram, Gauge, collectDefaultMetrics } from "prom-client";
import express from "express";

// Create a registry
const register = new Registry();

// Collect default Node.js metrics (event loop, GC, memory, etc.)
collectDefaultMetrics({ register });

// Custom metrics
const httpRequestDuration = new Histogram({
  name: "http_request_duration_seconds",
  help: "Duration of HTTP requests in seconds",
  labelNames: ["method", "route", "status_code"],
  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
  registers: [register],
});

const httpRequestTotal = new Counter({
  name: "http_requests_total",
  help: "Total number of HTTP requests",
  labelNames: ["method", "route", "status_code"],
  registers: [register],
});

const activeConnections = new Gauge({
  name: "active_connections",
  help: "Number of active WebSocket connections",
  registers: [register],
});

// Middleware to instrument all routes
const app = express();

app.use((req, res, next) => {
  const end = httpRequestDuration.startTimer({
    method: req.method,
    route: req.route?.path ?? req.path,
  });

  res.on("finish", () => {
    const labels = {
      method: req.method,
      route: req.route?.path ?? req.path,
      status_code: res.statusCode,
    };
    end(labels);
    httpRequestTotal.inc(labels);
  });

  next();
});

// Metrics endpoint — scraped by Prometheus/VM/Mimir agent
app.get("/metrics", async (req, res) => {
  res.set("Content-Type", register.contentType);
  res.end(await register.metrics());
});

// Example route with custom business metrics
const ordersCreated = new Counter({
  name: "orders_created_total",
  help: "Total orders created",
  labelNames: ["status", "payment_method"],
  registers: [register],
});

app.post("/orders", async (req, res) => {
  // ... create order
  ordersCreated.inc({ status: "success", payment_method: "stripe" });
  res.json({ orderId: "..." });
});

# docker-compose.yml — Prometheus + Grafana
version: "3.8"

services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.retention.time=30d"  # 30 days retention
      - "--storage.tsdb.path=/prometheus"
      - "--web.enable-remote-write-receiver"  # Accept remote writes

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      GF_SECURITY_ADMIN_PASSWORD: "admin"
      GF_USERS_ALLOW_SIGN_UP: "false"

  node-exporter:
    image: prom/node-exporter:latest
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro

volumes:
  prometheus_data:
  grafana_data:

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    environment: production

# Alerting
alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

# Load alerting rules
rule_files:
  - "alerts/*.yml"

# Scrape configs
scrape_configs:
  # Scrape Prometheus itself
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]

  # Scrape Node.js app
  - job_name: "my-app"
    static_configs:
      - targets: ["app:3000"]
    metrics_path: /metrics
    scheme: http

  # Scrape Node Exporter (system metrics)
  - job_name: "node-exporter"
    static_configs:
      - targets: ["node-exporter:9100"]

  # Kubernetes pod discovery (if using K8s)
  - job_name: "kubernetes-pods"
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: "true"

# alerts/nodejs.yml
groups:
  - name: nodejs
    rules:
      - alert: HighErrorRate
        expr: |
          rate(http_requests_total{status_code=~"5.."}[5m])
          /
          rate(http_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate: {{ $value | humanizePercentage }}"

      - alert: SlowRequests
        expr: |
          histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])) > 2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency above 2s: {{ $value }}s"

# docker-compose.yml — VictoriaMetrics replaces Prometheus storage
version: "3.8"

services:
  victoriametrics:
    image: victoriametrics/victoria-metrics:latest
    ports:
      - "8428:8428"
    volumes:
      - vm_data:/victoria-metrics-data
    command:
      - "--storageDataPath=/victoria-metrics-data"
      - "--retentionPeriod=365"  # 1 year retention (Prometheus: 15 days default)
      - "--httpListenAddr=:8428"

  vmagent:
    image: victoriametrics/vmagent:latest
    ports:
      - "8429:8429"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml  # Same config!
      - vmagent_data:/tmp/vmagent-remotewrite-data
    command:
      - "--promscrape.config=/etc/prometheus/prometheus.yml"
      - "--remoteWrite.url=http://victoriametrics:8428/api/v1/write"

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana

volumes:
  vm_data:
  vmagent_data:
  grafana_data:

# These PromQL queries work identically on Prometheus and VictoriaMetrics

# P99 request latency
histogram_quantile(
  0.99,
  rate(http_request_duration_seconds_bucket[5m])
)

# Request rate by route
sum(rate(http_requests_total[5m])) by (route)

# Error rate
sum(rate(http_requests_total{status_code=~"5.."}[5m]))
/
sum(rate(http_requests_total[5m]))

# Active connections
active_connections

# Memory usage
process_resident_memory_bytes / 1024 / 1024  # MB

# VictoriaMetrics adds MetricsQL extensions to PromQL

# Keep last value for sparse metrics
keep_last_value(some_gauge)

# Share of each time series in the total
sum(rate(http_requests_total[5m])) by (route)
/
ignoring(route) group_left sum(rate(http_requests_total[5m]))

# Running average
running_avg(rate(http_requests_total[5m]))

# docker-compose.yml — Mimir in single-binary mode (production uses microservices)
version: "3.8"

services:
  mimir:
    image: grafana/mimir:latest
    command:
      - "--config.file=/etc/mimir/mimir.yaml"
    ports:
      - "9009:9009"
    volumes:
      - ./mimir.yaml:/etc/mimir/mimir.yaml
      - mimir_data:/data

  grafana-agent:
    image: grafana/agent:latest
    ports:
      - "12345:12345"
    volumes:
      - ./agent.yaml:/etc/agent/config.yaml
    command:
      - "--config.file=/etc/agent/config.yaml"

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana

volumes:
  mimir_data:
  grafana_data:

# mimir.yaml
target: all  # Single binary mode

common:
  storage:
    backend: filesystem  # For dev; use s3 in production
    filesystem:
      dir: /data

blocks_storage:
  filesystem:
    dir: /data/blocks

alertmanager_storage:
  filesystem:
    dir: /data/alertmanager

ruler_storage:
  filesystem:
    dir: /data/rules

memberlist:
  join_members:
    - "mimir:7946"

limits:
  max_global_series_per_user: 1500000

# agent.yaml — Grafana Agent to scrape and send to Mimir
metrics:
  global:
    scrape_interval: 15s
    remote_write:
      - url: http://mimir:9009/api/v1/push
        headers:
          X-Scope-OrgID: demo-tenant  # Required for multi-tenancy

  configs:
    - name: default
      scrape_configs:
        - job_name: my-app
          static_configs:
            - targets: ["app:3000"]