Monitoring Applications with Grafana

version: '3.8'

services:
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=mysecretpassword
      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-piechart-panel
    volumes:
      - grafana-data:/var/lib/grafana
    depends_on:
      - prometheus

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=15d'
      - '--web.enable-lifecycle'

  node_exporter:
    image: prom/node-exporter:latest
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'

volumes:
  grafana-data:
  prometheus-data:

global:
  scrape_interval: 15s        # How often to scrape targets
  evaluation_interval: 15s    # How often to evaluate alert rules
  scrape_timeout: 10s

scrape_configs:
  # Prometheus monitors itself
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Host system metrics via Node Exporter
  - job_name: 'node_exporter'
    static_configs:
      - targets: ['node_exporter:9100']

  # Your application (must expose /metrics)
  - job_name: 'myapp'
    static_configs:
      - targets: ['host.docker.internal:8080']
    metrics_path: '/metrics'
    scrape_interval: 10s

const client = require('prom-client');
const express = require('express');

// Collect default Node.js metrics (heap, GC, event loop lag)
client.collectDefaultMetrics({ prefix: 'myapp_' });

// Custom counter: track every HTTP request
const httpRequestsTotal = new client.Counter({
  name: 'myapp_http_requests_total',
  help: 'Total HTTP requests',
  labelNames: ['method', 'status', 'route']
});

// Histogram: track request duration with buckets
const httpRequestDuration = new client.Histogram({
  name: 'myapp_http_request_duration_seconds',
  help: 'HTTP request latency in seconds',
  labelNames: ['method', 'route', 'status'],
  buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5]
});

// Gauge: current active connections
const activeConnections = new client.Gauge({
  name: 'myapp_active_connections',
  help: 'Number of active connections'
});

const app = express();

// Middleware: record metrics for every request
app.use((req, res, next) => {
  const end = httpRequestDuration.startTimer();
  activeConnections.inc();
  res.on('finish', () => {
    const route = req.route?.path || req.path;
    httpRequestsTotal.inc({ method: req.method, status: res.statusCode, route });
    end({ method: req.method, route, status: res.statusCode });
    activeConnections.dec();
  });
  next();
});

// Prometheus scrapes this endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', client.register.contentType);
  res.end(await client.register.metrics());
});

app.listen(8080);

<?php

namespace App\Http\Controllers;

use Prometheus\CollectorRegistry;
use Prometheus\RenderTextFormat;

class MetricsController extends Controller
{
    public function __invoke(CollectorRegistry $registry)
    {
        $renderer = new RenderTextFormat();
        $result = $renderer->render($registry->getMetricFamilySamples());

        return response($result, 200)
            ->header('Content-Type', RenderTextFormat::MIME_TYPE);
    }
}

// In a middleware or service provider, increment counters:
$counter = $registry->getOrRegisterCounter(
    'myapp',
    'http_requests_total',
    'Total HTTP requests',
    ['method', 'status']
);
$counter->incBy(1, [$request->method(), $response->status()]);

# Variable name: instance
# Data source: Prometheus
# Query: label_values(node_cpu_seconds_total, instance)
# This populates a dropdown with all scraped instances

# Then use it in panels:
100 - (avg by (instance) (
  rate(node_cpu_seconds_total{mode="idle", instance="$instance"}[5m])
) * 100)

# Multi-value variable (select multiple instances):
rate(myapp_http_requests_total{instance=~"$instance"}[5m])

# Navigate to: Alerting → Alert rules → New alert rule

# Step 1: Define query (A)
# Expression:
100 - (avg by (instance) (
  rate(node_cpu_seconds_total{mode="idle"}[5m])
) * 100)

# Step 2: Set condition (B)
# Type: Classic condition
# When: avg() of A IS ABOVE 85

# Step 3: Set evaluation
# Evaluate every: 1m
# For: 5m   ← must stay above threshold for 5 min to fire

# Step 4: Set labels for routing
severity: warning
team: infrastructure

# Alert message template:
# CPU on {{ $labels.instance }} is {{ printf "%.1f" $values.A.Value }}%
# — above the 85% threshold for 5 minutes

# Alerting → Contact points → Add contact point

# Type: Slack
# Webhook URL: https://hooks.slack.com/services/YOUR/WEBHOOK/URL
# Channel: #alerts-production

# Message template (optional, override default):
{{ define "slack.message" -}}
{{ if eq .Status "firing" }}🔴{{ else }}✅{{ end }}
*{{ .CommonLabels.alertname }}* — {{ .Status | toUpper }}
{{ range .Alerts }}
• *Instance:* {{ .Labels.instance }}
• *Severity:* {{ .Labels.severity }}
• *Value:* {{ .Annotations.value }}
{{ end }}
{{- end }}

groups:
  - name: myapp_recording_rules
    interval: 30s
    rules:
      # Pre-compute HTTP request rate to speed up dashboards
      - record: job:myapp_http_requests_total:rate5m
        expr: sum by (job) (rate(myapp_http_requests_total[5m]))

      # Pre-compute error rate percentage
      - record: job:myapp_http_error_rate:rate5m
        expr: |
          sum by (job) (rate(myapp_http_requests_total{status=~"5.."}[5m]))
          /
          sum by (job) (rate(myapp_http_requests_total[5m])) * 100

      # Pre-compute p95 latency
      - record: job:myapp_request_duration_p95:rate5m
        expr: |
          histogram_quantile(0.95,
            sum by (le, job) (
              rate(myapp_http_request_duration_seconds_bucket[5m])
            )
          )