Observability: Logs, Metrics and Traces

package main

import (
	"fmt"
	"sort"
	"strings"
)

// 1. Latency: 1,000 requests. Most take 50-100 ms; 2% hit a slow path and take 2 s.
func latency() {
	lat := make([]int, 1000)
	sum := 0
	for i := range lat {
		lat[i] = 50 + i*37%51
		if i%50 == 7 {
			lat[i] = 2000
		}
		sum += lat[i]
	}
	sort.Ints(lat)
	fmt.Println("1,000 requests, 2% of them hit a slow path:")
	fmt.Printf("  average %d ms, p50 %d ms, p95 %d ms, p99 %d ms, max %d ms\n", sum/1000, lat[499], lat[949], lat[989], lat[999])
}

// 2. A trace: one request as a tree of timed spans.
type span struct {
	name     string
	ms       int
	children []span
}

func printSpan(s span, depth int, slowest *span) {
	fmt.Printf("  %s%-*s %4d ms\n", strings.Repeat("  ", depth), 22-2*depth, s.name, s.ms)
	if len(s.children) == 0 && s.ms > slowest.ms {
		*slowest = s
	}
	for _, c := range s.children {
		printSpan(c, depth+1, slowest)
	}
}

func trace() {
	root := span{"GET /checkout", 420, []span{
		{"auth", 12, nil},
		{"load cart (db)", 40, nil},
		{"payment", 330, []span{
			{"fraud check", 280, nil},
			{"charge card", 45, nil},
		}},
		{"render", 30, nil},
	}}
	fmt.Println("trace of one slow checkout:")
	slowest := span{}
	printSpan(root, 0, &slowest)
	fmt.Printf("  most of the time: %s, %d ms of %d (%d%%)\n", slowest.name, slowest.ms, root.ms, slowest.ms*100/root.ms)
}

// 3. Alerts. SLO: 99.9% of requests succeed over 30 days, so the error budget is 0.1%.
// Error rates are in hundredths of a percent (150 = 1.50%).
func alerts() {
	errs := []int{5, 8, 150, 160, 20, 6}
	cpu := []int{85, 88, 40, 35, 60, 82}
	const budget = 10 // 0.10% in hundredths of a percent
	fmt.Println("six hours of errors and CPU; SLO 99.9% (budget 0.1%):")
	used := 0
	for h := range errs {
		burn := errs[h] * 10 / budget // tenths: 150 -> 150 (15.0x the allowed rate)
		page := "-"
		if burn >= 144 { // 14.4x for an hour = 2% of the month's budget gone
			page = "PAGE"
		}
		cpuAlert := "-"
		if cpu[h] > 80 {
			cpuAlert = "alert"
		}
		fmt.Printf("  hour %d: errors %d.%02d%%, burn %2d.%dx -> %-4s | CPU %d%% -> %s\n",
			h+1, errs[h]/100, errs[h]%100, burn/10, burn%10, page, cpu[h], cpuAlert)
		used += errs[h]
	}
	month := budget * 30 * 24 // the whole month's budget, in the same units
	fmt.Printf("  error budget used in these 6 hours: %d.%d%% of the month's\n", used*1000/month/10, used*1000/month%10)
}

func main() {
	latency()
	trace()
	alerts()
}

from dataclasses import dataclass, field


def latency() -> None:
    """1. Latency: 1,000 requests. Most take 50-100 ms; 2% hit a slow path and take 2 s."""
    lat = [2000 if i % 50 == 7 else 50 + i * 37 % 51 for i in range(1000)]
    total = sum(lat)
    lat.sort()
    print("1,000 requests, 2% of them hit a slow path:")
    print(f"  average {total // 1000} ms, p50 {lat[499]} ms, p95 {lat[949]} ms, p99 {lat[989]} ms, max {lat[999]} ms")


@dataclass
class Span:
    """2. A trace: one request as a tree of timed spans."""
    name: str
    ms: int
    children: list["Span"] = field(default_factory=list)


def print_span(s: Span, depth: int, slowest: list[Span]) -> None:
    print(f"  {'  ' * depth}{s.name:<{22 - 2 * depth}} {s.ms:4} ms")
    if not s.children and s.ms > slowest[0].ms:
        slowest[0] = s
    for c in s.children:
        print_span(c, depth + 1, slowest)


def trace() -> None:
    root = Span("GET /checkout", 420, [
        Span("auth", 12),
        Span("load cart (db)", 40),
        Span("payment", 330, [Span("fraud check", 280), Span("charge card", 45)]),
        Span("render", 30),
    ])
    print("trace of one slow checkout:")
    slowest = [Span("", 0)]
    print_span(root, 0, slowest)
    s = slowest[0]
    print(f"  most of the time: {s.name}, {s.ms} ms of {root.ms} ({s.ms * 100 // root.ms}%)")


def alerts() -> None:
    """3. Alerts. SLO: 99.9% of requests succeed over 30 days, so the error budget is 0.1%.
    Error rates are in hundredths of a percent (150 = 1.50%)."""
    errs = [5, 8, 150, 160, 20, 6]
    cpu = [85, 88, 40, 35, 60, 82]
    budget = 10  # 0.10% in hundredths of a percent
    print("six hours of errors and CPU; SLO 99.9% (budget 0.1%):")
    used = 0
    for h, (e, c) in enumerate(zip(errs, cpu)):
        burn = e * 10 // budget  # tenths: 150 -> 150 (15.0x the allowed rate)
        page = "PAGE" if burn >= 144 else "-"  # 14.4x for an hour = 2% of the month's budget gone
        cpu_alert = "alert" if c > 80 else "-"
        print(f"  hour {h + 1}: errors {e // 100}.{e % 100:02}%, burn {burn // 10:2}.{burn % 10}x -> {page:<4} | CPU {c}% -> {cpu_alert}")
        used += e
    month = budget * 30 * 24  # the whole month's budget, in the same units
    print(f"  error budget used in these 6 hours: {used * 1000 // month // 10}.{used * 1000 // month % 10}% of the month's")


latency()
trace()
alerts()

const div = (a: number, b: number) => Math.floor(a / b);

/** 1. Latency: 1,000 requests. Most take 50-100 ms; 2% hit a slow path and take 2 s. */
function latency() {
  const lat = Array.from({ length: 1000 }, (_, i) => (i % 50 === 7 ? 2000 : 50 + ((i * 37) % 51)));
  const sum = lat.reduce((a, b) => a + b, 0);
  lat.sort((a, b) => a - b);
  console.log("1,000 requests, 2% of them hit a slow path:");
  console.log(`  average ${div(sum, 1000)} ms, p50 ${lat[499]} ms, p95 ${lat[949]} ms, p99 ${lat[989]} ms, max ${lat[999]} ms`);
}

/** 2. A trace: one request as a tree of timed spans. */
type Span = { name: string; ms: number; children: Span[] };
const span = (name: string, ms: number, children: Span[] = []): Span => ({ name, ms, children });

function printSpan(s: Span, depth: number, slowest: { s: Span }) {
  console.log(`  ${"  ".repeat(depth)}${s.name.padEnd(22 - 2 * depth)} ${String(s.ms).padStart(4)} ms`);
  if (s.children.length === 0 && s.ms > slowest.s.ms) slowest.s = s;
  for (const c of s.children) printSpan(c, depth + 1, slowest);
}

function trace() {
  const root = span("GET /checkout", 420, [
    span("auth", 12),
    span("load cart (db)", 40),
    span("payment", 330, [span("fraud check", 280), span("charge card", 45)]),
    span("render", 30),
  ]);
  console.log("trace of one slow checkout:");
  const slowest = { s: span("", 0) };
  printSpan(root, 0, slowest);
  const s = slowest.s;
  console.log(`  most of the time: ${s.name}, ${s.ms} ms of ${root.ms} (${div(s.ms * 100, root.ms)}%)`);
}

/** 3. Alerts. SLO: 99.9% of requests succeed over 30 days, so the error budget is 0.1%.
 * Error rates are in hundredths of a percent (150 = 1.50%). */
function alerts() {
  const errs = [5, 8, 150, 160, 20, 6];
  const cpu = [85, 88, 40, 35, 60, 82];
  const budget = 10; // 0.10% in hundredths of a percent
  console.log("six hours of errors and CPU; SLO 99.9% (budget 0.1%):");
  let used = 0;
  errs.forEach((e, h) => {
    const burn = div(e * 10, budget); // tenths: 150 -> 150 (15.0x the allowed rate)
    const page = burn >= 144 ? "PAGE" : "-"; // 14.4x for an hour = 2% of the month's budget gone
    const cpuAlert = cpu[h]! > 80 ? "alert" : "-";
    console.log(
      `  hour ${h + 1}: errors ${div(e, 100)}.${String(e % 100).padStart(2, "0")}%, burn ${String(div(burn, 10)).padStart(2)}.${burn % 10}x -> ${page.padEnd(4)} | CPU ${cpu[h]}% -> ${cpuAlert}`,
    );
    used += e;
  });
  const month = budget * 30 * 24; // the whole month's budget, in the same units
  console.log(`  error budget used in these 6 hours: ${div(div(used * 1000, month), 10)}.${div(used * 1000, month) % 10}% of the month's`);
}

latency();
trace();
alerts();

/// 1. Latency: 1,000 requests. Most take 50-100 ms; 2% hit a slow path and take 2 s.
fn latency() {
    let mut lat: Vec<i32> = (0..1000).map(|i| if i % 50 == 7 { 2000 } else { 50 + i * 37 % 51 }).collect();
    let sum: i32 = lat.iter().sum();
    lat.sort();
    println!("1,000 requests, 2% of them hit a slow path:");
    println!("  average {} ms, p50 {} ms, p95 {} ms, p99 {} ms, max {} ms", sum / 1000, lat[499], lat[949], lat[989], lat[999]);
}

/// 2. A trace: one request as a tree of timed spans.
#[derive(Clone)]
struct Span {
    name: &'static str,
    ms: i32,
    children: Vec<Span>,
}

fn span(name: &'static str, ms: i32, children: Vec<Span>) -> Span {
    Span { name, ms, children }
}

fn print_span(s: &Span, depth: usize, slowest: &mut Span) {
    println!("  {}{:<w$} {:4} ms", "  ".repeat(depth), s.name, s.ms, w = 22 - 2 * depth);
    if s.children.is_empty() && s.ms > slowest.ms {
        *slowest = s.clone();
    }
    for c in &s.children {
        print_span(c, depth + 1, slowest);
    }
}

fn trace() {
    let root = span("GET /checkout", 420, vec![
        span("auth", 12, vec![]),
        span("load cart (db)", 40, vec![]),
        span("payment", 330, vec![span("fraud check", 280, vec![]), span("charge card", 45, vec![])]),
        span("render", 30, vec![]),
    ]);
    println!("trace of one slow checkout:");
    let mut slowest = span("", 0, vec![]);
    print_span(&root, 0, &mut slowest);
    println!("  most of the time: {}, {} ms of {} ({}%)", slowest.name, slowest.ms, root.ms, slowest.ms * 100 / root.ms);
}

/// 3. Alerts. SLO: 99.9% of requests succeed over 30 days, so the error budget is 0.1%.
/// Error rates are in hundredths of a percent (150 = 1.50%).
fn alerts() {
    let errs = [5, 8, 150, 160, 20, 6];
    let cpu = [85, 88, 40, 35, 60, 82];
    let budget = 10; // 0.10% in hundredths of a percent
    println!("six hours of errors and CPU; SLO 99.9% (budget 0.1%):");
    let mut used = 0;
    for (h, (&e, &c)) in errs.iter().zip(cpu.iter()).enumerate() {
        let burn = e * 10 / budget; // tenths: 150 -> 150 (15.0x the allowed rate)
        let page = if burn >= 144 { "PAGE" } else { "-" }; // 14.4x for an hour = 2% of the month's budget gone
        let cpu_alert = if c > 80 { "alert" } else { "-" };
        println!("  hour {}: errors {}.{:02}%, burn {:2}.{}x -> {page:<4} | CPU {c}% -> {cpu_alert}", h + 1, e / 100, e % 100, burn / 10, burn % 10);
        used += e;
    }
    let month = budget * 30 * 24; // the whole month's budget, in the same units
    println!("  error budget used in these 6 hours: {}.{}% of the month's", used * 1000 / month / 10, used * 1000 / month % 10);
}

fn main() {
    latency();
    trace();
    alerts();
}

import java.util.Arrays;
import java.util.List;

public class Main {
    /** 1. Latency: 1,000 requests. Most take 50-100 ms; 2% hit a slow path and take 2 s. */
    static void latency() {
        int[] lat = new int[1000];
        int sum = 0;
        for (int i = 0; i < 1000; i++) {
            lat[i] = i % 50 == 7 ? 2000 : 50 + i * 37 % 51;
            sum += lat[i];
        }
        Arrays.sort(lat);
        System.out.println("1,000 requests, 2% of them hit a slow path:");
        System.out.printf("  average %d ms, p50 %d ms, p95 %d ms, p99 %d ms, max %d ms%n", sum / 1000, lat[499], lat[949], lat[989], lat[999]);
    }

    /** 2. A trace: one request as a tree of timed spans. */
    record Span(String name, int ms, List<Span> children) {
        Span(String name, int ms) { this(name, ms, List.of()); }
    }

    static Span printSpan(Span s, int depth, Span slowest) {
        System.out.printf("  %s%-" + (22 - 2 * depth) + "s %4d ms%n", "  ".repeat(depth), s.name(), s.ms());
        if (s.children().isEmpty() && s.ms() > slowest.ms()) slowest = s;
        for (Span c : s.children()) slowest = printSpan(c, depth + 1, slowest);
        return slowest;
    }

    static void trace() {
        Span root = new Span("GET /checkout", 420, List.of(
                new Span("auth", 12),
                new Span("load cart (db)", 40),
                new Span("payment", 330, List.of(new Span("fraud check", 280), new Span("charge card", 45))),
                new Span("render", 30)));
        System.out.println("trace of one slow checkout:");
        Span s = printSpan(root, 0, new Span("", 0));
        System.out.printf("  most of the time: %s, %d ms of %d (%d%%)%n", s.name(), s.ms(), root.ms(), s.ms() * 100 / root.ms());
    }

    /** 3. Alerts. SLO: 99.9% of requests succeed over 30 days, so the error budget is 0.1%.
     *  Error rates are in hundredths of a percent (150 = 1.50%). */
    static void alerts() {
        int[] errs = {5, 8, 150, 160, 20, 6};
        int[] cpu = {85, 88, 40, 35, 60, 82};
        final int budget = 10; // 0.10% in hundredths of a percent
        System.out.println("six hours of errors and CPU; SLO 99.9% (budget 0.1%):");
        int used = 0;
        for (int h = 0; h < errs.length; h++) {
            int burn = errs[h] * 10 / budget; // tenths: 150 -> 150 (15.0x the allowed rate)
            String page = burn >= 144 ? "PAGE" : "-"; // 14.4x for an hour = 2% of the month's budget gone
            String cpuAlert = cpu[h] > 80 ? "alert" : "-";
            System.out.printf("  hour %d: errors %d.%02d%%, burn %2d.%dx -> %-4s | CPU %d%% -> %s%n",
                    h + 1, errs[h] / 100, errs[h] % 100, burn / 10, burn % 10, page, cpu[h], cpuAlert);
            used += errs[h];
        }
        int month = budget * 30 * 24; // the whole month's budget, in the same units
        System.out.printf("  error budget used in these 6 hours: %d.%d%% of the month's%n", used * 1000 / month / 10, used * 1000 / month % 10);
    }

    public static void main(String[] args) {
        latency();
        trace();
        alerts();
    }
}

#include <algorithm>
#include <cstdio>
#include <numeric>
#include <string>
#include <vector>

// 1. Latency: 1,000 requests. Most take 50-100 ms; 2% hit a slow path and take 2 s.
void latency() {
    std::vector<int> lat(1000);
    for (int i = 0; i < 1000; i++) lat[i] = i % 50 == 7 ? 2000 : 50 + i * 37 % 51;
    int sum = std::accumulate(lat.begin(), lat.end(), 0);
    std::sort(lat.begin(), lat.end());
    std::printf("1,000 requests, 2%% of them hit a slow path:\n");
    std::printf("  average %d ms, p50 %d ms, p95 %d ms, p99 %d ms, max %d ms\n", sum / 1000, lat[499], lat[949], lat[989], lat[999]);
}

// 2. A trace: one request as a tree of timed spans.
struct Span {
    std::string name;
    int ms;
    std::vector<Span> children;
};

void printSpan(const Span& s, int depth, const Span*& slowest) {
    std::printf("  %s%-*s %4d ms\n", std::string(2 * depth, ' ').c_str(), 22 - 2 * depth, s.name.c_str(), s.ms);
    if (s.children.empty() && s.ms > slowest->ms) slowest = &s;
    for (const auto& c : s.children) printSpan(c, depth + 1, slowest);
}

void trace() {
    Span root{"GET /checkout", 420, {
        {"auth", 12, {}},
        {"load cart (db)", 40, {}},
        {"payment", 330, {{"fraud check", 280, {}}, {"charge card", 45, {}}}},
        {"render", 30, {}},
    }};
    std::printf("trace of one slow checkout:\n");
    Span none{"", 0, {}};
    const Span* slowest = &none;
    printSpan(root, 0, slowest);
    std::printf("  most of the time: %s, %d ms of %d (%d%%)\n", slowest->name.c_str(), slowest->ms, root.ms, slowest->ms * 100 / root.ms);
}

// 3. Alerts. SLO: 99.9% of requests succeed over 30 days, so the error budget is 0.1%.
// Error rates are in hundredths of a percent (150 = 1.50%).
void alerts() {
    const int errs[] = {5, 8, 150, 160, 20, 6};
    const int cpu[] = {85, 88, 40, 35, 60, 82};
    const int budget = 10;  // 0.10% in hundredths of a percent
    std::printf("six hours of errors and CPU; SLO 99.9%% (budget 0.1%%):\n");
    int used = 0;
    for (int h = 0; h < 6; h++) {
        int burn = errs[h] * 10 / budget;              // tenths: 150 -> 150 (15.0x the allowed rate)
        const char* page = burn >= 144 ? "PAGE" : "-";  // 14.4x for an hour = 2% of the month's budget gone
        const char* cpuAlert = cpu[h] > 80 ? "alert" : "-";
        std::printf("  hour %d: errors %d.%02d%%, burn %2d.%dx -> %-4s | CPU %d%% -> %s\n", h + 1, errs[h] / 100, errs[h] % 100,
                    burn / 10, burn % 10, page, cpu[h], cpuAlert);
        used += errs[h];
    }
    int month = budget * 30 * 24;  // the whole month's budget, in the same units
    std::printf("  error budget used in these 6 hours: %d.%d%% of the month's\n", used * 1000 / month / 10, used * 1000 / month % 10);
}

int main() {
    latency();
    trace();
    alerts();
}