When Software Serves Many Users

package main

import (
	"fmt"
	"net"
	"net/http"
	"slices"
	"sync"
	"time"
)

// Each request needs 100 ms, like waiting for a database to answer.
func work() { time.Sleep(100 * time.Millisecond) }

// serve starts a real HTTP server on a free port and returns its address.
func serve(oneAtATime bool) string {
	var mu sync.Mutex
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if oneAtATime {
			mu.Lock() // only one request may be inside at a time
			defer mu.Unlock()
		}
		work()
		fmt.Fprint(w, "ok")
	})
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	go http.Serve(ln, handler)
	return "http://" + ln.Addr().String()
}

// load sends `users` requests at the same moment and returns when each one finished.
func load(url string, users int) []time.Duration {
	start := time.Now()
	done := make([]time.Duration, users)
	var wg sync.WaitGroup
	for i := range users {
		wg.Add(1)
		go func() {
			defer wg.Done()
			resp, err := http.Get(url)
			if err != nil {
				panic(err)
			}
			resp.Body.Close()
			done[i] = time.Since(start)
		}()
	}
	wg.Wait()
	slices.Sort(done)
	return done
}

func secs(d time.Duration) string {
	return fmt.Sprintf("%.1f s", d.Truncate(100*time.Millisecond).Seconds()) // rounded down to 0.1 s
}

func main() {
	const users = 20
	fmt.Printf("%d users click at the same moment; each request needs 100 ms\n", users)
	for _, mode := range []struct {
		name string
		one  bool
	}{{"one at a time", true}, {"side by side ", false}} {
		t := load(serve(mode.one), users)
		fmt.Printf("%s: first done after %s, last after %s\n", mode.name, secs(t[0]), secs(t[users-1]))
	}
}

const http = require("node:http");

// Each request needs 100 ms, like waiting for a database to answer.
const work = () => new Promise((resolve) => setTimeout(resolve, 100));

// serve starts a real HTTP server on a free port and resolves to its address.
function serve(oneAtATime) {
  let line = Promise.resolve(); // requests queue here when only one may run
  const server = http.createServer((req, res) => {
    const handle = () => work().then(() => res.end("ok"));
    if (oneAtATime) line = line.then(handle);
    else handle();
  });
  return new Promise((resolve) =>
    server.listen(0, "127.0.0.1", () => {
      server.unref();
      resolve(`http://127.0.0.1:${server.address().port}`);
    }),
  );
}

// load sends `users` requests at the same moment and resolves to when each one finished.
async function load(url, users) {
  const start = performance.now();
  const one = () =>
    new Promise((resolve, reject) => {
      http.get(url, (res) => {
        res.resume();
        res.on("end", () => resolve(performance.now() - start));
      }).on("error", reject);
    });
  const done = await Promise.all(Array.from({ length: users }, one));
  return done.sort((a, b) => a - b);
}

const secs = (ms) => `${(Math.floor(ms / 100) / 10).toFixed(1)} s`; // rounded down to 0.1 s

async function main() {
  const users = 20;
  console.log(`${users} users click at the same moment; each request needs 100 ms`);
  for (const [name, one] of [["one at a time", true], ["side by side ", false]]) {
    const t = await load(await serve(one), users);
    console.log(`${name}: first done after ${secs(t[0])}, last after ${secs(t[users - 1])}`);
  }
}

main();

use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::thread;
use std::time::{Duration, Instant};

// Each request needs 100 ms, like waiting for a database to answer.
fn work() {
    thread::sleep(Duration::from_millis(100));
}

// Reads the request, does the work, and answers with a tiny HTTP response.
fn handle(mut conn: TcpStream) {
    let mut buf = [0u8; 1024];
    let _ = conn.read(&mut buf);
    work();
    let _ = conn.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok");
}

// Starts a real HTTP server on a free port and returns its address.
fn serve(one_at_a_time: bool) -> String {
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let addr = listener.local_addr().unwrap().to_string();
    thread::spawn(move || {
        for conn in listener.incoming() {
            let conn = conn.unwrap();
            if one_at_a_time {
                handle(conn); // the next connection waits until this one is finished
            } else {
                thread::spawn(move || handle(conn)); // every connection gets its own thread
            }
        }
    });
    addr
}

// Sends `users` requests at the same moment and returns when each one finished.
fn load(addr: &str, users: usize) -> Vec<Duration> {
    let start = Instant::now();
    let workers: Vec<_> = (0..users)
        .map(|_| {
            let addr = addr.to_string();
            thread::spawn(move || {
                let mut conn = TcpStream::connect(&addr).unwrap();
                conn.write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n").unwrap();
                let mut reply = Vec::new();
                conn.read_to_end(&mut reply).unwrap();
                start.elapsed()
            })
        })
        .collect();
    let mut done: Vec<Duration> = workers.into_iter().map(|w| w.join().unwrap()).collect();
    done.sort();
    done
}

fn secs(d: Duration) -> String {
    format!("{:.1} s", (d.as_secs_f64() * 10.0).floor() / 10.0) // rounded down to 0.1 s
}

fn main() {
    let users = 20;
    println!("{users} users click at the same moment; each request needs 100 ms");
    for (name, one) in [("one at a time", true), ("side by side ", false)] {
        let t = load(&serve(one), users);
        println!("{name}: first done after {}, last after {}", secs(t[0]), secs(t[users - 1]));
    }
}

import math, threading, time, urllib.request
from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer


def work():
    """Each request needs 100 ms, like waiting for a database to answer."""
    time.sleep(0.1)


class Handler(BaseHTTPRequestHandler):
    def do_GET(self):
        work()
        self.send_response(200)
        self.end_headers()
        self.wfile.write(b"ok")

    def log_message(self, *args):  # keep the output quiet
        pass


def serve(one_at_a_time):
    """Starts a real HTTP server on a free port and returns its address."""
    # HTTPServer handles one request at a time; ThreadingHTTPServer gives each its own thread.
    cls = HTTPServer if one_at_a_time else ThreadingHTTPServer
    cls.request_queue_size = 64  # room for everyone to wait in line
    server = cls(("127.0.0.1", 0), Handler)
    threading.Thread(target=server.serve_forever, daemon=True).start()
    return f"http://127.0.0.1:{server.server_port}"


def load(url, users):
    """Sends `users` requests at the same moment and returns when each one finished."""
    start = time.perf_counter()
    done = []

    def one():
        urllib.request.urlopen(url).read()
        done.append(time.perf_counter() - start)

    threads = [threading.Thread(target=one) for _ in range(users)]
    for t in threads:
        t.start()
    for t in threads:
        t.join()
    return sorted(done)


def secs(s):
    return f"{math.floor(s * 10) / 10:.1f} s"  # rounded down to 0.1 s


users = 20
print(f"{users} users click at the same moment; each request needs 100 ms")
for name, one in [("one at a time", True), ("side by side ", False)]:
    t = load(serve(one), users)
    print(f"{name}: first done after {secs(t[0])}, last after {secs(t[-1])}")

import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;

public class Step1 {
    // Each request needs 100 ms, like waiting for a database to answer.
    static void work() throws InterruptedException {
        Thread.sleep(100);
    }

    // Starts a real HTTP server on a free port and returns its address.
    static String serve(boolean oneAtATime) throws Exception {
        HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 64);
        server.createContext("/", ex -> {
            try { work(); } catch (InterruptedException e) { throw new RuntimeException(e); }
            byte[] body = "ok".getBytes();
            ex.sendResponseHeaders(200, body.length);
            ex.getResponseBody().write(body);
            ex.close();
        });
        // One thread handles requests one at a time; virtual threads give each request its own.
        server.setExecutor(oneAtATime ? Executors.newSingleThreadExecutor() : Executors.newVirtualThreadPerTaskExecutor());
        server.start();
        return "http://127.0.0.1:" + server.getAddress().getPort() + "/";
    }

    // Sends `users` requests at the same moment and returns when each one finished (ms).
    static List<Double> load(String url, int users) {
        HttpClient client = HttpClient.newHttpClient();
        // Warm-up: the JVM loads classes and opens its first connection on the first call.
        // Paying that cost here keeps it out of the timing below.
        client.sendAsync(HttpRequest.newBuilder(URI.create(url)).build(), HttpResponse.BodyHandlers.ofString()).join();
        long start = System.nanoTime();
        List<CompletableFuture<Double>> calls = new ArrayList<>();
        for (int i = 0; i < users; i++) {
            calls.add(client.sendAsync(HttpRequest.newBuilder(URI.create(url)).build(), HttpResponse.BodyHandlers.ofString())
                    .thenApply(r -> (System.nanoTime() - start) / 1e6));
        }
        List<Double> done = new ArrayList<>(calls.stream().map(CompletableFuture::join).toList());
        Collections.sort(done);
        return done;
    }

    static String secs(double ms) {
        return String.format("%.1f s", Math.floor(ms / 100) / 10.0); // rounded down to 0.1 s
    }

    public static void main(String[] args) throws Exception {
        int users = 20;
        System.out.printf("%d users click at the same moment; each request needs 100 ms%n", users);
        for (Object[] mode : new Object[][] {{"one at a time", true}, {"side by side ", false}}) {
            List<Double> t = load(serve((Boolean) mode[1]), users);
            System.out.printf("%s: first done after %s, last after %s%n", mode[0], secs(t.get(0)), secs(t.get(users - 1)));
        }
        System.exit(0);
    }
}

package main

import (
	"fmt"
	"math"
)

const serviceMs = 100.0 // one request keeps the server busy for 100 ms

func main() {
	fmt.Println("one server, one request at a time, 100 ms each (users arrive at random)")
	fmt.Println("users/s   busy   avg wait   avg total")
	for _, rate := range []float64{2, 5, 8, 9, 9.5, 10, 12} {
		busy := rate * serviceMs / 1000 // share of each second the server is working
		switch {
		case busy < 1:
			wait := busy / (1 - busy) * serviceMs // average time spent in line
			fmt.Printf("%7.1f  %4.0f%%  %6.0f ms  %7.0f ms\n", rate, busy*100, math.Round(wait), math.Round(wait+serviceMs))
		case busy == 1:
			fmt.Printf("%7.1f  %4.0f%%  the line never settles down\n", rate, busy*100)
		default:
			extra := rate - 1000/serviceMs // arrivals the server can't keep up with
			fmt.Printf("%7.1f  %4.0f%%  the line grows by %.0f people every second\n", rate, busy*100, extra)
		}
	}
}

const serviceMs = 100; // one request keeps the server busy for 100 ms
const pad = (v, width, digits = 0) => v.toFixed(digits).padStart(width);

console.log("one server, one request at a time, 100 ms each (users arrive at random)");
console.log("users/s   busy   avg wait   avg total");
for (const rate of [2, 5, 8, 9, 9.5, 10, 12]) {
  const busy = (rate * serviceMs) / 1000; // share of each second the server is working
  if (busy < 1) {
    const wait = (busy / (1 - busy)) * serviceMs; // average time spent in line
    console.log(`${pad(rate, 7, 1)}  ${pad(busy * 100, 4)}%  ${pad(Math.round(wait), 6)} ms  ${pad(Math.round(wait + serviceMs), 7)} ms`);
  } else if (busy === 1) {
    console.log(`${pad(rate, 7, 1)}  ${pad(busy * 100, 4)}%  the line never settles down`);
  } else {
    const extra = rate - 1000 / serviceMs; // arrivals the server can't keep up with
    console.log(`${pad(rate, 7, 1)}  ${pad(busy * 100, 4)}%  the line grows by ${extra.toFixed(0)} people every second`);
  }
}

const SERVICE_MS: f64 = 100.0; // one request keeps the server busy for 100 ms

fn main() {
    println!("one server, one request at a time, 100 ms each (users arrive at random)");
    println!("users/s   busy   avg wait   avg total");
    for rate in [2.0, 5.0, 8.0, 9.0, 9.5, 10.0, 12.0_f64] {
        let busy = rate * SERVICE_MS / 1000.0; // share of each second the server is working
        if busy < 1.0 {
            let wait = busy / (1.0 - busy) * SERVICE_MS; // average time spent in line
            println!("{:7.1}  {:4.0}%  {:6.0} ms  {:7.0} ms", rate, busy * 100.0, wait.round(), (wait + SERVICE_MS).round());
        } else if busy == 1.0 {
            println!("{:7.1}  {:4.0}%  the line never settles down", rate, busy * 100.0);
        } else {
            let extra = rate - 1000.0 / SERVICE_MS; // arrivals the server can't keep up with
            println!("{:7.1}  {:4.0}%  the line grows by {:.0} people every second", rate, busy * 100.0, extra);
        }
    }
}

SERVICE_MS = 100  # one request keeps the server busy for 100 ms

print("one server, one request at a time, 100 ms each (users arrive at random)")
print("users/s   busy   avg wait   avg total")
for rate in [2, 5, 8, 9, 9.5, 10, 12]:
    busy = rate * SERVICE_MS / 1000  # share of each second the server is working
    if busy < 1:
        wait = busy / (1 - busy) * SERVICE_MS  # average time spent in line
        print(f"{rate:7.1f}  {busy * 100:4.0f}%  {round(wait):6d} ms  {round(wait + SERVICE_MS):7d} ms")
    elif busy == 1:
        print(f"{rate:7.1f}  {busy * 100:4.0f}%  the line never settles down")
    else:
        extra = rate - 1000 / SERVICE_MS  # arrivals the server can't keep up with
        print(f"{rate:7.1f}  {busy * 100:4.0f}%  the line grows by {extra:.0f} people every second")

public class Step2 {
    static final double SERVICE_MS = 100; // one request keeps the server busy for 100 ms

    public static void main(String[] args) {
        System.out.println("one server, one request at a time, 100 ms each (users arrive at random)");
        System.out.println("users/s   busy   avg wait   avg total");
        for (double rate : new double[] {2, 5, 8, 9, 9.5, 10, 12}) {
            double busy = rate * SERVICE_MS / 1000; // share of each second the server is working
            if (busy < 1) {
                double wait = busy / (1 - busy) * SERVICE_MS; // average time spent in line
                System.out.printf("%7.1f  %4.0f%%  %6d ms  %7d ms%n", rate, busy * 100, Math.round(wait), Math.round(wait + SERVICE_MS));
            } else if (busy == 1) {
                System.out.printf("%7.1f  %4.0f%%  the line never settles down%n", rate, busy * 100);
            } else {
                double extra = rate - 1000 / SERVICE_MS; // arrivals the server can't keep up with
                System.out.printf("%7.1f  %4.0f%%  the line grows by %.0f people every second%n", rate, busy * 100, extra);
            }
        }
    }
}