Model Training and Evaluation

# A fraud model's scores on 20 payments, with the truth (1 = fraud). Fraud is rare: 4 of 20.
SCORES = [0.95, 0.80, 0.62, 0.40, 0.70, 0.55, 0.30, 0.25, 0.20, 0.18,
          0.15, 0.12, 0.10, 0.08, 0.07, 0.05, 0.04, 0.03, 0.02, 0.01]
TRUTH = [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

def confusion(threshold):
    tp = fp = fn = tn = 0
    for s, y in zip(SCORES, TRUTH):
        flagged = s >= threshold
        if flagged and y: tp += 1
        elif flagged: fp += 1
        elif y: fn += 1
        else: tn += 1
    return tp, fp, fn, tn

tp, fp, fn, tn = confusion(0.5)
print("threshold 0.5        actual fraud  actual ok")
print(f"  flagged as fraud   {tp:>12}  {fp:>9}")
print(f"  let through        {fn:>12}  {tn:>9}")
print(f"accuracy {(tp + tn) / 20:.2f}   (a model that never flags anything: {16 / 20:.2f})")

print("\nthreshold  precision  recall")
for t in (0.3, 0.5, 0.7, 0.9):
    tp, fp, fn, tn = confusion(t)
    print(f"{t:>9}  {tp / (tp + fp):>9.2f}  {tp / (tp + fn):>6.2f}")

# ROC-AUC: the chance that a random fraud scores higher than a random normal payment.
pos = [s for s, y in zip(SCORES, TRUTH) if y]
neg = [s for s, y in zip(SCORES, TRUTH) if not y]
wins = sum(p > n for p in pos for n in neg)
print(f"\nROC-AUC = {wins}/{len(pos) * len(neg)} = {wins / (len(pos) * len(neg)):.3f}")

# 5-fold cross-validation of a one-threshold model on 20 students (hours, passed).
STUDENTS = [(1.0, 0), (6.5, 1), (2.5, 0), (8.0, 1), (4.0, 0), (5.5, 1), (3.0, 0), (7.0, 1), (4.5, 1), (2.0, 0),
            (6.0, 1), (3.5, 0), (5.0, 1), (1.5, 0), (4.2, 1), (4.8, 0), (3.8, 0), (5.8, 1), (2.8, 0), (6.2, 1)]
def fit(rows):
    def acc(t):
        return sum((h >= t) == bool(y) for h, y in rows)
    return max(sorted(h for h, _ in rows), key=acc)
fold_scores = []
for k in range(5):
    test = STUDENTS[k * 4:(k + 1) * 4]
    train = STUDENTS[:k * 4] + STUDENTS[(k + 1) * 4:]
    t = fit(train)
    fold_scores.append(sum((h >= t) == bool(y) for h, y in test) / len(test))
    print(f"fold {k + 1}: threshold {t}, test accuracy {fold_scores[-1]:.2f}")
mean = sum(fold_scores) / 5
std = (sum((s - mean) ** 2 for s in fold_scores) / 5) ** 0.5
print(f"cross-validated accuracy {mean:.2f} ± {std:.2f}")

package main

import (
	"fmt"
	"math"
	"sort"
)

// A fraud model's scores on 20 payments, with the truth (1 = fraud). Fraud is rare: 4 of 20.
var scores = []float64{0.95, 0.80, 0.62, 0.40, 0.70, 0.55, 0.30, 0.25, 0.20, 0.18,
	0.15, 0.12, 0.10, 0.08, 0.07, 0.05, 0.04, 0.03, 0.02, 0.01}
var truth = []int{1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

func confusion(threshold float64) (tp, fp, fn, tn int) {
	for i, s := range scores {
		flagged := s >= threshold
		switch {
		case flagged && truth[i] == 1:
			tp++
		case flagged:
			fp++
		case truth[i] == 1:
			fn++
		default:
			tn++
		}
	}
	return
}

type student struct {
	hours  float64
	passed bool
}

func correct(rows []student, t float64) int {
	n := 0
	for _, r := range rows {
		if (r.hours >= t) == r.passed {
			n++
		}
	}
	return n
}

func fit(rows []student) float64 {
	var ts []float64
	for _, r := range rows {
		ts = append(ts, r.hours)
	}
	sort.Float64s(ts)
	best := ts[0]
	for _, t := range ts {
		if correct(rows, t) > correct(rows, best) {
			best = t
		}
	}
	return best
}

func main() {
	tp, fp, fn, tn := confusion(0.5)
	fmt.Println("threshold 0.5        actual fraud  actual ok")
	fmt.Printf("  flagged as fraud   %12d  %9d\n", tp, fp)
	fmt.Printf("  let through        %12d  %9d\n", fn, tn)
	fmt.Printf("accuracy %.2f   (a model that never flags anything: %.2f)\n", float64(tp+tn)/20, 16.0/20)

	fmt.Println("\nthreshold  precision  recall")
	for _, t := range []float64{0.3, 0.5, 0.7, 0.9} {
		tp, fp, fn, _ := confusion(t)
		fmt.Printf("%9v  %9.2f  %6.2f\n", t, float64(tp)/float64(tp+fp), float64(tp)/float64(tp+fn))
	}

	// ROC-AUC: the chance that a random fraud scores higher than a random normal payment.
	wins, pairs := 0, 0
	for i, p := range scores {
		for j, n := range scores {
			if truth[i] == 1 && truth[j] == 0 {
				pairs++
				if p > n {
					wins++
				}
			}
		}
	}
	fmt.Printf("\nROC-AUC = %d/%d = %.3f\n", wins, pairs, float64(wins)/float64(pairs))

	// 5-fold cross-validation of a one-threshold model on 20 students (hours, passed).
	students := []student{{1.0, false}, {6.5, true}, {2.5, false}, {8.0, true}, {4.0, false}, {5.5, true}, {3.0, false},
		{7.0, true}, {4.5, true}, {2.0, false}, {6.0, true}, {3.5, false}, {5.0, true}, {1.5, false}, {4.2, true},
		{4.8, false}, {3.8, false}, {5.8, true}, {2.8, false}, {6.2, true}}
	var foldScores []float64
	for k := 0; k < 5; k++ {
		test := students[k*4 : (k+1)*4]
		train := append(append([]student{}, students[:k*4]...), students[(k+1)*4:]...)
		t := fit(train)
		foldScores = append(foldScores, float64(correct(test, t))/float64(len(test)))
		fmt.Printf("fold %d: threshold %v, test accuracy %.2f\n", k+1, t, foldScores[k])
	}
	mean, sq := 0.0, 0.0
	for _, s := range foldScores {
		mean += s / 5
	}
	for _, s := range foldScores {
		sq += (s - mean) * (s - mean) / 5
	}
	fmt.Printf("cross-validated accuracy %.2f ± %.2f\n", mean, math.Sqrt(sq))
}