附录 C · 手撕代码速查

import numpy as np

def softmax(z):                                   # z: (N, C)
    z = z - z.max(axis=1, keepdims=True)          # ⭐ 减最大值，防 exp 溢出
    e = np.exp(z)
    return e / e.sum(axis=1, keepdims=True)

def cross_entropy(z, y):                          # y: (N,) 整数标签
    p = softmax(z)                                # (N, C)
    n = z.shape[0]
    loss = -np.log(p[np.arange(n), y] + 1e-12).mean()   # ⭐ 只取真实类那一项
    grad = p.copy()
    grad[np.arange(n), y] -= 1.0                  # ⭐ dL/dz = p − onehot(y)
    grad /= n                                     # ⭐ 前向用了 mean()，反向就要除 n
    return loss, grad

np.random.seed(0)
z = np.random.randn(4, 3) * 2
y = np.array([0, 2, 1, 1])
loss, grad = cross_entropy(z, y)
print(f"loss = {loss:.4f}")

# 数值梯度校验（和第 8 章 / 附录 B 的 grad_check 是同一招）
eps, num = 1e-5, np.zeros_like(z)
for i in range(z.shape[0]):
    for j in range(z.shape[1]):
        zp = z.copy(); zp[i, j] += eps
        zm = z.copy(); zm[i, j] -= eps
        num[i, j] = (cross_entropy(zp, y)[0] - cross_entropy(zm, y)[0]) / (2 * eps)
print("解析梯度 vs 数值梯度，最大误差:", np.abs(grad - num).max())

loss = 3.0697
解析梯度 vs 数值梯度，最大误差: 2.2784607534021006e-10

import torch
import torch.nn.functional as F

def attention(q, k, v, causal=False):             # q,k,v: (B, H, L, d)
    d = q.size(-1)
    s = q @ k.transpose(-2, -1) / d ** 0.5        # ⭐ 除 √d_k：第 14 章那个方差推导
    if causal:
        L = s.size(-1)
        mask = torch.ones(L, L, dtype=torch.bool, device=q.device).triu(1)  # ⭐ 严格上三角
        s = s.masked_fill(mask, float("-inf"))    # ⭐ 填 -inf，不是填 0
    w = s.softmax(dim=-1)                         # (B, H, L, L)
    return w @ v, w                               # (B, H, L, d)

torch.manual_seed(0)
q, k, v = (torch.randn(2, 4, 6, 16) for _ in range(3))
out, w = attention(q, k, v, causal=True)
print("输出形状", tuple(out.shape), " 权重形状", tuple(w.shape))
print("第 1 个位置只看得到自己:", w[0, 0, 0].tolist())
print("每行权重和:", w.sum(-1)[0, 0].tolist())
ref = F.scaled_dot_product_attention(q, k, v, is_causal=True)
print("与 PyTorch 官方实现最大误差:", (out - ref).abs().max().item())

输出形状 (2, 4, 6, 16)  权重形状 (2, 4, 6, 6)
第 1 个位置只看得到自己: [1.0, 0.0, 0.0, 0.0, 0.0, 0.0]
每行权重和: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
与 PyTorch 官方实现最大误差: 2.384185791015625e-07

import torch
import torch.nn as nn

class MultiHeadAttention(nn.Module):
    def __init__(self, d_model, n_head):
        super().__init__()
        assert d_model % n_head == 0                        # ⭐ 必须整除
        self.h, self.dk = n_head, d_model // n_head
        self.qkv = nn.Linear(d_model, 3 * d_model)          # 一次算完 Q K V
        self.proj = nn.Linear(d_model, d_model)             # ⭐ 输出投影，别漏

    def forward(self, x, causal=True):                      # x: (B, L, d_model)
        B, L, D = x.shape
        q, k, v = self.qkv(x).chunk(3, dim=-1)              # 各 (B, L, D)
        # ⭐ 必须先 view 再 transpose：(B,L,D) -> (B,L,h,dk) -> (B,h,L,dk)
        q, k, v = [t.view(B, L, self.h, self.dk).transpose(1, 2) for t in (q, k, v)]
        s = q @ k.transpose(-2, -1) / self.dk ** 0.5        # (B, h, L, L)
        if causal:
            m = torch.ones(L, L, dtype=torch.bool, device=x.device).triu(1)
            s = s.masked_fill(m, float("-inf"))
        o = s.softmax(-1) @ v                               # (B, h, L, dk)
        o = o.transpose(1, 2).contiguous().view(B, L, D)    # ⭐ 拼回 (B, L, D)
        return self.proj(o)

torch.manual_seed(0)
mha = MultiHeadAttention(64, 8)
x = torch.randn(2, 10, 64)
print("输出形状", tuple(mha(x).shape))
print("参数量", sum(p.numel() for p in mha.parameters()))

输出形状 (2, 10, 64)
参数量 16640

import torch
import torch.nn as nn

class LayerNorm(nn.Module):
    def __init__(self, dim, eps=1e-5):
        super().__init__()
        self.g = nn.Parameter(torch.ones(dim))               # ⭐ 缩放，初始 1
        self.b = nn.Parameter(torch.zeros(dim))              # ⭐ 平移，初始 0
        self.eps = eps

    def forward(self, x):                                    # x: (..., dim)
        mu = x.mean(-1, keepdim=True)                        # ⭐ 只在最后一维统计
        var = x.var(-1, unbiased=False, keepdim=True)        # ⭐ 有偏方差，分母是 n
        return (x - mu) / torch.sqrt(var + self.eps) * self.g + self.b
        #                          ↑ eps 在 sqrt 里面

torch.manual_seed(0)
x = torch.randn(3, 5, 8) * 4 + 7
mine, ref = LayerNorm(8), nn.LayerNorm(8)
print("与 nn.LayerNorm 最大误差:", (mine(x) - ref(x)).abs().max().item())
y = mine(x)
print("归一化后一个样本的均值 / 标准差:",
      round(y.mean(-1)[0, 0].item(), 6), round(y.std(-1, unbiased=False)[0, 0].item(), 4))

与 nn.LayerNorm 最大误差: 3.5762786865234375e-07
归一化后一个样本的均值 / 标准差: -0.0 1.0

import torch
import torch.nn as nn

class BatchNorm1d(nn.Module):
    def __init__(self, dim, eps=1e-5, momentum=0.1):
        super().__init__()
        self.g = nn.Parameter(torch.ones(dim))
        self.b = nn.Parameter(torch.zeros(dim))
        self.register_buffer("running_mean", torch.zeros(dim))   # ⭐ buffer 不是 Parameter
        self.register_buffer("running_var", torch.ones(dim))
        self.eps, self.momentum = eps, momentum

    def forward(self, x):                                    # x: (N, dim)
        if self.training:                                    # ⭐ 训练：用这一批的统计量
            mu = x.mean(0)
            var = x.var(0, unbiased=False)                   # ⭐ 归一化用有偏
            n = x.shape[0]
            with torch.no_grad():
                self.running_mean.mul_(1 - self.momentum).add_(self.momentum * mu)
                # ⭐ 但 running_var 存的是无偏（× n/(n−1)），和 PyTorch 对齐
                self.running_var.mul_(1 - self.momentum).add_(
                    self.momentum * var * n / (n - 1))
        else:                                                # ⭐ 推理：用滑动平均
            mu, var = self.running_mean, self.running_var
        return (x - mu) / torch.sqrt(var + self.eps) * self.g + self.b

torch.manual_seed(0)
mine, ref = BatchNorm1d(4), nn.BatchNorm1d(4)
for _ in range(20):
    x = torch.randn(32, 4) * 3 + 1
    mine(x); ref(x)                                          # 训练模式，累积 running stats
print("running_mean 误差:", (mine.running_mean - ref.running_mean).abs().max().item())
print("running_var  误差:", (mine.running_var - ref.running_var).abs().max().item())
mine.eval(); ref.eval()
xt = torch.randn(5, 4)
print("eval 模式输出误差:", (mine(xt) - ref(xt)).abs().max().item())

running_mean 误差: 1.1920928955078125e-07
running_var  误差: 0.0
eval 模式输出误差: 1.1920928955078125e-07

import torch

def dropout(x, p=0.5, training=True):
    if not training or p == 0:
        return x                                   # ⭐ 推理：原样返回，什么都不做
    if p == 1:
        return torch.zeros_like(x)                 # 边界情况，否则下面除 0
    mask = (torch.rand_like(x) >= p).float()       # ⭐ 保留概率是 1−p
    return x * mask / (1 - p)                      # ⭐ inverted dropout：训练时就除回来

torch.manual_seed(0)
x = torch.ones(200000)
y = dropout(x, p=0.3, training=True)
print(f"被置零的比例 {(y == 0).float().mean():.4f}  （应 ≈ p = 0.3）")
print(f"训练时输出均值 {y.mean():.4f}   推理时输出均值 {dropout(x, 0.3, False).mean():.4f}")

被置零的比例 0.2988  （应 ≈ p = 0.3）
训练时输出均值 1.0018   推理时输出均值 1.0000

import numpy as np

def conv2d(x, w, b=None, stride=1, pad=0):
    N, Cin, H, W = x.shape                          # x: (N, Cin, H, W)
    Cout, _, kh, kw = w.shape                       # w: (Cout, Cin, kh, kw)
    Ho = (H + 2 * pad - kh) // stride + 1           # ⭐ 输出尺寸公式，向下取整
    Wo = (W + 2 * pad - kw) // stride + 1
    xp = np.pad(x, ((0, 0), (0, 0), (pad, pad), (pad, pad)))   # ⭐ 只补 H/W 两维
    out = np.zeros((N, Cout, Ho, Wo))
    for i in range(Ho):
        for j in range(Wo):
            patch = xp[:, :, i*stride:i*stride+kh, j*stride:j*stride+kw]  # (N,Cin,kh,kw)
            # ⭐ 对 Cin、kh、kw 三个维度一起求和 —— 通道是一次吃掉全部
            out[:, :, i, j] = np.tensordot(patch, w, axes=([1, 2, 3], [1, 2, 3]))
    return out + (b.reshape(1, -1, 1, 1) if b is not None else 0)

np.random.seed(0)
x = np.random.randn(2, 3, 8, 8)
w = np.random.randn(4, 3, 3, 3)
b = np.random.randn(4)
out = conv2d(x, w, b, stride=2, pad=1)
print("输出形状", out.shape, " 参数量", w.size + b.size)

import torch
import torch.nn.functional as F
ref = F.conv2d(torch.tensor(x), torch.tensor(w), torch.tensor(b), stride=2, padding=1)
print("与 F.conv2d 最大误差:", np.abs(out - ref.numpy()).max())

输出形状 (2, 4, 4, 4)  参数量 112
与 F.conv2d 最大误差: 0.0

import numpy as np

def kmeans(X, k, iters=100, seed=0):
    rng = np.random.default_rng(seed)
    C = X[rng.choice(len(X), k, replace=False)].copy()   # ⭐ 从样本里挑初始中心
    lab = np.zeros(len(X), int)
    for _ in range(iters):
        d = ((X[:, None, :] - C[None, :, :]) ** 2).sum(-1)   # (n,k) 广播算距离
        lab = d.argmin(1)                                    # ① 归到最近的中心
        newC = np.array([X[lab == j].mean(0) if (lab == j).any() else C[j]
                         for j in range(k)])                 # ② 移到重心（⭐ 处理空簇）
        if np.allclose(newC, C):                             # ⭐ 中心不动 = 收敛
            break
        C = newC
    return lab, C, ((X - C[lab]) ** 2).sum()                 # 第三个返回值是惯性

rng = np.random.default_rng(0)
X = np.vstack([rng.normal(m, 0.6, (150, 2)) for m in ([0, 0], [4, 4], [0, 5])])
best = min((kmeans(X, 3, seed=s) for s in range(10)), key=lambda r: r[2])   # ⭐ n_init=10
print(f"惯性 = {best[2]:.1f}   每簇样本数 = {np.bincount(best[0])}")

from sklearn.cluster import KMeans
print(f"sklearn 惯性 = {KMeans(3, n_init=10, random_state=0).fit(X).inertia_:.1f}")

惯性 = 303.1   每簇样本数 = [149 150 151]
sklearn 惯性 = 303.1

import numpy as np

def auc(y, score):
    y, score = np.asarray(y), np.asarray(score)
    order = np.argsort(score, kind="mergesort")      # ⭐ O(n log n)，瓶颈就这一行
    ranks = np.empty(len(y), float)
    ranks[order] = np.arange(1, len(y) + 1)          # ⭐ 秩从 1 开始
    s = score[order]
    i = 0
    while i < len(s):                                # ⭐ 分数相同的必须取平均秩
        j = i
        while j + 1 < len(s) and s[j + 1] == s[i]:
            j += 1
        if j > i:
            ranks[order[i:j + 1]] = (i + j + 2) / 2
        i = j + 1
    P, N = int((y == 1).sum()), int((y == 0).sum())
    return (ranks[y == 1].sum() - P * (P + 1) / 2) / (P * N)   # ⭐ 秩和公式

rng = np.random.default_rng(0)
y = rng.integers(0, 2, 2000)
score = np.round(rng.normal(y * 1.2, 1.0), 1)        # 四舍五入，制造大量并列
print(f"我的 AUC  {auc(y, score):.6f}")

from sklearn.metrics import roc_auc_score
print(f"sklearn   {roc_auc_score(y, score):.6f}")
print(f"分数整体 ×100 再 +5 之后 {auc(y, score * 100 + 5):.6f}")

我的 AUC  0.807631
sklearn   0.807631
分数整体 ×100 再 +5 之后 0.807631

import numpy as np

def fit_logreg(X, y, lr=0.5, epochs=2000, l2=0.0):
    n, d = X.shape
    w, b = np.zeros(d), 0.0                       # ⭐ 逻辑回归可以零初始化（神经网络不行）
    for _ in range(epochs):
        p = 1 / (1 + np.exp(-(X @ w + b)))        # sigmoid
        # ⭐ 梯度正好是 (预测 − 真实)：第 3 章那个经典考点
        gw = X.T @ (p - y) / n + l2 * w           # ⭐ 别忘了 /n
        gb = (p - y).mean()                       # ⭐ 偏置不加 L2
        w -= lr * gw
        b -= lr * gb
    return w, b

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

X, y = load_breast_cancer(return_X_y=True)
Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=.3, random_state=42, stratify=y)
sc = StandardScaler().fit(Xtr)                    # ⭐ 必须标准化，且只 fit 训练集
Xtr, Xte = sc.transform(Xtr), sc.transform(Xte)
w, b = fit_logreg(Xtr, ytr)
print(f"手写逻辑回归 测试集准确率 {((Xte @ w + b > 0) == yte).mean():.1%}")

from sklearn.linear_model import LogisticRegression
print(f"sklearn      测试集准确率 "
      f"{LogisticRegression(max_iter=5000).fit(Xtr, ytr).score(Xte, yte):.1%}")

手写逻辑回归 测试集准确率 98.2%
sklearn      测试集准确率 98.8%