Why Log, Softmax & Likelihood

import numpy as np
import torch
import torch.nn.functional as F

# ─────────────────────────────────────────────────────────────
# 1. LOG OF PROBABILITIES
# ─────────────────────────────────────────────────────────────
print("Probability  →  −log(p)  [the loss]")
for p in [0.99, 0.5, 0.1, 0.01]:
    print(f"  p = {p:.2f}  →  loss = {-np.log(p):.3f}")
# p=0.99 → 0.010  (model was right, tiny loss)
# p=0.01 → 4.605  (model was catastrophically wrong)


# ─────────────────────────────────────────────────────────────
# 2. LIKELIHOOD — the coin toss example
# ─────────────────────────────────────────────────────────────
# Observed: H, H, T, H (data is FIXED)
# Try different theta values to find maximum likelihood
def likelihood(theta):
    """L(theta | H,H,T,H) = theta³ × (1-theta)"""
    return theta**3 * (1 - theta)

print("\nLikelihood for H,H,T,H:")
for theta in [0.3, 0.5, 0.75, 0.8, 0.9]:
    print(f"  theta={theta} → L={likelihood(theta):.4f}")
# theta=0.75 gives maximum → that's the MLE estimate (3/4 were Heads)


# ─────────────────────────────────────────────────────────────
# 3. SOFTMAX + TEMPERATURE
# ─────────────────────────────────────────────────────────────
def softmax(z, tau=1.0):
    """Softmax with temperature τ. Standard softmax when τ=1."""
    z_scaled = z / tau
    exp_z = np.exp(z_scaled - np.max(z_scaled))  # subtract max for stability
    return exp_z / np.sum(exp_z)

logits = np.array([2.1, 1.0, -0.5])

print("\nTemperature effect on same logits [2.1, 1.0, -0.5]:")
for tau in [0.5, 1.0, 3.0]:
    probs = softmax(logits, tau)
    print(f"  τ={tau}: {probs.round(3)}")
# τ=0.5 → [0.93, 0.06, 0.01]  ← sharp/confident
# τ=1.0 → [0.71, 0.24, 0.05]  ← standard
# τ=3.0 → [0.43, 0.35, 0.22]  ← flat/creative


# ─────────────────────────────────────────────────────────────
# 4. CROSS-ENTROPY LOSS — manually, with summation and y_i
# ─────────────────────────────────────────────────────────────
def cross_entropy_full(probs, one_hot_labels):
    """
    Full formula: L = −Σᵢ yᵢ·log(pᵢ)
    Shows the summation and yᵢ explicitly.
    """
    # Each term: y_i * log(p_i). All terms with y_i=0 vanish.
    terms = one_hot_labels * np.log(probs + 1e-10)
    print(f"  Terms (y_i * log(p_i)): {terms.round(3)}")
    print(f"  Non-zero terms: only the true class survives!")
    return -np.sum(terms)

# 3-class: cat (true), dog, car
probs_good = np.array([0.7, 0.2, 0.1])
probs_bad  = np.array([0.1, 0.8, 0.1])
y_true     = np.array([1.0, 0.0, 0.0])  # one-hot: cat is correct

print("\nGood model (cat=0.7):")
loss_good = cross_entropy_full(probs_good, y_true)
print(f"  Loss = {loss_good:.3f}")  # ≈ 0.357

print("\nBad model (cat=0.1):")
loss_bad = cross_entropy_full(probs_bad, y_true)
print(f"  Loss = {loss_bad:.3f}")   # ≈ 2.303


# ─────────────────────────────────────────────────────────────
# 5. GRADIENT = prediction − truth  (the beautiful result)
# ─────────────────────────────────────────────────────────────
def compute_gradient(logits, true_class_idx):
    """
    Gradient of cross-entropy loss wrt logits.
    Result: gradient[k] = p[k] - y[k]  (prediction minus truth)
    """
    probs = softmax(logits)
    y = np.zeros_like(probs)
    y[true_class_idx] = 1.0
    gradient = probs - y          # gradient = p - y
    return gradient, probs

logits_example = np.array([1.2, 0.5, 0.1])  # cat, dog, car
grad, probs = compute_gradient(logits_example, true_class_idx=0)  # cat is true

print("\nGradient computation:")
classes = ["cat (true)", "dog", "car"]
for i, (c, p, g) in enumerate(zip(classes, probs, grad)):
    direction = "push UP ↑" if g < 0 else "push DOWN ↓"
    print(f"  {c:12s}: p={p:.3f}  grad={g:+.3f}  → {direction}")


# ─────────────────────────────────────────────────────────────
# 6. KL DIVERGENCE
# ─────────────────────────────────────────────────────────────
def kl_divergence(P, Q):
    """KL(P || Q). P = true dist, Q = model. Always >= 0."""
    P, Q = np.array(P), np.array(Q) + 1e-10
    return np.sum(P * np.log(P / Q))

true_dist  = [0.7, 0.2, 0.1]
close_pred = [0.6, 0.3, 0.1]  # close to true
far_pred   = [0.1, 0.1, 0.8]  # far from true

print(f"\nKL(true ‖ close): {kl_divergence(true_dist, close_pred):.4f}")
print(f"KL(true ‖ far):   {kl_divergence(true_dist, far_pred):.4f}")


# ─────────────────────────────────────────────────────────────
# 7. VERIFY WITH PYTORCH — our scratch results should match
# ─────────────────────────────────────────────────────────────
logits_t = torch.tensor([[1.2, 0.5, 0.1]])
target_t = torch.tensor([0])  # cat = class 0

pt_loss = F.cross_entropy(logits_t, target_t)
manual  = -np.log(softmax(np.array([1.2, 0.5, 0.1]))[0])

print(f"\nManual loss:  {manual:.4f}")
print(f"PyTorch loss: {pt_loss.item():.4f}")
# Should match exactly — confirming our understanding is correct