┌─────────────────────────────────────────────────────────────┐
│                 MODEL COMPRESSION TECHNIQUES                 │
├─────────────────────────────────────────────────────────────┤
│                                                              │
│  QUANTIZATION         PRUNING            DISTILLATION       │
│  ─────────────        ──────────         ────────────       │
│  FP32 → INT8          Remove weights     Teacher→Student    │
│  2-4x smaller         50-90% sparse      10-100x smaller    │
│  1.5-3x faster        2-4x faster        Same accuracy      │
│                                                              │
│  ARCHITECTURE         LOW-RANK           NEURAL ARCH        │
│  ─────────────        ──────────         ────────────       │
│  MobileNet            Matrix decomp      AutoML search      │
│  EfficientNet         LoRA adapters      Hardware-aware     │
│  Depth-separable      Rank reduction     Latency targets    │
│                                                              │
└─────────────────────────────────────────────────────────────┘

import torch
from torch.quantization import quantize_dynamic, quantize_static

undefined

import torch.quantization as quant

class QuantizedModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.quant = quant.QuantStub()
        self.dequant = quant.DeQuantStub()
        self.layers = nn.Sequential(
            nn.Linear(784, 256),
            nn.ReLU(),
            nn.Linear(256, 10)
        )

    def forward(self, x):
        x = self.quant(x)
        x = self.layers(x)
        x = self.dequant(x)
        return x

undefined

import torch.nn.utils.prune as prune

undefined

def iterative_pruning(model, train_loader, target_sparsity=0.9):
    current_sparsity = 0
    sparsity_schedule = [0.5, 0.75, 0.9]

    for target in sparsity_schedule:
        # Prune
        for name, module in model.named_modules():
            if isinstance(module, nn.Linear):
                prune.l1_unstructured(module, 'weight', amount=target)

        # Fine-tune
        for epoch in range(fine_tune_epochs):
            train_epoch(model, train_loader)

        # Measure sparsity
        total_zeros = sum((p == 0).sum().item() for p in model.parameters())
        total_params = sum(p.numel() for p in model.parameters())
        current_sparsity = total_zeros / total_params
        print(f"Sparsity: {current_sparsity:.2%}")

    return model

class DistillationLoss(nn.Module):
    def __init__(self, temperature=4.0, alpha=0.5):
        super().__init__()
        self.temperature = temperature
        self.alpha = alpha
        self.ce_loss = nn.CrossEntropyLoss()
        self.kl_loss = nn.KLDivLoss(reduction='batchmean')

    def forward(self, student_logits, teacher_logits, labels):
        # Hard label loss
        hard_loss = self.ce_loss(student_logits, labels)

        # Soft label loss (distillation)
        soft_student = F.log_softmax(student_logits / self.temperature, dim=1)
        soft_teacher = F.softmax(teacher_logits / self.temperature, dim=1)
        soft_loss = self.kl_loss(soft_student, soft_teacher) * (self.temperature ** 2)

        return self.alpha * hard_loss + (1 - self.alpha) * soft_loss

undefined

class DepthSeparableConv(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size=3):
        super().__init__()
        self.depthwise = nn.Conv2d(
            in_channels, in_channels, kernel_size,
            padding=kernel_size//2, groups=in_channels
        )
        self.pointwise = nn.Conv2d(in_channels, out_channels, 1)

    def forward(self, x):
        x = self.depthwise(x)
        x = self.pointwise(x)
        return x

undefined

class InvertedResidual(nn.Module):
    def __init__(self, in_ch, out_ch, stride, expand_ratio):
        super().__init__()
        hidden_dim = in_ch * expand_ratio
        self.use_residual = stride == 1 and in_ch == out_ch

        self.conv = nn.Sequential(
            # Expand
            nn.Conv2d(in_ch, hidden_dim, 1, bias=False),
            nn.BatchNorm2d(hidden_dim),
            nn.ReLU6(inplace=True),
            # Depthwise
            nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False),
            nn.BatchNorm2d(hidden_dim),
            nn.ReLU6(inplace=True),
            # Project
            nn.Conv2d(hidden_dim, out_ch, 1, bias=False),
            nn.BatchNorm2d(out_ch),
        )

    def forward(self, x):
        if self.use_residual:
            return x + self.conv(x)
        return self.conv(x)

import torch.nn.utils.parametrize as parametrize

class LowRankLinear(nn.Module):
    def __init__(self, in_features, out_features, rank):
        super().__init__()
        self.A = nn.Linear(in_features, rank, bias=False)
        self.B = nn.Linear(rank, out_features, bias=True)

    def forward(self, x):
        return self.B(self.A(x))

    nn.init.kaiming_uniform_(self.lora_A.weight)
    nn.init.zeros_(self.lora_B.weight)

def forward(self, x):
    return self.original(x) + self.scaling * self.lora_B(self.lora_A(x))

undefined

def measure_efficiency(model, input_shape, device='cuda'):
    import time

    model = model.to(device)
    model.eval()

    # Model size
    param_size = sum(p.numel() * p.element_size() for p in model.parameters())
    buffer_size = sum(b.numel() * b.element_size() for b in model.buffers())
    size_mb = (param_size + buffer_size) / 1024 / 1024

    # FLOPs (using thop)
    from thop import profile
    dummy_input = torch.randn(1, *input_shape).to(device)
    flops, params = profile(model, inputs=(dummy_input,))

    # Latency
    warmup = 10
    iterations = 100

    for _ in range(warmup):
        model(dummy_input)

    torch.cuda.synchronize()
    start = time.time()
    for _ in range(iterations):
        model(dummy_input)
    torch.cuda.synchronize()
    latency_ms = (time.time() - start) / iterations * 1000

    return {
        "size_mb": size_mb,
        "params": params,
        "flops": flops,
        "latency_ms": latency_ms,
        "throughput": 1000 / latency_ms
    }