PyTorch Dynamic Quantization

Real-World Scenario

A team ships a fraud-scoring model that runs on every checkout request. The model file is 30 MB, most of it in the Linear layers doing the actual scoring. Dynamic quantization converts those weights to int8 after training — no retraining, no new dataset — and the question everyone assumes the answer to before measuring is “does that also make it faster?”

Quantizing a Model

import torch
import torch.nn as nn

torch.manual_seed(0)

class FraudDetector(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(1000, 2000), nn.ReLU(),
            nn.Linear(2000, 2000), nn.ReLU(),
            nn.Linear(2000, 1000), nn.ReLU(),
            nn.Linear(1000, 1), nn.Sigmoid(),
        )

    def forward(self, x):
        return self.net(x)

model = FraudDetector()
model.eval()   # required — quantize_dynamic assumes inference mode

quantized = torch.quantization.quantize_dynamic(
    model, {nn.Linear}, dtype=torch.qint8
)

print(quantized.net[0])

$ python quantize_basic.py
DynamicQuantizedLinear(in_features=1000, out_features=2000, dtype=torch.qint8, qscheme=torch.per_tensor_affine)

quantize_dynamic walks the model, replaces every nn.Linear with a DynamicQuantizedLinear, and leaves everything else (here, the ReLU/Sigmoid activations) untouched. {nn.Linear} is the set of layer types to convert — swap in {nn.LSTM, nn.GRU} too if the model has recurrent layers; convolutions aren’t supported by this quantization mode.

Model Size: A Real, Reliable Win

import torch
import torch.nn as nn
import os

torch.manual_seed(0)

class FraudDetector(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(1000, 2000), nn.ReLU(),
            nn.Linear(2000, 2000), nn.ReLU(),
            nn.Linear(2000, 1000), nn.ReLU(),
            nn.Linear(1000, 1), nn.Sigmoid(),
        )

    def forward(self, x):
        return self.net(x)

model = FraudDetector()
model.eval()
quantized = torch.quantization.quantize_dynamic(model, {nn.Linear}, dtype=torch.qint8)

torch.save(model.state_dict(), "fp32_model.pt")
torch.save(quantized.state_dict(), "int8_model.pt")

fp32_size = os.path.getsize("fp32_model.pt") / 1024 / 1024
int8_size = os.path.getsize("int8_model.pt") / 1024 / 1024

print(f"FP32 model size: {fp32_size:.2f} MB")
print(f"INT8 model size: {int8_size:.2f} MB")
print(f"Size reduction:  {fp32_size / int8_size:.2f}x")

$ python quantize_size.py
FP32 model size: 30.54 MB
INT8 model size: 7.66 MB
Size reduction:  3.99x

Every weight goes from 4 bytes (float32) to 1 byte (int8) — a ~4x reduction that holds regardless of model architecture or batch size, because it’s purely a property of the stored weights, not of how the model is run.

Latency: Not Automatic

import torch
import torch.nn as nn
import time
import statistics

torch.manual_seed(0)

class FraudDetector(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(1000, 2000), nn.ReLU(),
            nn.Linear(2000, 2000), nn.ReLU(),
            nn.Linear(2000, 1000), nn.ReLU(),
            nn.Linear(1000, 1), nn.Sigmoid(),
        )

    def forward(self, x):
        return self.net(x)

model = FraudDetector()
model.eval()
quantized = torch.quantization.quantize_dynamic(model, {nn.Linear}, dtype=torch.qint8)

x = torch.randn(1, 1000)

def time_calls(fn, n_calls=100, n_trials=15):
    """Return per-call latency (ms) for each of several independent trials."""
    times = []
    with torch.no_grad():
        for _ in range(20):
            fn(x)   # warm up
        for _ in range(n_trials):
            start = time.perf_counter()
            for _ in range(n_calls):
                fn(x)
            times.append((time.perf_counter() - start) / n_calls * 1000)
    return times

fp32_times = time_calls(model)
int8_times = time_calls(quantized)
fp32_median = statistics.median(fp32_times)
int8_median = statistics.median(int8_times)

print(f"FP32 median: {fp32_median:.4f} ms/call  (min {min(fp32_times):.4f}, max {max(fp32_times):.4f})")
print(f"INT8 median: {int8_median:.4f} ms/call  (min {min(int8_times):.4f}, max {max(int8_times):.4f})")
print(f"Speedup (median): {fp32_median / int8_median:.2f}x")

$ python quantize_latency.py
FP32 median: 4.3076 ms/call  (min 2.8690, max 6.4212)
INT8 median: 3.5220 ms/call  (min 2.8059, max 5.1699)
Speedup (median): 1.22x

A single warm-up-then-measure-once pass (the obvious way to write this) is not reliable — on this same machine, one such single-pass run gave a 1.58x speedup and another gave a 0.97x speedup (quantization measuring slightly slower), purely from other processes competing for CPU between runs. Running 15 independent trials and comparing medians, as above, is what actually holds up: the FP32 and INT8 ranges above overlap at the edges, but the medians consistently favor INT8 across repeated runs of this exact script. Treat a single latency number from any quantization benchmark — this one included — with suspicion; ask how many trials it’s a median of. This is on top of the 4x size reduction from the previous section, which doesn’t have this noise problem because it isn’t a timing measurement. The latency win itself also depends on the model being large enough for cheaper int8 matrix multiplies to outweigh the quantize/dequantize overhead added on every call — rerun the same methodology on a much smaller model and even the median flips:

import torch
import torch.nn as nn
import time
import statistics

torch.manual_seed(0)

class TinyDetector(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(20, 256), nn.ReLU(),
            nn.Linear(256, 256), nn.ReLU(),
            nn.Linear(256, 64), nn.ReLU(),
            nn.Linear(64, 1), nn.Sigmoid(),
        )

    def forward(self, x):
        return self.net(x)

tiny = TinyDetector()
tiny.eval()
tiny_quantized = torch.quantization.quantize_dynamic(tiny, {nn.Linear}, dtype=torch.qint8)

x = torch.randn(1, 20)

def time_calls(fn, n_calls=500, n_trials=15):
    times = []
    with torch.no_grad():
        for _ in range(20):
            fn(x)
        for _ in range(n_trials):
            start = time.perf_counter()
            for _ in range(n_calls):
                fn(x)
            times.append((time.perf_counter() - start) / n_calls * 1000)
    return times

fp32_times = time_calls(tiny)
int8_times = time_calls(tiny_quantized)

print(f"FP32 median: {statistics.median(fp32_times):.4f} ms/call")
print(f"INT8 median: {statistics.median(int8_times):.4f} ms/call")

$ python quantize_tiny.py
FP32 median: 0.9024 ms/call
INT8 median: 2.9116 ms/call

On this much smaller model (4 Linear layers, largest is 256 units), the quantized version is slower — over 3x slower by median, and this direction held across every repeat while writing this lesson, unlike the large model’s noisier result above. The per-call cost of quantizing the activation tensor and dequantizing the int8 output back to float32 outweighs any savings from cheaper int8 arithmetic, because there simply isn’t much matrix-multiply work to make cheaper. Size reduction is a property of the weights; the latency win only appears once the compute is large enough for it to matter, and even then, measure it properly (multiple trials, not one run) before trusting the number.

Practice

import torch
import torch.nn as nn

class SmallCNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv = nn.Conv2d(3, 16, kernel_size=3)
        self.fc = nn.Linear(16 * 30 * 30, 10)

    def forward(self, x):
        x = self.conv(x)
        return self.fc(x.flatten(1))

model = SmallCNN()
model.eval()
quantized = torch.quantization.quantize_dynamic(model, {nn.Linear}, dtype=torch.qint8)

print(type(quantized.conv).__name__)
print(quantized.fc)

Conv2d
DynamicQuantizedLinear(in_features=14400, out_features=10, dtype=torch.qint8, qscheme=torch.per_tensor_affine)

import torch
import torch.nn as nn

torch.manual_seed(0)

class FraudDetector(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(1000, 2000), nn.ReLU(),
            nn.Linear(2000, 2000), nn.ReLU(),
            nn.Linear(2000, 1000), nn.ReLU(),
            nn.Linear(1000, 1), nn.Sigmoid(),
        )

    def forward(self, x):
        return self.net(x)

model = FraudDetector()
model.eval()
quantized = torch.quantization.quantize_dynamic(model, {nn.Linear}, dtype=torch.qint8)

x = torch.randn(10, 1000)
with torch.no_grad():
    fp32_out = model(x)
    int8_out = quantized(x)

print(f"Max absolute difference: {(fp32_out - int8_out).abs().max().item():.6f}")

Max absolute difference: 0.000294

import torch
import torch.nn as nn
import os

class Tiny(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(10, 10), nn.Linear(10, 1))

    def forward(self, x):
        return self.net(x)

model = Tiny()
model.eval()
quantized = torch.quantization.quantize_dynamic(model, {nn.Linear}, dtype=torch.qint8)

torch.save(model.state_dict(), "tiny_fp32.pt")
torch.save(quantized.state_dict(), "tiny_int8.pt")

fp32_size = os.path.getsize("tiny_fp32.pt")
int8_size = os.path.getsize("tiny_int8.pt")
print(f"FP32: {fp32_size} bytes")
print(f"INT8: {int8_size} bytes")

FP32: 2865 bytes
INT8: 3945 bytes

import torch

print(f"Active quantized engine: {torch.backends.quantized.engine}")
print(f"Supported engines:       {torch.backends.quantized.supported_engines}")

Active quantized engine: onednn
Supported engines:       ['onednn']