Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions docs/BENCHMARKS.md
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,10 @@ metrics when `enable_checksum=True`.
#### 2.1.4 Quantized Matrix Multiplication (`gemm_quantized`) [experimental]

Evaluates end-to-end quantized matrix multiplication (`bfloat16` or `float32`
inputs `A[M, K]` and `B[K, N]` dynamically quantized row-wise/column-wise to
inputs `A[M, K]` and `B[K, N]` quantized row-wise/column-wise to
`float8_e4m3fn`, `float8_e5m2`, or `int8` inside the timed step before
`lax.dot_general`).
`lax.dot_general`, with dynamic or static scaling and optional output
dequantization).

| Parameter | Type | Default | Description |
| :--- | :--- | :--- | :--- |
Expand All @@ -269,6 +270,8 @@ inputs `A[M, K]` and `B[K, N]` dynamically quantized row-wise/column-wise to
| `in_dtype` | `str` | `"bfloat16"` | Unquantized input operand dtype (e.g., `bfloat16`, `float32`). |
| `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized matmul operand dtype (e.g., `float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. |
| `out_dtype` | `str` | `"bfloat16"` | Output tensor dtype (e.g., `bfloat16`, `float32`). |
| `scaling` | `str` | `"dynamic"` | Scale calibration: `"dynamic"` (per-row/column `amax` computed on the fly) or `"static"` (fixed clip bound `0.5 * Q_MAX`, i.e., compile-time scale `0.5`). |
| `dequantize` | `bool` | `False` | When `True`, multiply the matmul accumulator by the row and column dequantization scales before casting to `out_dtype`. |
| `seed` | `int` | `0` | PRNG seed for initializing input tensors in HBM. |

---
Expand Down
84 changes: 73 additions & 11 deletions src/accelerator_microbenchmarks/benchmarks/matmul.py
Original file line number Diff line number Diff line change
Expand Up @@ -709,24 +709,59 @@ def calculate_metrics(self, times_ms: list[float]) -> dict[str, Any]:
return super(GemmChainedBenchmark, self).calculate_metrics(times_ms)


def _quantize_per_channel_dynamic(
class QuantScaling(constants.ParamEnum):
"""Calibration method used to derive the quantization scaling factors.

Attributes:
DYNAMIC: Scales are computed on the fly from the absolute maximum of each
row or column (`scale = amax / Q_MAX`).
STATIC: Uses a fixed, compile-time clip bound `0.5 * Q_MAX` (`scale = 0.5`)
with no `amax` reduction. A non-unit (`!= 1.0`) power-of-2 scale prevents
XLA from folding away the scale/dequantize multiplications as identity
no-ops while scaling inputs without mantissa rounding loss.
"""

DYNAMIC = "dynamic"
STATIC = "static"


def _quantize_per_channel(
x: jax.Array,
*,
reduce_axis: int,
quant_dtype: jax.typing.DTypeLike,
) -> jax.Array:
"""Dynamically quantizes `x` per channel along `reduce_axis`."""
scaling: QuantScaling,
) -> tuple[jax.Array, jax.Array]:
"""Quantizes `x` per channel along `reduce_axis`."""
q_dtype = jnp.dtype(quant_dtype)
q_max = float(dtypes.get_dtype_max(q_dtype))
x_f32 = x.astype(jnp.float32)
amax = jnp.max(jnp.abs(x_f32), axis=reduce_axis, keepdims=True)
inv_scale_f32 = q_max / jnp.maximum(amax, 1e-7)

if scaling == QuantScaling.STATIC:
scale_shape = list(x.shape)
scale_shape[reduce_axis] = 1
# Use a non-unit power-of-2 bound (`0.5 * q_max` -> `scale = 0.5`) so XLA
# cannot elide `x * inv_scale` or `acc * (sf_a * sf_b)` as multiply-by-1
# no-ops.
static_absmax = 0.5 * q_max
scale_f32 = jnp.full(scale_shape, static_absmax / q_max, dtype=jnp.float32)
inv_scale_f32 = jnp.reciprocal(scale_f32)
elif scaling == QuantScaling.DYNAMIC:
amax = jnp.max(jnp.abs(x_f32), axis=reduce_axis, keepdims=True)
clamped_amax = jnp.maximum(amax, 1e-7)
scale_f32 = clamped_amax / q_max
inv_scale_f32 = q_max / clamped_amax
else:
raise ValueError(
f"Unsupported scaling mode '{scaling}'; expected 'dynamic' or 'static'."
)

q = x_f32 * inv_scale_f32
if jnp.issubdtype(q_dtype, jnp.integer):
q = jnp.clip(jnp.round(q), -q_max, q_max)
else:
q = jnp.clip(q, -q_max, q_max)
return q.astype(q_dtype)
return q.astype(q_dtype), scale_f32


@dataclasses.dataclass
Expand All @@ -748,11 +783,29 @@ class QuantizedGemmParams(MatmulShapeParams):
)
},
)
scaling: QuantScaling = dataclasses.field(
default=QuantScaling.DYNAMIC,
metadata={
"help": (
"Scale calibration: 'dynamic' (amax computed on the fly) or"
" 'static' (fixed clip bound 0.5 * Q_MAX, i.e. scale = 0.5)."
)
},
)
dequantize: bool = dataclasses.field(
default=False,
metadata={
"help": (
"If True, multiply the accumulator by the row and column"
" dequantization scales before casting to out_dtype."
)
},
)


@registry.benchmark_registry.register("gemm_quantized", is_experimental=True)
class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]):
"""[EXPERIMENTAL] GEMM with dynamic per-channel quantization of A and B."""
"""[EXPERIMENTAL] GEMM with per-channel quantization of A and B."""

Config = QuantizedGemmParams # pylint: disable=invalid-name
requires_ici: bool = False
Expand All @@ -763,6 +816,8 @@ class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]):
("m", report.format_str),
("k", report.format_str),
("n", report.format_str),
("scaling", report.format_str),
("dequantize", report.format_str),
("total_flops", report.format_2f),
("wall_clock_p50_ms", report.format_4f),
("wall_clock_tflops_per_chip", report.format_2f),
Expand All @@ -787,32 +842,39 @@ def setup(self):
@jax.jit
def quant_gemm_fn(a, b):
with jax.named_scope(constants.MARKER):
qa = _quantize_per_channel_dynamic(
qa, sf_a = _quantize_per_channel(
a,
reduce_axis=1,
quant_dtype=quant_dtype,
scaling=cfg.scaling,
)
qb = _quantize_per_channel_dynamic(
qb, sf_b = _quantize_per_channel(
b,
reduce_axis=0,
quant_dtype=quant_dtype,
scaling=cfg.scaling,
)
acc = jax.lax.dot_general(
qa,
qb,
dimension_numbers=(((1,), (0,)), ((), ())),
preferred_element_type=dot_acc_dtype,
)
if cfg.dequantize:
acc = acc.astype(jnp.float32) * (sf_a * sf_b)
return acc.astype(out_dtype)

self._jit_fn = quant_gemm_fn

def get_run_identifier(self) -> str:
cfg = self.config
return (
run_id = (
f"m_{cfg.m}_k_{cfg.k}_n_{cfg.n}_"
f"{cfg.in_dtype}_to_{cfg.quant_dtype}_out_{cfg.out_dtype}"
f"{cfg.in_dtype}_to_{cfg.quant_dtype}_{cfg.scaling}_out_{cfg.out_dtype}"
)
if cfg.dequantize:
run_id += "_dequant"
return run_id

def generate_inputs(self) -> tuple[Any, ...]:
cfg = self.config
Expand Down
149 changes: 132 additions & 17 deletions tests/test_matmul.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ def _setup_benchmark(self, **kwargs):
params.update(kwargs)
config = matmul.GemmParams(**params)
self.bm = matmul.GeneralizedGemmBenchmark(
config=config, hardware_spec=system.TPU7X_HARDWARE_SPEC, mesh=self.mock_mesh
config=config,
hardware_spec=system.TPU7X_HARDWARE_SPEC,
mesh=self.mock_mesh,
)
self.bm.setup()

Expand Down Expand Up @@ -203,7 +205,9 @@ def test_is_xprof_op(self):
"""Test that is_xprof_op correctly identifies convolution fusion events."""
config = matmul.GemmParams(m=128, n=128, k=128)
self.bm = matmul.GeneralizedGemmBenchmark(
config=config, hardware_spec=system.TPU7X_HARDWARE_SPEC, mesh=self.mock_mesh
config=config,
hardware_spec=system.TPU7X_HARDWARE_SPEC,
mesh=self.mock_mesh,
)
self.assertTrue(
self.bm.match_xprof_op_fallback(
Expand Down Expand Up @@ -766,6 +770,7 @@ def _setup_benchmark(self, **kwargs):
self.bm.setup()

def test_benchmark_registered(self):
"""Tests that gemm_quantized is registered in the benchmark registry."""
bm_class = registry.benchmark_registry.get_benchmark("gemm_quantized")
self.assertEqual(bm_class, matmul.QuantizedGemmBenchmark)

Expand All @@ -775,7 +780,7 @@ def test_generate_inputs_and_run_op(self, qd):
self._setup_benchmark(quant_dtype=qd)
self.assertEqual(
self.bm.get_run_identifier(),
f"m_64_k_128_n_64_bfloat16_to_{qd}_out_bfloat16",
f"m_64_k_128_n_64_bfloat16_to_{qd}_dynamic_out_bfloat16",
)
inputs = self.bm.generate_inputs()
self.assertLen(inputs, 2)
Expand All @@ -785,26 +790,134 @@ def test_generate_inputs_and_run_op(self, qd):
self.assertEqual(out.dtype, jnp.bfloat16)

quantize_fn = jax.jit(
matmul._quantize_per_channel_dynamic, # pylint: disable=protected-access
static_argnames=("reduce_axis", "quant_dtype"),
matmul._quantize_per_channel, # pylint: disable=protected-access
static_argnames=("reduce_axis", "quant_dtype", "scaling"),
)
qa, _ = quantize_fn(
a,
reduce_axis=1,
quant_dtype=qd,
scaling=matmul.QuantScaling.DYNAMIC,
)
qb, _ = quantize_fn(
b,
reduce_axis=0,
quant_dtype=qd,
scaling=matmul.QuantScaling.DYNAMIC,
)
qa = quantize_fn(a, reduce_axis=1, quant_dtype=qd)
qb = quantize_fn(b, reduce_axis=0, quant_dtype=qd)
expected = jnp.matmul(qa.astype(jnp.float32), qb.astype(jnp.float32))
np.testing.assert_allclose(out.astype(jnp.float32), expected, rtol=2e-2)

def test_quantize_per_channel_dynamic_and_dtype_constraints(self):
"""Tests int8 rounding and DTYPE_CONSTRAINTS validation."""
x = jnp.array([[127.0, 0.6], [63.5, -63.5]], dtype=jnp.bfloat16)
q_int8 = (
matmul._quantize_per_channel_dynamic( # pylint: disable=protected-access
x, reduce_axis=1, quant_dtype=jnp.int8
)
@parameterized.parameters(
(
"dynamic",
False,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_dynamic_out_bfloat16",
25690112.0,
),
(
"static",
False,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_static_out_bfloat16",
3211264.0,
),
(
"dynamic",
True,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_dynamic_out_bfloat16_dequant",
802816.0,
),
(
"static",
True,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_static_out_bfloat16_dequant",
802816.0,
),
)
def test_scaling_and_dequantize_variants(
self, scaling, dequantize, expected_run_id, expected_val
):
"""Tests run IDs and numerical outputs across scaling and dequantize modes."""
self._setup_benchmark(scaling=scaling, dequantize=dequantize)
self.assertEqual(self.bm.get_run_identifier(), expected_run_id)
inputs = self.bm.generate_inputs()
out = self.bm.run_op(*inputs)
self.assertEqual(out.shape, (64, 64))
self.assertEqual(out.dtype, jnp.bfloat16)
self.assertTrue(bool(jnp.all(jnp.isfinite(out))))

# Verify exact numerical quantization and dequantization with dyadic inputs:
# A = 112.0 (dynamic sf_a = 0.25, qa = 448; static sf_a = 0.5, qa = 224)
# B = 56.0 (dynamic sf_b = 0.125, qb = 448; static sf_b = 0.5, qb = 112)
a = jnp.full((64, 128), 112.0, dtype=jnp.bfloat16)
b = jnp.full((128, 64), 56.0, dtype=jnp.bfloat16)
exact_out = self.bm.run_op(a, b)
np.testing.assert_allclose(
np.asarray(exact_out, dtype=np.float32),
np.full((64, 64), expected_val, dtype=np.float32),
)
self.assertEqual(q_int8.dtype, jnp.int8)
np.testing.assert_array_equal(
np.asarray(q_int8), np.array([[127, 1], [127, -127]], dtype=np.int8)

def test_int8_quant_dtype_rounds_and_accumulates_int32(self):
"""Tests that int8 quantization rounds to nearest integer and uses int32."""
self._setup_benchmark(
m=2,
k=2,
n=2,
quant_dtype="int8",
scaling="dynamic",
dequantize=True,
)
# Row 0 of A has amax = 127.0 -> scale = 1.0; 0.6 rounds to 1 (not 0).
a = jnp.array([[127.0, 0.6], [63.5, -63.5]], dtype=jnp.bfloat16)
b = jnp.array([[127.0, 63.5], [127.0, -63.5]], dtype=jnp.bfloat16)
out = self.bm.run_op(a, b)
self.assertEqual(out.shape, (2, 2))
self.assertEqual(out.dtype, jnp.bfloat16)
# Row 0 dot Col 0: q_a=[127, 1], q_b=[127, 127], sf_a=1.0, sf_b=1.0
# -> (127*127 + 1*127) * 1.0 = 16256.0 (would be 16129.0 without round).
self.assertEqual(float(out[0, 0]), 16256.0)

def test_static_scaling_clips_outliers_to_qmax(self):
"""Tests that static scaling clips values exceeding 0.5 * q_max to q_max."""
self._setup_benchmark(
m=2,
k=2,
n=2,
quant_dtype="float8_e4m3fn",
scaling="static",
dequantize=False,
)
# Values > 224.0 exceed q_max (448.0) after dividing by static scale 0.5;
# without clipping to [-q_max, q_max], casting 1000.0 to float8_e4m3fn
# overflows to NaN.
a = jnp.array([[500.0, -500.0], [112.0, 0.0]], dtype=jnp.bfloat16)
b = jnp.array([[1.0, 0.0], [0.0, 1.0]], dtype=jnp.bfloat16)
out = self.bm.run_op(a, b)
self.assertTrue(bool(jnp.all(jnp.isfinite(out))))
# q_a[0] clips to [448.0, -448.0] and q_b is [[2.0, 0.0], [0.0, 2.0]].
np.testing.assert_allclose(
np.asarray(out, dtype=np.float32),
np.array([[896.0, -896.0], [448.0, 0.0]], dtype=np.float32),
)
qa, sf_a = matmul._quantize_per_channel( # pylint: disable=protected-access
a,
reduce_axis=1,
quant_dtype="float8_e4m3fn",
scaling=matmul.QuantScaling.STATIC,
)
np.testing.assert_allclose(
np.asarray(qa, dtype=np.float32),
np.array([[448.0, -448.0], [224.0, 0.0]], dtype=np.float32),
)
np.testing.assert_allclose(
np.asarray(sf_a, dtype=np.float32),
np.array([[0.5], [0.5]], dtype=np.float32),
)

def test_params_validation(self):
"""Tests that invalid scaling and quant_dtype values raise ValueError."""
with self.assertRaisesRegex(ValueError, "Invalid scaling"):
matmul.QuantizedGemmParams(**{"scaling": "invalid"})
with self.assertRaises(ValueError):
matmul.QuantizedGemmParams(quant_dtype="uint8")
with self.assertRaises(ValueError):
Expand All @@ -819,6 +932,7 @@ def test_quantize_per_channel_dynamic_and_dtype_constraints(self):
)

def test_flops_bytes_and_compute_dtype(self):
"""Tests FLOPs, bytes, and compute dtype reporting."""
self._setup_benchmark(m=64, k=128, n=64)
self.assertEqual(self.bm.get_compute_dtype(), "float8_e4m3fn")
self.assertEqual(self.bm.get_total_flops(), float(2 * 64 * 128 * 64))
Expand All @@ -828,6 +942,7 @@ def test_flops_bytes_and_compute_dtype(self):
self.assertEqual(self.bm.get_total_bytes(), expected_bytes)

def test_schema_coverage(self):
"""Tests that benchmark output matches the report schema."""
self._setup_benchmark()
test_report_utils.assert_schema_matches_output(self, self.bm)

Expand Down
Loading