Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions docs/BENCHMARKS.md
Original file line number Diff line number Diff line change
Expand Up @@ -272,17 +272,18 @@ metrics when `enable_checksum=True`.

Evaluates end-to-end quantized matrix multiplication (`bfloat16` or `float32`
inputs `A[M, K]` and `B[K, N]` quantized row-wise/column-wise to
`float8_e4m3fn`, `float8_e5m2`, or `int8` inside the timed step before
`lax.dot_general`, with dynamic or static scaling and optional output
dequantization).
`float8_e4m3fn`, `float8_e5m2`, or `int8`—or mixed-precision `FP8 × INT4` with
`rhs_quant_dtype="int4"`—inside the timed step before `lax.dot_general`, with
dynamic or static scaling and optional output dequantization).

| Parameter | Type | Default | Description |
| :--- | :--- | :--- | :--- |
| `m` | `int` | `1024` | Matrix dimension `M` (rows of `A` and output matrix `C`). |
| `k` | `int` | `1024` | Contracting dimension `K` (columns of `A` and rows of `B`). |
| `n` | `int` | `1024` | Matrix dimension `N` (columns of `B` and output matrix `C`). |
| `in_dtype` | `str` | `"bfloat16"` | Unquantized input operand dtype (e.g., `bfloat16`, `float32`). |
| `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized matmul operand dtype (e.g., `float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. |
| `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized LHS (and default RHS) operand dtype (`float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. |
| `rhs_quant_dtype` | `str` | `` `quant_dtype` `` | Quantized RHS (`B`) operand dtype. Defaults to `quant_dtype`; also supports `"int4"` when `quant_dtype` is `float8_e4m3fn` or `float8_e5m2` (`W4A8`). |
| `out_dtype` | `str` | `"bfloat16"` | Output tensor dtype (e.g., `bfloat16`, `float32`). |
| `scaling` | `str` | `"dynamic"` | Scale calibration: `"dynamic"` (per-row/column `amax` computed on the fly) or `"static"` (fixed clip bound `0.5 * Q_MAX`, i.e., compile-time scale `0.5`). |
| `dequantize` | `bool` | `False` | When `True`, multiply the matmul accumulator by the row and column dequantization scales before casting to `out_dtype`. |
Expand Down
72 changes: 65 additions & 7 deletions src/accelerator_microbenchmarks/benchmarks/matmul.py
Original file line number Diff line number Diff line change
Expand Up @@ -771,15 +771,31 @@ class QuantizedGemmParams(MatmulShapeParams):
DTYPE_CONSTRAINTS = {
"in_dtype": _UNQUANTIZED_FLOAT_DTYPES,
"quant_dtype": (jnp.float8_e4m3fn, jnp.float8_e5m2, jnp.int8),
"rhs_quant_dtype": (
jnp.float8_e4m3fn,
jnp.float8_e5m2,
jnp.int8,
jnp.int4,
),
"out_dtype": _UNQUANTIZED_FLOAT_DTYPES,
}

quant_dtype: dtypes.DType = base.dtype_field(
default=dtypes.DType.FLOAT8_E4M3FN,
metadata={
"help": (
"Quantized operand data type fed to the MXU (e.g."
" 'float8_e4m3fn', 'int8')."
"Quantized LHS (and default RHS) operand data type fed to the"
" MXU (e.g. 'float8_e4m3fn', 'int8')."
)
},
)
rhs_quant_dtype: dtypes.DType = base.dtype_field(
default=dtypes.DType.FLOAT8_E4M3FN,
metadata={
"help": (
"Quantized RHS (B) operand data type. Defaults to `quant_dtype`;"
" also supports 'int4' when `quant_dtype` is FP8"
" ('float8_e4m3fn' or 'float8_e5m2')."
)
},
)
Expand All @@ -802,6 +818,40 @@ class QuantizedGemmParams(MatmulShapeParams):
},
)

def __post_init__(self):
if "rhs_quant_dtype" not in self._explicit_fields:
self.rhs_quant_dtype = self.quant_dtype
super().__post_init__()
if self.rhs_quant_dtype != self.quant_dtype and not (
self.rhs_quant_dtype == dtypes.DType.INT4
and self.quant_dtype in (
dtypes.DType.FLOAT8_E4M3FN,
dtypes.DType.FLOAT8_E5M2,
)
):
raise ValueError(
"rhs_quant_dtype must match quant_dtype or be 'int4' paired with an"
" FP8 quant_dtype ('float8_e4m3fn' or 'float8_e5m2'), got"
f" quant_dtype='{self.quant_dtype}',"
f" rhs_quant_dtype='{self.rhs_quant_dtype}'."
)

def with_generation_defaults(
self, param_defaults: dict[str, Any]
) -> base.BaseBenchmarkParams:
if (
param_defaults
and "quant_dtype" in param_defaults
and "quant_dtype" not in self._explicit_fields
and "rhs_quant_dtype" not in param_defaults
and "rhs_quant_dtype" not in self._explicit_fields
):
param_defaults = {
**param_defaults,
"rhs_quant_dtype": param_defaults["quant_dtype"],
}
return super().with_generation_defaults(param_defaults)


@registry.benchmark_registry.register("gemm_quantized", is_experimental=True)
class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]):
Expand All @@ -812,6 +862,7 @@ class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]):
REPORT_SCHEMA: Sequence[tuple[str, Callable[[Any], str]]] = (
("in_dtype", report.format_str),
("quant_dtype", report.format_str),
("rhs_quant_dtype", report.format_str),
("out_dtype", report.format_str),
("m", report.format_str),
("k", report.format_str),
Expand All @@ -826,18 +877,20 @@ class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]):
)

def get_compute_dtype(self) -> str:
"""Returns the quantized dtype, which determines the MXU peak TFLOPS."""
"""Returns the LHS quantized dtype, which determines the MXU peak TFLOPS."""
return str(self.config.quant_dtype)

def setup(self):
if self.mesh is None:
raise ValueError("Mesh not initialized.")
cfg = self.config
quant_dtype = dtypes.parse_dtype(cfg.quant_dtype)
rhs_quant_dtype = dtypes.parse_dtype(cfg.rhs_quant_dtype)
out_dtype = dtypes.parse_dtype(cfg.out_dtype)
dot_acc_dtype = (
jnp.int32 if jnp.issubdtype(quant_dtype, jnp.integer) else jnp.float32
both_int = jnp.issubdtype(quant_dtype, jnp.integer) and jnp.issubdtype(
rhs_quant_dtype, jnp.integer
)
dot_acc_dtype = jnp.int32 if both_int else jnp.float32

@jax.jit
def quant_gemm_fn(a, b):
Expand All @@ -851,7 +904,7 @@ def quant_gemm_fn(a, b):
qb, sf_b = _quantize_per_channel(
b,
reduce_axis=0,
quant_dtype=quant_dtype,
quant_dtype=rhs_quant_dtype,
scaling=cfg.scaling,
)
acc = jax.lax.dot_general(
Expand All @@ -868,9 +921,14 @@ def quant_gemm_fn(a, b):

def get_run_identifier(self) -> str:
cfg = self.config
quant_str = (
f"{cfg.quant_dtype}"
if cfg.rhs_quant_dtype == cfg.quant_dtype
else f"{cfg.quant_dtype}_rhs_{cfg.rhs_quant_dtype}"
)
run_id = (
f"m_{cfg.m}_k_{cfg.k}_n_{cfg.n}_"
f"{cfg.in_dtype}_to_{cfg.quant_dtype}_{cfg.scaling}_out_{cfg.out_dtype}"
f"{cfg.in_dtype}_to_{quant_str}_{cfg.scaling}_out_{cfg.out_dtype}"
)
if cfg.dequantize:
run_id += "_dequant"
Expand Down
92 changes: 85 additions & 7 deletions tests/test_matmul.py
Original file line number Diff line number Diff line change
Expand Up @@ -774,14 +774,22 @@ def test_benchmark_registered(self):
bm_class = registry.benchmark_registry.get_benchmark("gemm_quantized")
self.assertEqual(bm_class, matmul.QuantizedGemmBenchmark)

@parameterized.parameters("float8_e4m3fn", "int8")
def test_generate_inputs_and_run_op(self, qd):
@parameterized.parameters(
("float8_e4m3fn", "float8_e4m3fn"),
("float8_e5m2", "float8_e5m2"),
("int8", "int8"),
("float8_e4m3fn", "int4"),
("float8_e5m2", "int4"),
)
def test_generate_inputs_and_run_op(self, qd, rhs_qd):
"""Tests input generation and numerical output against quantized matmul."""
self._setup_benchmark(quant_dtype=qd)
self._setup_benchmark(quant_dtype=qd, rhs_quant_dtype=rhs_qd)
expected_quant_str = qd if rhs_qd == qd else f"{qd}_rhs_{rhs_qd}"
self.assertEqual(
self.bm.get_run_identifier(),
f"m_64_k_128_n_64_bfloat16_to_{qd}_dynamic_out_bfloat16",
f"m_64_k_128_n_64_bfloat16_to_{expected_quant_str}_dynamic_out_bfloat16",
)
self.assertEqual(self.bm.get_compute_dtype(), qd)
inputs = self.bm.generate_inputs()
self.assertLen(inputs, 2)
a, b = inputs
Expand All @@ -802,43 +810,63 @@ def test_generate_inputs_and_run_op(self, qd):
qb, _ = quantize_fn(
b,
reduce_axis=0,
quant_dtype=qd,
quant_dtype=rhs_qd,
scaling=matmul.QuantScaling.DYNAMIC,
)
expected = jnp.matmul(qa.astype(jnp.float32), qb.astype(jnp.float32))
np.testing.assert_allclose(out.astype(jnp.float32), expected, rtol=2e-2)

@parameterized.parameters(
(
"float8_e4m3fn",
"dynamic",
False,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_dynamic_out_bfloat16",
25690112.0,
),
(
"float8_e4m3fn",
"static",
False,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_static_out_bfloat16",
3211264.0,
),
(
"float8_e4m3fn",
"dynamic",
True,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_dynamic_out_bfloat16_dequant",
802816.0,
),
(
"float8_e4m3fn",
"static",
True,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_static_out_bfloat16_dequant",
802816.0,
),
(
"int4",
"dynamic",
False,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_rhs_int4_dynamic_out_bfloat16",
401408.0,
),
(
"int4",
"dynamic",
True,
"m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_rhs_int4_dynamic_out_bfloat16_dequant",
802816.0,
),
)
def test_scaling_and_dequantize_variants(
self, scaling, dequantize, expected_run_id, expected_val
self, rhs_qd, scaling, dequantize, expected_run_id, expected_val
):
"""Tests run IDs and numerical outputs across scaling and dequantize modes."""
self._setup_benchmark(scaling=scaling, dequantize=dequantize)
self._setup_benchmark(
rhs_quant_dtype=rhs_qd, scaling=scaling, dequantize=dequantize
)
self.assertEqual(self.bm.get_run_identifier(), expected_run_id)
inputs = self.bm.generate_inputs()
out = self.bm.run_op(*inputs)
Expand Down Expand Up @@ -876,6 +904,22 @@ def test_int8_quant_dtype_rounds_and_accumulates_int32(self):
# Row 0 dot Col 0: q_a=[127, 1], q_b=[127, 127], sf_a=1.0, sf_b=1.0
# -> (127*127 + 1*127) * 1.0 = 16256.0 (would be 16129.0 without round).
self.assertEqual(float(out[0, 0]), 16256.0)
x_int4 = jnp.array([[7.0, 3.6], [3.5, -3.5]], dtype=jnp.bfloat16)
q_int4, sf_int4 = matmul._quantize_per_channel( # pylint: disable=protected-access
x_int4,
reduce_axis=1,
quant_dtype=jnp.int4,
scaling=matmul.QuantScaling.DYNAMIC,
)
self.assertEqual(q_int4.dtype, jnp.int4)
np.testing.assert_array_equal(
np.asarray(q_int4.astype(jnp.int8)),
np.array([[7, 4], [7, -7]], dtype=np.int8),
)
np.testing.assert_allclose(
np.asarray(sf_int4, dtype=np.float32),
np.array([[1.0], [0.5]], dtype=np.float32),
)

def test_static_scaling_clips_outliers_to_qmax(self):
"""Tests that static scaling clips values exceeding 0.5 * q_max to q_max."""
Expand Down Expand Up @@ -916,12 +960,46 @@ def test_static_scaling_clips_outliers_to_qmax(self):

def test_params_validation(self):
"""Tests that invalid scaling and quant_dtype values raise ValueError."""
# Default rhs_quant_dtype tracks quant_dtype when omitted, including across
# with_generation_defaults().
self.assertEqual(
matmul.QuantizedGemmParams(quant_dtype="int8").rhs_quant_dtype,
"int8",
)
self.assertEqual(
matmul.QuantizedGemmParams(quant_dtype="float8_e5m2").rhs_quant_dtype,
"float8_e5m2",
)
default_with_int8 = matmul.QuantizedGemmParams().with_generation_defaults(
{"quant_dtype": "int8"}
)
self.assertEqual(default_with_int8.quant_dtype, "int8")
self.assertEqual(default_with_int8.rhs_quant_dtype, "int8")
explicit_int4_with_e5m2 = matmul.QuantizedGemmParams(
rhs_quant_dtype="int4"
).with_generation_defaults({"quant_dtype": "float8_e5m2"})
self.assertEqual(explicit_int4_with_e5m2.quant_dtype, "float8_e5m2")
self.assertEqual(explicit_int4_with_e5m2.rhs_quant_dtype, "int4")
with self.assertRaisesRegex(ValueError, "Invalid scaling"):
matmul.QuantizedGemmParams(**{"scaling": "invalid"})
with self.assertRaises(ValueError):
matmul.QuantizedGemmParams(quant_dtype="uint8")
with self.assertRaises(ValueError):
matmul.QuantizedGemmParams(quant_dtype="bfloat16")
with self.assertRaises(ValueError):
matmul.QuantizedGemmParams(rhs_quant_dtype="bfloat16")
with self.assertRaises(ValueError):
matmul.QuantizedGemmParams(**{"rhs_quant_dtype": None})
with self.assertRaisesRegex(
ValueError, "rhs_quant_dtype must match quant_dtype or be 'int4'"
):
matmul.QuantizedGemmParams(quant_dtype="int8", rhs_quant_dtype="int4")
with self.assertRaisesRegex(
ValueError, "rhs_quant_dtype must match quant_dtype or be 'int4'"
):
matmul.QuantizedGemmParams(
quant_dtype="float8_e4m3fn", rhs_quant_dtype="int8"
)
with self.assertRaises(ValueError):
matmul.QuantizedGemmParams(in_dtype="float64")
with self.assertRaises(ValueError):
Expand Down
Loading