diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md
index ed4f9284..6299b30b 100644
--- a/docs/BENCHMARKS.md
+++ b/docs/BENCHMARKS.md
@@ -18,6 +18,7 @@ This guide provides the reference for **TPUMS (TPU Microbenchmark Suite)** param
- [2.1.1 GEMM Throttling (`gemm_throttling`) [experimental]](#211-gemm-throttling-gemm_throttling-experimental)
- [2.1.2 Chained Matrix Multiplication & Determinism Check (`gemm_chained`) [experimental]](#212-chained-matrix-multiplication--determinism-check-gemm_chained-experimental)
- [2.1.3 GEMM Pulse (`gemm_pulse`) [experimental]](#213-gemm-pulse-gemm_pulse-experimental)
+ - [2.1.4 Quantized Matrix Multiplication (`gemm_quantized`) [experimental]](#214-quantized-matrix-multiplication-gemm_quantized-experimental)
- [2.2 HBM Memory Bandwidth (`hbm`)](#22-hbm-memory-bandwidth-hbm)
- [2.3 Host I/O Bandwidth (`host_to_device`, `device_to_host`)](#23-host-io-bandwidth-host_to_device-device_to_host)
- [2.4 Device-to-Device Transfers (`device_to_device`)](#24-inter-chip-interconnect-device_to_device)
@@ -252,6 +253,24 @@ final cycle ends with an `idle_s` pause followed by traced
`gemm_throttling` soaking and thermal metrics, plus all `gemm_chained` checksum
metrics when `enable_checksum=True`.
+
+#### 2.1.4 Quantized Matrix Multiplication (`gemm_quantized`) [experimental]
+
+Evaluates end-to-end quantized matrix multiplication (`bfloat16` or `float32`
+inputs `A[M, K]` and `B[K, N]` dynamically quantized row-wise/column-wise to
+`float8_e4m3fn`, `float8_e5m2`, or `int8` inside the timed step before
+`lax.dot_general`).
+
+| Parameter | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| `m` | `int` | `1024` | Matrix dimension `M` (rows of `A` and output matrix `C`). |
+| `k` | `int` | `1024` | Contracting dimension `K` (columns of `A` and rows of `B`). |
+| `n` | `int` | `1024` | Matrix dimension `N` (columns of `B` and output matrix `C`). |
+| `in_dtype` | `str` | `"bfloat16"` | Unquantized input operand dtype (e.g., `bfloat16`, `float32`). |
+| `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized matmul operand dtype (e.g., `float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. |
+| `out_dtype` | `str` | `"bfloat16"` | Output tensor dtype (e.g., `bfloat16`, `float32`). |
+| `seed` | `int` | `0` | PRNG seed for initializing input tensors in HBM. |
+
---
diff --git a/src/accelerator_microbenchmarks/benchmarks/matmul.py b/src/accelerator_microbenchmarks/benchmarks/matmul.py
index edd67d16..752231d0 100644
--- a/src/accelerator_microbenchmarks/benchmarks/matmul.py
+++ b/src/accelerator_microbenchmarks/benchmarks/matmul.py
@@ -14,10 +14,16 @@
import jax.numpy as jnp
import numpy as np
+# Unquantized floating-point dtypes supported for inputs and outputs.
+_UNQUANTIZED_FLOAT_DTYPES = (
+ jnp.bfloat16,
+ jnp.float32,
+)
+
@dataclasses.dataclass
-class GemmParams(base.BaseBenchmarkParams):
- """Configuration parameters for GEMM benchmark."""
+class MatmulShapeParams(base.BaseBenchmarkParams):
+ """Shared matrix shape, dtype, and seed parameters for GEMM benchmarks."""
m: int = dataclasses.field(
default=1024,
@@ -52,11 +58,16 @@ class GemmParams(base.BaseBenchmarkParams):
"help": "Output accumulation/result data type (e.g. bfloat16)."
},
)
-
seed: int = dataclasses.field(
default=0,
metadata={"min": 0, "help": "Random seed for tensor initialization."},
)
+
+
+@dataclasses.dataclass
+class GemmParams(MatmulShapeParams):
+ """Configuration parameters for GEMM benchmark."""
+
use_scaling_factors: bool = dataclasses.field(
default=False,
metadata={"help": "Apply row-wise/column-wise FP8 scaling factors."},
@@ -134,6 +145,29 @@ def calculate_throughput_metrics(
f"{prefix}_tflops_per_device": tflops_per_sec,
}
+ def _replicated_normal(
+ self, key: jax.Array, shape: tuple[int, ...], dtype: Any
+ ) -> jax.Array:
+ """Samples a normal array replicated on every mesh device.
+
+ Generating under `jit` with a replicated `out_shardings` (as in `hbm.py`)
+ produces each shard on its own device, and the key is passed as a host
+ array, so no data moves between devices. The result equals eager sampling
+ with the same key.
+ """
+ # Replicated sharding for computation ops to avoid discrepancies
+ if self.mesh is None:
+ raise ValueError("Mesh not initialized.")
+ replicated_sharding = jax.sharding.NamedSharding(
+ self.mesh, jax.sharding.PartitionSpec(None, None)
+ )
+ generate = jax.jit(
+ _sample_normal,
+ static_argnums=(1, 2),
+ out_shardings=replicated_sharding,
+ )
+ return generate(np.asarray(key), shape, dtype)
+
@registry.benchmark_registry.register("gemm", aliases=["gemm_generalized"])
class GeneralizedGemmBenchmark(BaseGemmBenchmark[TGemmParams]):
@@ -207,29 +241,6 @@ def get_run_identifier(self) -> str:
run_identifier += "_sf"
return run_identifier
- def _replicated_normal(
- self, key: jax.Array, shape: tuple[int, ...], dtype: Any
- ) -> jax.Array:
- """Samples a normal array replicated on every mesh device.
-
- Generating under `jit` with a replicated `out_shardings` (as in `hbm.py`)
- produces each shard on its own device, and the key is passed as a host
- array, so no data moves between devices. The result equals eager sampling
- with the same key.
- """
- # Replicated sharding for computation ops to avoid discrepancies
- if self.mesh is None:
- raise ValueError("Mesh not initialized.")
- replicated_sharding = jax.sharding.NamedSharding(
- self.mesh, jax.sharding.PartitionSpec(None, None)
- )
- generate = jax.jit(
- _sample_normal,
- static_argnums=(1, 2),
- out_shardings=replicated_sharding,
- )
- return generate(np.asarray(key), shape, dtype)
-
def generate_inputs(self) -> tuple[Any, ...]:
(
m,
@@ -696,3 +707,134 @@ def calculate_metrics(self, times_ms: list[float]) -> dict[str, Any]:
if self.config.enable_checksum:
return super().calculate_metrics(times_ms)
return super(GemmChainedBenchmark, self).calculate_metrics(times_ms)
+
+
+def _quantize_per_channel_dynamic(
+ x: jax.Array,
+ *,
+ reduce_axis: int,
+ quant_dtype: jax.typing.DTypeLike,
+) -> jax.Array:
+ """Dynamically quantizes `x` per channel along `reduce_axis`."""
+ q_dtype = jnp.dtype(quant_dtype)
+ q_max = float(dtypes.get_dtype_max(q_dtype))
+ x_f32 = x.astype(jnp.float32)
+ amax = jnp.max(jnp.abs(x_f32), axis=reduce_axis, keepdims=True)
+ inv_scale_f32 = q_max / jnp.maximum(amax, 1e-7)
+ q = x_f32 * inv_scale_f32
+ if jnp.issubdtype(q_dtype, jnp.integer):
+ q = jnp.clip(jnp.round(q), -q_max, q_max)
+ else:
+ q = jnp.clip(q, -q_max, q_max)
+ return q.astype(q_dtype)
+
+
+@dataclasses.dataclass
+class QuantizedGemmParams(MatmulShapeParams):
+ """Configuration parameters for the `gemm_quantized` microbenchmark."""
+
+ DTYPE_CONSTRAINTS = {
+ "in_dtype": _UNQUANTIZED_FLOAT_DTYPES,
+ "quant_dtype": (jnp.float8_e4m3fn, jnp.float8_e5m2, jnp.int8),
+ "out_dtype": _UNQUANTIZED_FLOAT_DTYPES,
+ }
+
+ quant_dtype: dtypes.DType = base.dtype_field(
+ default=dtypes.DType.FLOAT8_E4M3FN,
+ metadata={
+ "help": (
+ "Quantized operand data type fed to the MXU (e.g."
+ " 'float8_e4m3fn', 'int8')."
+ )
+ },
+ )
+
+
+@registry.benchmark_registry.register("gemm_quantized", is_experimental=True)
+class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]):
+ """[EXPERIMENTAL] GEMM with dynamic per-channel quantization of A and B."""
+
+ Config = QuantizedGemmParams # pylint: disable=invalid-name
+ requires_ici: bool = False
+ REPORT_SCHEMA: Sequence[tuple[str, Callable[[Any], str]]] = (
+ ("in_dtype", report.format_str),
+ ("quant_dtype", report.format_str),
+ ("out_dtype", report.format_str),
+ ("m", report.format_str),
+ ("k", report.format_str),
+ ("n", report.format_str),
+ ("total_flops", report.format_2f),
+ ("wall_clock_p50_ms", report.format_4f),
+ ("wall_clock_tflops_per_chip", report.format_2f),
+ ("xprof_p50_ms", report.format_4f),
+ ("xprof_tflops_per_chip", report.format_2f),
+ )
+
+ def get_compute_dtype(self) -> str:
+ """Returns the quantized dtype, which determines the MXU peak TFLOPS."""
+ return str(self.config.quant_dtype)
+
+ def setup(self):
+ if self.mesh is None:
+ raise ValueError("Mesh not initialized.")
+ cfg = self.config
+ quant_dtype = dtypes.parse_dtype(cfg.quant_dtype)
+ out_dtype = dtypes.parse_dtype(cfg.out_dtype)
+ dot_acc_dtype = (
+ jnp.int32 if jnp.issubdtype(quant_dtype, jnp.integer) else jnp.float32
+ )
+
+ @jax.jit
+ def quant_gemm_fn(a, b):
+ with jax.named_scope(constants.MARKER):
+ qa = _quantize_per_channel_dynamic(
+ a,
+ reduce_axis=1,
+ quant_dtype=quant_dtype,
+ )
+ qb = _quantize_per_channel_dynamic(
+ b,
+ reduce_axis=0,
+ quant_dtype=quant_dtype,
+ )
+ acc = jax.lax.dot_general(
+ qa,
+ qb,
+ dimension_numbers=(((1,), (0,)), ((), ())),
+ preferred_element_type=dot_acc_dtype,
+ )
+ return acc.astype(out_dtype)
+
+ self._jit_fn = quant_gemm_fn
+
+ def get_run_identifier(self) -> str:
+ cfg = self.config
+ return (
+ f"m_{cfg.m}_k_{cfg.k}_n_{cfg.n}_"
+ f"{cfg.in_dtype}_to_{cfg.quant_dtype}_out_{cfg.out_dtype}"
+ )
+
+ def generate_inputs(self) -> tuple[Any, ...]:
+ cfg = self.config
+ in_dtype = dtypes.parse_dtype(cfg.in_dtype)
+ key_a, key_b = jax.random.split(jax.random.PRNGKey(cfg.seed))
+ a = self._replicated_normal(key_a, (cfg.m, cfg.k), in_dtype)
+ b = self._replicated_normal(key_b, (cfg.k, cfg.n), in_dtype)
+ return (a, b)
+
+ def run_op(self, *args) -> jax.Array:
+ if self._jit_fn is None:
+ raise ValueError("JIT function not initialized.")
+ return self._jit_fn(*args)
+
+ def get_total_bytes(self) -> float:
+ """Returns 1-pass algorithmic HBM bytes (read A, read B, write OUT)."""
+ cfg = self.config
+ m, k, n = cfg.m, cfg.k, cfg.n
+ in_bytes = dtypes.get_dtype_bytes(cfg.in_dtype)
+ out_bytes = dtypes.get_dtype_bytes(cfg.out_dtype)
+ return float((m * k + k * n) * in_bytes + m * n * out_bytes)
+
+ def get_total_flops(self) -> float:
+ m, k, n = self.config.m, self.config.k, self.config.n
+ return float(2 * m * n * k)
diff --git a/src/accelerator_microbenchmarks/op_flags.yaml b/src/accelerator_microbenchmarks/op_flags.yaml
index ce9b6ba9..9f0ed6d6 100644
--- a/src/accelerator_microbenchmarks/op_flags.yaml
+++ b/src/accelerator_microbenchmarks/op_flags.yaml
@@ -11,19 +11,6 @@ gemm_flags: &gemm_flags
- "--xla_tpu_vmem_scavenging_mode=NONE"
- "--xla_tpu_dvfs_p_state=7"
-gemm_numerics_flags: &gemm_numerics_flags
- flags:
- - "--xla_tpu_enable_async_collective_fusion=false"
- - "--xla_tpu_enable_async_collective_fusion_multiple_steps=true"
- - "--xla_tpu_overlap_compute_collective_tc=true"
- - "--xla_enable_async_all_gather=true"
- - "--xla_enable_async_collective_permute=true"
- - "--xla_tpu_enable_all_experimental_scheduler_features=true"
- - "--xla_tpu_accumulate_into_mrb=true"
- - "--xla_tpu_scoped_vmem_limit_kib=65536"
- - "--xla_tpu_allow_conv_input_fusion_with_downcast_convert=true"
- - "--xla_tpu_dvfs_p_state=7"
-
all_reduce:
flags:
- "--xla_jf_debug_level=3"
@@ -123,6 +110,7 @@ tokamax_splash_attention:
- "--xla_tpu_dvfs_p_state=7"
gemm: *gemm_flags
+gemm_quantized: *gemm_flags
gemm_throttling: *gemm_flags
gemm_pulse: *gemm_flags
gemm_chained: *gemm_flags
diff --git a/tests/test_benchmark_loader.py b/tests/test_benchmark_loader.py
index f7163494..af995bf2 100644
--- a/tests/test_benchmark_loader.py
+++ b/tests/test_benchmark_loader.py
@@ -34,6 +34,7 @@ def test_load_all_benchmarks_expected_benchmarks(self):
"gemm_throttling",
"gemm_pulse",
"gemm_chained",
+ "gemm_quantized",
"hbm",
"hbm_bandwidth",
"attention_flashed",
@@ -93,6 +94,9 @@ def test_experimental_benchmarks(self):
)
self.assertTrue(registry.benchmark_registry.is_experimental("gemm_chained"))
self.assertTrue(registry.benchmark_registry.is_experimental("gemm_pulse"))
+ self.assertTrue(
+ registry.benchmark_registry.is_experimental("gemm_quantized")
+ )
def test_all_benchmarks_have_valid_config(self):
"""Verify that every registered benchmark defines a Config dataclass."""
@@ -165,6 +169,7 @@ def test_benchmarks_without_ici(self):
"gemm_throttling",
"gemm_pulse",
"gemm_chained",
+ "gemm_quantized",
"hbm",
"hbm_bandwidth",
"host_to_device",
diff --git a/tests/test_matmul.py b/tests/test_matmul.py
index 1e113a6c..bb49e891 100644
--- a/tests/test_matmul.py
+++ b/tests/test_matmul.py
@@ -729,5 +729,108 @@ def test_schema_coverage(self):
)
+class QuantizedGemmBenchmarkTest(parameterized.TestCase):
+ """Unit tests for QuantizedGemmBenchmark."""
+
+ def setUp(self):
+ super().setUp()
+ self._platform_patcher = mock.patch(
+ "accelerator_microbenchmarks.core.platform.get_platform_info",
+ return_value=test_report_utils.DEFAULT_TEST_PLATFORM_INFO,
+ )
+ self._platform_patcher.start()
+ self.addCleanup(self._platform_patcher.stop)
+ self.mock_mesh = jax.sharding.Mesh(
+ np.array(jax.devices()), axis_names=("device",)
+ )
+
+ def _setup_benchmark(self, **kwargs):
+ """Initializes a QuantizedGemmBenchmark instance for testing."""
+ params = {
+ "m": 64,
+ "k": 128,
+ "n": 64,
+ "in_dtype": "bfloat16",
+ "quant_dtype": "float8_e4m3fn",
+ "out_dtype": "bfloat16",
+ "warmup_tries": 1,
+ "num_runs": 2,
+ }
+ params.update(kwargs)
+ cfg = matmul.QuantizedGemmParams(**params)
+ self.bm = matmul.QuantizedGemmBenchmark(
+ config=cfg,
+ hardware_spec=system.TPU7X_HARDWARE_SPEC,
+ mesh=self.mock_mesh,
+ )
+ self.bm.setup()
+
+ def test_benchmark_registered(self):
+ bm_class = registry.benchmark_registry.get_benchmark("gemm_quantized")
+ self.assertEqual(bm_class, matmul.QuantizedGemmBenchmark)
+
+ @parameterized.parameters("float8_e4m3fn", "int8")
+ def test_generate_inputs_and_run_op(self, qd):
+ """Tests input generation and numerical output against quantized matmul."""
+ self._setup_benchmark(quant_dtype=qd)
+ self.assertEqual(
+ self.bm.get_run_identifier(),
+ f"m_64_k_128_n_64_bfloat16_to_{qd}_out_bfloat16",
+ )
+ inputs = self.bm.generate_inputs()
+ self.assertLen(inputs, 2)
+ a, b = inputs
+ out = self.bm.run_op(a, b)
+ self.assertEqual(out.shape, (64, 64))
+ self.assertEqual(out.dtype, jnp.bfloat16)
+
+ quantize_fn = jax.jit(
+ matmul._quantize_per_channel_dynamic, # pylint: disable=protected-access
+ static_argnames=("reduce_axis", "quant_dtype"),
+ )
+ qa = quantize_fn(a, reduce_axis=1, quant_dtype=qd)
+ qb = quantize_fn(b, reduce_axis=0, quant_dtype=qd)
+ expected = jnp.matmul(qa.astype(jnp.float32), qb.astype(jnp.float32))
+ np.testing.assert_allclose(out.astype(jnp.float32), expected, rtol=2e-2)
+
+ def test_quantize_per_channel_dynamic_and_dtype_constraints(self):
+ """Tests int8 rounding and DTYPE_CONSTRAINTS validation."""
+ x = jnp.array([[127.0, 0.6], [63.5, -63.5]], dtype=jnp.bfloat16)
+ q_int8 = (
+ matmul._quantize_per_channel_dynamic( # pylint: disable=protected-access
+ x, reduce_axis=1, quant_dtype=jnp.int8
+ )
+ )
+ self.assertEqual(q_int8.dtype, jnp.int8)
+ np.testing.assert_array_equal(
+ np.asarray(q_int8), np.array([[127, 1], [127, -127]], dtype=np.int8)
+ )
+ with self.assertRaises(ValueError):
+ matmul.QuantizedGemmParams(quant_dtype="uint8")
+ with self.assertRaises(ValueError):
+ matmul.QuantizedGemmParams(quant_dtype="bfloat16")
+ with self.assertRaises(ValueError):
+ matmul.QuantizedGemmParams(in_dtype="float64")
+ with self.assertRaises(ValueError):
+ matmul.QuantizedGemmParams(out_dtype="float16")
+ self.assertTrue(issubclass(matmul.GemmParams, matmul.MatmulShapeParams))
+ self.assertTrue(
+ issubclass(matmul.QuantizedGemmParams, matmul.MatmulShapeParams)
+ )
+
+ def test_flops_bytes_and_compute_dtype(self):
+ self._setup_benchmark(m=64, k=128, n=64)
+ self.assertEqual(self.bm.get_compute_dtype(), "float8_e4m3fn")
+ self.assertEqual(self.bm.get_total_flops(), float(2 * 64 * 128 * 64))
+ # 1-pass algorithmic HBM bytes: read A (64x128 bf16) + read B (128x64 bf16)
+ # + write OUT (64x64 bf16).
+ expected_bytes = float((64 * 128 + 128 * 64) * 2 + 64 * 64 * 2)
+ self.assertEqual(self.bm.get_total_bytes(), expected_bytes)
+
+ def test_schema_coverage(self):
+ self._setup_benchmark()
+ test_report_utils.assert_schema_matches_output(self, self.bm)
+
+
if __name__ == "__main__":
absltest.main()