From 79da05ccf8bdb927456658c51f889c3682e5bc32 Mon Sep 17 00:00:00 2001 From: TPUMS Team Date: Sun, 15 Nov 2026 22:34:26 -0800 Subject: [PATCH] Add the experimental `gemm_quantized` microbenchmark (`QuantizedGemmBenchmark` and `QuantizedGemmParams`) in `benchmarks/matmul.py` to measure end-to-end matrix multiplication with on-the-fly dynamic per-channel quantization of both `A[M, K]` and `B[K, N]` operands. * Subclasses `BaseGemmBenchmark` in `benchmarks/matmul.py` (sharing `_replicated_normal`, compute roofline efficiency, XProf fallback matching, and `2 * M * N * K` FLOP throughput calculation) and extracts `MatmulShapeParams` for shared `(M, K, N)` shape, dtype, and seed fields. * Quantizes `bfloat16`/`float32` operands dynamically per row/column (`inv_scale = Q_MAX / amax`, `q = clip(x * inv_scale, -Q_MAX, Q_MAX)`) to `quant_dtype` (`float8_e4m3fn`, `float8_e5m2`, `int8`) inside the timed step before `jax.lax.dot_general`. * Counts 1-pass algorithmic HBM bytes (reading `A` and `B` once plus writing the output) and uses `quant_dtype` for the hardware MXU peak TFLOPS ceiling. PiperOrigin-RevId: 1022517434 --- docs/BENCHMARKS.md | 19 ++ .../benchmarks/matmul.py | 194 +++++++++++++++--- src/accelerator_microbenchmarks/op_flags.yaml | 14 +- tests/test_benchmark_loader.py | 5 + tests/test_matmul.py | 103 ++++++++++ 5 files changed, 296 insertions(+), 39 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index ed4f9284..6299b30b 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -18,6 +18,7 @@ This guide provides the reference for **TPUMS (TPU Microbenchmark Suite)** param - [2.1.1 GEMM Throttling (`gemm_throttling`) [experimental]](#211-gemm-throttling-gemm_throttling-experimental) - [2.1.2 Chained Matrix Multiplication & Determinism Check (`gemm_chained`) [experimental]](#212-chained-matrix-multiplication--determinism-check-gemm_chained-experimental) - [2.1.3 GEMM Pulse (`gemm_pulse`) [experimental]](#213-gemm-pulse-gemm_pulse-experimental) + - [2.1.4 Quantized Matrix Multiplication (`gemm_quantized`) [experimental]](#214-quantized-matrix-multiplication-gemm_quantized-experimental) - [2.2 HBM Memory Bandwidth (`hbm`)](#22-hbm-memory-bandwidth-hbm) - [2.3 Host I/O Bandwidth (`host_to_device`, `device_to_host`)](#23-host-io-bandwidth-host_to_device-device_to_host) - [2.4 Device-to-Device Transfers (`device_to_device`)](#24-inter-chip-interconnect-device_to_device) @@ -252,6 +253,24 @@ final cycle ends with an `idle_s` pause followed by traced `gemm_throttling` soaking and thermal metrics, plus all `gemm_chained` checksum metrics when `enable_checksum=True`. + +#### 2.1.4 Quantized Matrix Multiplication (`gemm_quantized`) [experimental] + +Evaluates end-to-end quantized matrix multiplication (`bfloat16` or `float32` +inputs `A[M, K]` and `B[K, N]` dynamically quantized row-wise/column-wise to +`float8_e4m3fn`, `float8_e5m2`, or `int8` inside the timed step before +`lax.dot_general`). + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `m` | `int` | `1024` | Matrix dimension `M` (rows of `A` and output matrix `C`). | +| `k` | `int` | `1024` | Contracting dimension `K` (columns of `A` and rows of `B`). | +| `n` | `int` | `1024` | Matrix dimension `N` (columns of `B` and output matrix `C`). | +| `in_dtype` | `str` | `"bfloat16"` | Unquantized input operand dtype (e.g., `bfloat16`, `float32`). | +| `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized matmul operand dtype (e.g., `float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. | +| `out_dtype` | `str` | `"bfloat16"` | Output tensor dtype (e.g., `bfloat16`, `float32`). | +| `seed` | `int` | `0` | PRNG seed for initializing input tensors in HBM. | + --- diff --git a/src/accelerator_microbenchmarks/benchmarks/matmul.py b/src/accelerator_microbenchmarks/benchmarks/matmul.py index edd67d16..752231d0 100644 --- a/src/accelerator_microbenchmarks/benchmarks/matmul.py +++ b/src/accelerator_microbenchmarks/benchmarks/matmul.py @@ -14,10 +14,16 @@ import jax.numpy as jnp import numpy as np +# Unquantized floating-point dtypes supported for inputs and outputs. +_UNQUANTIZED_FLOAT_DTYPES = ( + jnp.bfloat16, + jnp.float32, +) + @dataclasses.dataclass -class GemmParams(base.BaseBenchmarkParams): - """Configuration parameters for GEMM benchmark.""" +class MatmulShapeParams(base.BaseBenchmarkParams): + """Shared matrix shape, dtype, and seed parameters for GEMM benchmarks.""" m: int = dataclasses.field( default=1024, @@ -52,11 +58,16 @@ class GemmParams(base.BaseBenchmarkParams): "help": "Output accumulation/result data type (e.g. bfloat16)." }, ) - seed: int = dataclasses.field( default=0, metadata={"min": 0, "help": "Random seed for tensor initialization."}, ) + + +@dataclasses.dataclass +class GemmParams(MatmulShapeParams): + """Configuration parameters for GEMM benchmark.""" + use_scaling_factors: bool = dataclasses.field( default=False, metadata={"help": "Apply row-wise/column-wise FP8 scaling factors."}, @@ -134,6 +145,29 @@ def calculate_throughput_metrics( f"{prefix}_tflops_per_device": tflops_per_sec, } + def _replicated_normal( + self, key: jax.Array, shape: tuple[int, ...], dtype: Any + ) -> jax.Array: + """Samples a normal array replicated on every mesh device. + + Generating under `jit` with a replicated `out_shardings` (as in `hbm.py`) + produces each shard on its own device, and the key is passed as a host + array, so no data moves between devices. The result equals eager sampling + with the same key. + """ + # Replicated sharding for computation ops to avoid discrepancies + if self.mesh is None: + raise ValueError("Mesh not initialized.") + replicated_sharding = jax.sharding.NamedSharding( + self.mesh, jax.sharding.PartitionSpec(None, None) + ) + generate = jax.jit( + _sample_normal, + static_argnums=(1, 2), + out_shardings=replicated_sharding, + ) + return generate(np.asarray(key), shape, dtype) + @registry.benchmark_registry.register("gemm", aliases=["gemm_generalized"]) class GeneralizedGemmBenchmark(BaseGemmBenchmark[TGemmParams]): @@ -207,29 +241,6 @@ def get_run_identifier(self) -> str: run_identifier += "_sf" return run_identifier - def _replicated_normal( - self, key: jax.Array, shape: tuple[int, ...], dtype: Any - ) -> jax.Array: - """Samples a normal array replicated on every mesh device. - - Generating under `jit` with a replicated `out_shardings` (as in `hbm.py`) - produces each shard on its own device, and the key is passed as a host - array, so no data moves between devices. The result equals eager sampling - with the same key. - """ - # Replicated sharding for computation ops to avoid discrepancies - if self.mesh is None: - raise ValueError("Mesh not initialized.") - replicated_sharding = jax.sharding.NamedSharding( - self.mesh, jax.sharding.PartitionSpec(None, None) - ) - generate = jax.jit( - _sample_normal, - static_argnums=(1, 2), - out_shardings=replicated_sharding, - ) - return generate(np.asarray(key), shape, dtype) - def generate_inputs(self) -> tuple[Any, ...]: ( m, @@ -696,3 +707,134 @@ def calculate_metrics(self, times_ms: list[float]) -> dict[str, Any]: if self.config.enable_checksum: return super().calculate_metrics(times_ms) return super(GemmChainedBenchmark, self).calculate_metrics(times_ms) + + +def _quantize_per_channel_dynamic( + x: jax.Array, + *, + reduce_axis: int, + quant_dtype: jax.typing.DTypeLike, +) -> jax.Array: + """Dynamically quantizes `x` per channel along `reduce_axis`.""" + q_dtype = jnp.dtype(quant_dtype) + q_max = float(dtypes.get_dtype_max(q_dtype)) + x_f32 = x.astype(jnp.float32) + amax = jnp.max(jnp.abs(x_f32), axis=reduce_axis, keepdims=True) + inv_scale_f32 = q_max / jnp.maximum(amax, 1e-7) + q = x_f32 * inv_scale_f32 + if jnp.issubdtype(q_dtype, jnp.integer): + q = jnp.clip(jnp.round(q), -q_max, q_max) + else: + q = jnp.clip(q, -q_max, q_max) + return q.astype(q_dtype) + + +@dataclasses.dataclass +class QuantizedGemmParams(MatmulShapeParams): + """Configuration parameters for the `gemm_quantized` microbenchmark.""" + + DTYPE_CONSTRAINTS = { + "in_dtype": _UNQUANTIZED_FLOAT_DTYPES, + "quant_dtype": (jnp.float8_e4m3fn, jnp.float8_e5m2, jnp.int8), + "out_dtype": _UNQUANTIZED_FLOAT_DTYPES, + } + + quant_dtype: dtypes.DType = base.dtype_field( + default=dtypes.DType.FLOAT8_E4M3FN, + metadata={ + "help": ( + "Quantized operand data type fed to the MXU (e.g." + " 'float8_e4m3fn', 'int8')." + ) + }, + ) + + +@registry.benchmark_registry.register("gemm_quantized", is_experimental=True) +class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]): + """[EXPERIMENTAL] GEMM with dynamic per-channel quantization of A and B.""" + + Config = QuantizedGemmParams # pylint: disable=invalid-name + requires_ici: bool = False + REPORT_SCHEMA: Sequence[tuple[str, Callable[[Any], str]]] = ( + ("in_dtype", report.format_str), + ("quant_dtype", report.format_str), + ("out_dtype", report.format_str), + ("m", report.format_str), + ("k", report.format_str), + ("n", report.format_str), + ("total_flops", report.format_2f), + ("wall_clock_p50_ms", report.format_4f), + ("wall_clock_tflops_per_chip", report.format_2f), + ("xprof_p50_ms", report.format_4f), + ("xprof_tflops_per_chip", report.format_2f), + ) + + def get_compute_dtype(self) -> str: + """Returns the quantized dtype, which determines the MXU peak TFLOPS.""" + return str(self.config.quant_dtype) + + def setup(self): + if self.mesh is None: + raise ValueError("Mesh not initialized.") + cfg = self.config + quant_dtype = dtypes.parse_dtype(cfg.quant_dtype) + out_dtype = dtypes.parse_dtype(cfg.out_dtype) + dot_acc_dtype = ( + jnp.int32 if jnp.issubdtype(quant_dtype, jnp.integer) else jnp.float32 + ) + + @jax.jit + def quant_gemm_fn(a, b): + with jax.named_scope(constants.MARKER): + qa = _quantize_per_channel_dynamic( + a, + reduce_axis=1, + quant_dtype=quant_dtype, + ) + qb = _quantize_per_channel_dynamic( + b, + reduce_axis=0, + quant_dtype=quant_dtype, + ) + acc = jax.lax.dot_general( + qa, + qb, + dimension_numbers=(((1,), (0,)), ((), ())), + preferred_element_type=dot_acc_dtype, + ) + return acc.astype(out_dtype) + + self._jit_fn = quant_gemm_fn + + def get_run_identifier(self) -> str: + cfg = self.config + return ( + f"m_{cfg.m}_k_{cfg.k}_n_{cfg.n}_" + f"{cfg.in_dtype}_to_{cfg.quant_dtype}_out_{cfg.out_dtype}" + ) + + def generate_inputs(self) -> tuple[Any, ...]: + cfg = self.config + in_dtype = dtypes.parse_dtype(cfg.in_dtype) + key_a, key_b = jax.random.split(jax.random.PRNGKey(cfg.seed)) + a = self._replicated_normal(key_a, (cfg.m, cfg.k), in_dtype) + b = self._replicated_normal(key_b, (cfg.k, cfg.n), in_dtype) + return (a, b) + + def run_op(self, *args) -> jax.Array: + if self._jit_fn is None: + raise ValueError("JIT function not initialized.") + return self._jit_fn(*args) + + def get_total_bytes(self) -> float: + """Returns 1-pass algorithmic HBM bytes (read A, read B, write OUT).""" + cfg = self.config + m, k, n = cfg.m, cfg.k, cfg.n + in_bytes = dtypes.get_dtype_bytes(cfg.in_dtype) + out_bytes = dtypes.get_dtype_bytes(cfg.out_dtype) + return float((m * k + k * n) * in_bytes + m * n * out_bytes) + + def get_total_flops(self) -> float: + m, k, n = self.config.m, self.config.k, self.config.n + return float(2 * m * n * k) diff --git a/src/accelerator_microbenchmarks/op_flags.yaml b/src/accelerator_microbenchmarks/op_flags.yaml index ce9b6ba9..9f0ed6d6 100644 --- a/src/accelerator_microbenchmarks/op_flags.yaml +++ b/src/accelerator_microbenchmarks/op_flags.yaml @@ -11,19 +11,6 @@ gemm_flags: &gemm_flags - "--xla_tpu_vmem_scavenging_mode=NONE" - "--xla_tpu_dvfs_p_state=7" -gemm_numerics_flags: &gemm_numerics_flags - flags: - - "--xla_tpu_enable_async_collective_fusion=false" - - "--xla_tpu_enable_async_collective_fusion_multiple_steps=true" - - "--xla_tpu_overlap_compute_collective_tc=true" - - "--xla_enable_async_all_gather=true" - - "--xla_enable_async_collective_permute=true" - - "--xla_tpu_enable_all_experimental_scheduler_features=true" - - "--xla_tpu_accumulate_into_mrb=true" - - "--xla_tpu_scoped_vmem_limit_kib=65536" - - "--xla_tpu_allow_conv_input_fusion_with_downcast_convert=true" - - "--xla_tpu_dvfs_p_state=7" - all_reduce: flags: - "--xla_jf_debug_level=3" @@ -123,6 +110,7 @@ tokamax_splash_attention: - "--xla_tpu_dvfs_p_state=7" gemm: *gemm_flags +gemm_quantized: *gemm_flags gemm_throttling: *gemm_flags gemm_pulse: *gemm_flags gemm_chained: *gemm_flags diff --git a/tests/test_benchmark_loader.py b/tests/test_benchmark_loader.py index f7163494..af995bf2 100644 --- a/tests/test_benchmark_loader.py +++ b/tests/test_benchmark_loader.py @@ -34,6 +34,7 @@ def test_load_all_benchmarks_expected_benchmarks(self): "gemm_throttling", "gemm_pulse", "gemm_chained", + "gemm_quantized", "hbm", "hbm_bandwidth", "attention_flashed", @@ -93,6 +94,9 @@ def test_experimental_benchmarks(self): ) self.assertTrue(registry.benchmark_registry.is_experimental("gemm_chained")) self.assertTrue(registry.benchmark_registry.is_experimental("gemm_pulse")) + self.assertTrue( + registry.benchmark_registry.is_experimental("gemm_quantized") + ) def test_all_benchmarks_have_valid_config(self): """Verify that every registered benchmark defines a Config dataclass.""" @@ -165,6 +169,7 @@ def test_benchmarks_without_ici(self): "gemm_throttling", "gemm_pulse", "gemm_chained", + "gemm_quantized", "hbm", "hbm_bandwidth", "host_to_device", diff --git a/tests/test_matmul.py b/tests/test_matmul.py index 1e113a6c..bb49e891 100644 --- a/tests/test_matmul.py +++ b/tests/test_matmul.py @@ -729,5 +729,108 @@ def test_schema_coverage(self): ) +class QuantizedGemmBenchmarkTest(parameterized.TestCase): + """Unit tests for QuantizedGemmBenchmark.""" + + def setUp(self): + super().setUp() + self._platform_patcher = mock.patch( + "accelerator_microbenchmarks.core.platform.get_platform_info", + return_value=test_report_utils.DEFAULT_TEST_PLATFORM_INFO, + ) + self._platform_patcher.start() + self.addCleanup(self._platform_patcher.stop) + self.mock_mesh = jax.sharding.Mesh( + np.array(jax.devices()), axis_names=("device",) + ) + + def _setup_benchmark(self, **kwargs): + """Initializes a QuantizedGemmBenchmark instance for testing.""" + params = { + "m": 64, + "k": 128, + "n": 64, + "in_dtype": "bfloat16", + "quant_dtype": "float8_e4m3fn", + "out_dtype": "bfloat16", + "warmup_tries": 1, + "num_runs": 2, + } + params.update(kwargs) + cfg = matmul.QuantizedGemmParams(**params) + self.bm = matmul.QuantizedGemmBenchmark( + config=cfg, + hardware_spec=system.TPU7X_HARDWARE_SPEC, + mesh=self.mock_mesh, + ) + self.bm.setup() + + def test_benchmark_registered(self): + bm_class = registry.benchmark_registry.get_benchmark("gemm_quantized") + self.assertEqual(bm_class, matmul.QuantizedGemmBenchmark) + + @parameterized.parameters("float8_e4m3fn", "int8") + def test_generate_inputs_and_run_op(self, qd): + """Tests input generation and numerical output against quantized matmul.""" + self._setup_benchmark(quant_dtype=qd) + self.assertEqual( + self.bm.get_run_identifier(), + f"m_64_k_128_n_64_bfloat16_to_{qd}_out_bfloat16", + ) + inputs = self.bm.generate_inputs() + self.assertLen(inputs, 2) + a, b = inputs + out = self.bm.run_op(a, b) + self.assertEqual(out.shape, (64, 64)) + self.assertEqual(out.dtype, jnp.bfloat16) + + quantize_fn = jax.jit( + matmul._quantize_per_channel_dynamic, # pylint: disable=protected-access + static_argnames=("reduce_axis", "quant_dtype"), + ) + qa = quantize_fn(a, reduce_axis=1, quant_dtype=qd) + qb = quantize_fn(b, reduce_axis=0, quant_dtype=qd) + expected = jnp.matmul(qa.astype(jnp.float32), qb.astype(jnp.float32)) + np.testing.assert_allclose(out.astype(jnp.float32), expected, rtol=2e-2) + + def test_quantize_per_channel_dynamic_and_dtype_constraints(self): + """Tests int8 rounding and DTYPE_CONSTRAINTS validation.""" + x = jnp.array([[127.0, 0.6], [63.5, -63.5]], dtype=jnp.bfloat16) + q_int8 = ( + matmul._quantize_per_channel_dynamic( # pylint: disable=protected-access + x, reduce_axis=1, quant_dtype=jnp.int8 + ) + ) + self.assertEqual(q_int8.dtype, jnp.int8) + np.testing.assert_array_equal( + np.asarray(q_int8), np.array([[127, 1], [127, -127]], dtype=np.int8) + ) + with self.assertRaises(ValueError): + matmul.QuantizedGemmParams(quant_dtype="uint8") + with self.assertRaises(ValueError): + matmul.QuantizedGemmParams(quant_dtype="bfloat16") + with self.assertRaises(ValueError): + matmul.QuantizedGemmParams(in_dtype="float64") + with self.assertRaises(ValueError): + matmul.QuantizedGemmParams(out_dtype="float16") + self.assertTrue(issubclass(matmul.GemmParams, matmul.MatmulShapeParams)) + self.assertTrue( + issubclass(matmul.QuantizedGemmParams, matmul.MatmulShapeParams) + ) + + def test_flops_bytes_and_compute_dtype(self): + self._setup_benchmark(m=64, k=128, n=64) + self.assertEqual(self.bm.get_compute_dtype(), "float8_e4m3fn") + self.assertEqual(self.bm.get_total_flops(), float(2 * 64 * 128 * 64)) + # 1-pass algorithmic HBM bytes: read A (64x128 bf16) + read B (128x64 bf16) + # + write OUT (64x64 bf16). + expected_bytes = float((64 * 128 + 128 * 64) * 2 + 64 * 64 * 2) + self.assertEqual(self.bm.get_total_bytes(), expected_bytes) + + def test_schema_coverage(self): + self._setup_benchmark() + test_report_utils.assert_schema_matches_output(self, self.bm) + + if __name__ == "__main__": absltest.main()