From 7140a71da8b78108fd71f51cccbd556317af79b7 Mon Sep 17 00:00:00 2001 From: TPUMS Team Date: Mon, 16 Nov 2026 21:21:54 -0800 Subject: [PATCH] Extend the experimental `gemm_quantized` benchmark in `benchmarks/matmul.py` to support static per-channel scaling and optional output dequantization on top of the dynamic per-channel baseline. * Add `QuantScaling` (`"dynamic"` and `"static"`, using fixed clip bound `0.5 * Q_MAX` and compile-time scale `0.5`) to `QuantizedGemmParams`. * Add `dequantize` (`bool`, default `False`) to multiply the `dot_general` accumulator by the row and column dequantization scales (`sf_a * sf_b`) before casting to `out_dtype`. * Include `scaling` and `dequantize` in `QuantizedGemmBenchmark.REPORT_SCHEMA` and `get_run_identifier()`. PiperOrigin-RevId: 1023375939 --- docs/BENCHMARKS.md | 7 +- .../benchmarks/matmul.py | 84 ++++++++-- tests/test_matmul.py | 149 ++++++++++++++++-- 3 files changed, 210 insertions(+), 30 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 8cc69f04..5c178c60 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -257,9 +257,10 @@ metrics when `enable_checksum=True`. #### 2.1.4 Quantized Matrix Multiplication (`gemm_quantized`) [experimental] Evaluates end-to-end quantized matrix multiplication (`bfloat16` or `float32` -inputs `A[M, K]` and `B[K, N]` dynamically quantized row-wise/column-wise to +inputs `A[M, K]` and `B[K, N]` quantized row-wise/column-wise to `float8_e4m3fn`, `float8_e5m2`, or `int8` inside the timed step before -`lax.dot_general`). +`lax.dot_general`, with dynamic or static scaling and optional output +dequantization). | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | @@ -269,6 +270,8 @@ inputs `A[M, K]` and `B[K, N]` dynamically quantized row-wise/column-wise to | `in_dtype` | `str` | `"bfloat16"` | Unquantized input operand dtype (e.g., `bfloat16`, `float32`). | | `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized matmul operand dtype (e.g., `float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. | | `out_dtype` | `str` | `"bfloat16"` | Output tensor dtype (e.g., `bfloat16`, `float32`). | +| `scaling` | `str` | `"dynamic"` | Scale calibration: `"dynamic"` (per-row/column `amax` computed on the fly) or `"static"` (fixed clip bound `0.5 * Q_MAX`, i.e., compile-time scale `0.5`). | +| `dequantize` | `bool` | `False` | When `True`, multiply the matmul accumulator by the row and column dequantization scales before casting to `out_dtype`. | | `seed` | `int` | `0` | PRNG seed for initializing input tensors in HBM. | --- diff --git a/src/accelerator_microbenchmarks/benchmarks/matmul.py b/src/accelerator_microbenchmarks/benchmarks/matmul.py index 752231d0..d5936538 100644 --- a/src/accelerator_microbenchmarks/benchmarks/matmul.py +++ b/src/accelerator_microbenchmarks/benchmarks/matmul.py @@ -709,24 +709,59 @@ def calculate_metrics(self, times_ms: list[float]) -> dict[str, Any]: return super(GemmChainedBenchmark, self).calculate_metrics(times_ms) -def _quantize_per_channel_dynamic( +class QuantScaling(constants.ParamEnum): + """Calibration method used to derive the quantization scaling factors. + + Attributes: + DYNAMIC: Scales are computed on the fly from the absolute maximum of each + row or column (`scale = amax / Q_MAX`). + STATIC: Uses a fixed, compile-time clip bound `0.5 * Q_MAX` (`scale = 0.5`) + with no `amax` reduction. A non-unit (`!= 1.0`) power-of-2 scale prevents + XLA from folding away the scale/dequantize multiplications as identity + no-ops while scaling inputs without mantissa rounding loss. + """ + + DYNAMIC = "dynamic" + STATIC = "static" + + +def _quantize_per_channel( x: jax.Array, *, reduce_axis: int, quant_dtype: jax.typing.DTypeLike, -) -> jax.Array: - """Dynamically quantizes `x` per channel along `reduce_axis`.""" + scaling: QuantScaling, +) -> tuple[jax.Array, jax.Array]: + """Quantizes `x` per channel along `reduce_axis`.""" q_dtype = jnp.dtype(quant_dtype) q_max = float(dtypes.get_dtype_max(q_dtype)) x_f32 = x.astype(jnp.float32) - amax = jnp.max(jnp.abs(x_f32), axis=reduce_axis, keepdims=True) - inv_scale_f32 = q_max / jnp.maximum(amax, 1e-7) + + if scaling == QuantScaling.STATIC: + scale_shape = list(x.shape) + scale_shape[reduce_axis] = 1 + # Use a non-unit power-of-2 bound (`0.5 * q_max` -> `scale = 0.5`) so XLA + # cannot elide `x * inv_scale` or `acc * (sf_a * sf_b)` as multiply-by-1 + # no-ops. + static_absmax = 0.5 * q_max + scale_f32 = jnp.full(scale_shape, static_absmax / q_max, dtype=jnp.float32) + inv_scale_f32 = jnp.reciprocal(scale_f32) + elif scaling == QuantScaling.DYNAMIC: + amax = jnp.max(jnp.abs(x_f32), axis=reduce_axis, keepdims=True) + clamped_amax = jnp.maximum(amax, 1e-7) + scale_f32 = clamped_amax / q_max + inv_scale_f32 = q_max / clamped_amax + else: + raise ValueError( + f"Unsupported scaling mode '{scaling}'; expected 'dynamic' or 'static'." + ) + q = x_f32 * inv_scale_f32 if jnp.issubdtype(q_dtype, jnp.integer): q = jnp.clip(jnp.round(q), -q_max, q_max) else: q = jnp.clip(q, -q_max, q_max) - return q.astype(q_dtype) + return q.astype(q_dtype), scale_f32 @dataclasses.dataclass @@ -748,11 +783,29 @@ class QuantizedGemmParams(MatmulShapeParams): ) }, ) + scaling: QuantScaling = dataclasses.field( + default=QuantScaling.DYNAMIC, + metadata={ + "help": ( + "Scale calibration: 'dynamic' (amax computed on the fly) or" + " 'static' (fixed clip bound 0.5 * Q_MAX, i.e. scale = 0.5)." + ) + }, + ) + dequantize: bool = dataclasses.field( + default=False, + metadata={ + "help": ( + "If True, multiply the accumulator by the row and column" + " dequantization scales before casting to out_dtype." + ) + }, + ) @registry.benchmark_registry.register("gemm_quantized", is_experimental=True) class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]): - """[EXPERIMENTAL] GEMM with dynamic per-channel quantization of A and B.""" + """[EXPERIMENTAL] GEMM with per-channel quantization of A and B.""" Config = QuantizedGemmParams # pylint: disable=invalid-name requires_ici: bool = False @@ -763,6 +816,8 @@ class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]): ("m", report.format_str), ("k", report.format_str), ("n", report.format_str), + ("scaling", report.format_str), + ("dequantize", report.format_str), ("total_flops", report.format_2f), ("wall_clock_p50_ms", report.format_4f), ("wall_clock_tflops_per_chip", report.format_2f), @@ -787,15 +842,17 @@ def setup(self): @jax.jit def quant_gemm_fn(a, b): with jax.named_scope(constants.MARKER): - qa = _quantize_per_channel_dynamic( + qa, sf_a = _quantize_per_channel( a, reduce_axis=1, quant_dtype=quant_dtype, + scaling=cfg.scaling, ) - qb = _quantize_per_channel_dynamic( + qb, sf_b = _quantize_per_channel( b, reduce_axis=0, quant_dtype=quant_dtype, + scaling=cfg.scaling, ) acc = jax.lax.dot_general( qa, @@ -803,16 +860,21 @@ def quant_gemm_fn(a, b): dimension_numbers=(((1,), (0,)), ((), ())), preferred_element_type=dot_acc_dtype, ) + if cfg.dequantize: + acc = acc.astype(jnp.float32) * (sf_a * sf_b) return acc.astype(out_dtype) self._jit_fn = quant_gemm_fn def get_run_identifier(self) -> str: cfg = self.config - return ( + run_id = ( f"m_{cfg.m}_k_{cfg.k}_n_{cfg.n}_" - f"{cfg.in_dtype}_to_{cfg.quant_dtype}_out_{cfg.out_dtype}" + f"{cfg.in_dtype}_to_{cfg.quant_dtype}_{cfg.scaling}_out_{cfg.out_dtype}" ) + if cfg.dequantize: + run_id += "_dequant" + return run_id def generate_inputs(self) -> tuple[Any, ...]: cfg = self.config diff --git a/tests/test_matmul.py b/tests/test_matmul.py index bb49e891..2a8e1682 100644 --- a/tests/test_matmul.py +++ b/tests/test_matmul.py @@ -45,7 +45,9 @@ def _setup_benchmark(self, **kwargs): params.update(kwargs) config = matmul.GemmParams(**params) self.bm = matmul.GeneralizedGemmBenchmark( - config=config, hardware_spec=system.TPU7X_HARDWARE_SPEC, mesh=self.mock_mesh + config=config, + hardware_spec=system.TPU7X_HARDWARE_SPEC, + mesh=self.mock_mesh, ) self.bm.setup() @@ -203,7 +205,9 @@ def test_is_xprof_op(self): """Test that is_xprof_op correctly identifies convolution fusion events.""" config = matmul.GemmParams(m=128, n=128, k=128) self.bm = matmul.GeneralizedGemmBenchmark( - config=config, hardware_spec=system.TPU7X_HARDWARE_SPEC, mesh=self.mock_mesh + config=config, + hardware_spec=system.TPU7X_HARDWARE_SPEC, + mesh=self.mock_mesh, ) self.assertTrue( self.bm.match_xprof_op_fallback( @@ -766,6 +770,7 @@ def _setup_benchmark(self, **kwargs): self.bm.setup() def test_benchmark_registered(self): + """Tests that gemm_quantized is registered in the benchmark registry.""" bm_class = registry.benchmark_registry.get_benchmark("gemm_quantized") self.assertEqual(bm_class, matmul.QuantizedGemmBenchmark) @@ -775,7 +780,7 @@ def test_generate_inputs_and_run_op(self, qd): self._setup_benchmark(quant_dtype=qd) self.assertEqual( self.bm.get_run_identifier(), - f"m_64_k_128_n_64_bfloat16_to_{qd}_out_bfloat16", + f"m_64_k_128_n_64_bfloat16_to_{qd}_dynamic_out_bfloat16", ) inputs = self.bm.generate_inputs() self.assertLen(inputs, 2) @@ -785,26 +790,134 @@ def test_generate_inputs_and_run_op(self, qd): self.assertEqual(out.dtype, jnp.bfloat16) quantize_fn = jax.jit( - matmul._quantize_per_channel_dynamic, # pylint: disable=protected-access - static_argnames=("reduce_axis", "quant_dtype"), + matmul._quantize_per_channel, # pylint: disable=protected-access + static_argnames=("reduce_axis", "quant_dtype", "scaling"), + ) + qa, _ = quantize_fn( + a, + reduce_axis=1, + quant_dtype=qd, + scaling=matmul.QuantScaling.DYNAMIC, + ) + qb, _ = quantize_fn( + b, + reduce_axis=0, + quant_dtype=qd, + scaling=matmul.QuantScaling.DYNAMIC, ) - qa = quantize_fn(a, reduce_axis=1, quant_dtype=qd) - qb = quantize_fn(b, reduce_axis=0, quant_dtype=qd) expected = jnp.matmul(qa.astype(jnp.float32), qb.astype(jnp.float32)) np.testing.assert_allclose(out.astype(jnp.float32), expected, rtol=2e-2) - def test_quantize_per_channel_dynamic_and_dtype_constraints(self): - """Tests int8 rounding and DTYPE_CONSTRAINTS validation.""" - x = jnp.array([[127.0, 0.6], [63.5, -63.5]], dtype=jnp.bfloat16) - q_int8 = ( - matmul._quantize_per_channel_dynamic( # pylint: disable=protected-access - x, reduce_axis=1, quant_dtype=jnp.int8 - ) + @parameterized.parameters( + ( + "dynamic", + False, + "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_dynamic_out_bfloat16", + 25690112.0, + ), + ( + "static", + False, + "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_static_out_bfloat16", + 3211264.0, + ), + ( + "dynamic", + True, + "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_dynamic_out_bfloat16_dequant", + 802816.0, + ), + ( + "static", + True, + "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_static_out_bfloat16_dequant", + 802816.0, + ), + ) + def test_scaling_and_dequantize_variants( + self, scaling, dequantize, expected_run_id, expected_val + ): + """Tests run IDs and numerical outputs across scaling and dequantize modes.""" + self._setup_benchmark(scaling=scaling, dequantize=dequantize) + self.assertEqual(self.bm.get_run_identifier(), expected_run_id) + inputs = self.bm.generate_inputs() + out = self.bm.run_op(*inputs) + self.assertEqual(out.shape, (64, 64)) + self.assertEqual(out.dtype, jnp.bfloat16) + self.assertTrue(bool(jnp.all(jnp.isfinite(out)))) + + # Verify exact numerical quantization and dequantization with dyadic inputs: + # A = 112.0 (dynamic sf_a = 0.25, qa = 448; static sf_a = 0.5, qa = 224) + # B = 56.0 (dynamic sf_b = 0.125, qb = 448; static sf_b = 0.5, qb = 112) + a = jnp.full((64, 128), 112.0, dtype=jnp.bfloat16) + b = jnp.full((128, 64), 56.0, dtype=jnp.bfloat16) + exact_out = self.bm.run_op(a, b) + np.testing.assert_allclose( + np.asarray(exact_out, dtype=np.float32), + np.full((64, 64), expected_val, dtype=np.float32), ) - self.assertEqual(q_int8.dtype, jnp.int8) - np.testing.assert_array_equal( - np.asarray(q_int8), np.array([[127, 1], [127, -127]], dtype=np.int8) + + def test_int8_quant_dtype_rounds_and_accumulates_int32(self): + """Tests that int8 quantization rounds to nearest integer and uses int32.""" + self._setup_benchmark( + m=2, + k=2, + n=2, + quant_dtype="int8", + scaling="dynamic", + dequantize=True, ) + # Row 0 of A has amax = 127.0 -> scale = 1.0; 0.6 rounds to 1 (not 0). + a = jnp.array([[127.0, 0.6], [63.5, -63.5]], dtype=jnp.bfloat16) + b = jnp.array([[127.0, 63.5], [127.0, -63.5]], dtype=jnp.bfloat16) + out = self.bm.run_op(a, b) + self.assertEqual(out.shape, (2, 2)) + self.assertEqual(out.dtype, jnp.bfloat16) + # Row 0 dot Col 0: q_a=[127, 1], q_b=[127, 127], sf_a=1.0, sf_b=1.0 + # -> (127*127 + 1*127) * 1.0 = 16256.0 (would be 16129.0 without round). + self.assertEqual(float(out[0, 0]), 16256.0) + + def test_static_scaling_clips_outliers_to_qmax(self): + """Tests that static scaling clips values exceeding 0.5 * q_max to q_max.""" + self._setup_benchmark( + m=2, + k=2, + n=2, + quant_dtype="float8_e4m3fn", + scaling="static", + dequantize=False, + ) + # Values > 224.0 exceed q_max (448.0) after dividing by static scale 0.5; + # without clipping to [-q_max, q_max], casting 1000.0 to float8_e4m3fn + # overflows to NaN. + a = jnp.array([[500.0, -500.0], [112.0, 0.0]], dtype=jnp.bfloat16) + b = jnp.array([[1.0, 0.0], [0.0, 1.0]], dtype=jnp.bfloat16) + out = self.bm.run_op(a, b) + self.assertTrue(bool(jnp.all(jnp.isfinite(out)))) + # q_a[0] clips to [448.0, -448.0] and q_b is [[2.0, 0.0], [0.0, 2.0]]. + np.testing.assert_allclose( + np.asarray(out, dtype=np.float32), + np.array([[896.0, -896.0], [448.0, 0.0]], dtype=np.float32), + ) + qa, sf_a = matmul._quantize_per_channel( # pylint: disable=protected-access + a, + reduce_axis=1, + quant_dtype="float8_e4m3fn", + scaling=matmul.QuantScaling.STATIC, + ) + np.testing.assert_allclose( + np.asarray(qa, dtype=np.float32), + np.array([[448.0, -448.0], [224.0, 0.0]], dtype=np.float32), + ) + np.testing.assert_allclose( + np.asarray(sf_a, dtype=np.float32), + np.array([[0.5], [0.5]], dtype=np.float32), + ) + + def test_params_validation(self): + """Tests that invalid scaling and quant_dtype values raise ValueError.""" + with self.assertRaisesRegex(ValueError, "Invalid scaling"): + matmul.QuantizedGemmParams(**{"scaling": "invalid"}) with self.assertRaises(ValueError): matmul.QuantizedGemmParams(quant_dtype="uint8") with self.assertRaises(ValueError): @@ -819,6 +932,7 @@ def test_quantize_per_channel_dynamic_and_dtype_constraints(self): ) def test_flops_bytes_and_compute_dtype(self): + """Tests FLOPs, bytes, and compute dtype reporting.""" self._setup_benchmark(m=64, k=128, n=64) self.assertEqual(self.bm.get_compute_dtype(), "float8_e4m3fn") self.assertEqual(self.bm.get_total_flops(), float(2 * 64 * 128 * 64)) @@ -828,6 +942,7 @@ def test_flops_bytes_and_compute_dtype(self): self.assertEqual(self.bm.get_total_bytes(), expected_bytes) def test_schema_coverage(self): + """Tests that benchmark output matches the report schema.""" self._setup_benchmark() test_report_utils.assert_schema_matches_output(self, self.bm)