diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index fca0bc44..7d0d10dc 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -272,9 +272,9 @@ metrics when `enable_checksum=True`. Evaluates end-to-end quantized matrix multiplication (`bfloat16` or `float32` inputs `A[M, K]` and `B[K, N]` quantized row-wise/column-wise to -`float8_e4m3fn`, `float8_e5m2`, or `int8` inside the timed step before -`lax.dot_general`, with dynamic or static scaling and optional output -dequantization). +`float8_e4m3fn`, `float8_e5m2`, or `int8`—or mixed-precision `FP8 × INT4` with +`rhs_quant_dtype="int4"`—inside the timed step before `lax.dot_general`, with +dynamic or static scaling and optional output dequantization). | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | @@ -282,7 +282,8 @@ dequantization). | `k` | `int` | `1024` | Contracting dimension `K` (columns of `A` and rows of `B`). | | `n` | `int` | `1024` | Matrix dimension `N` (columns of `B` and output matrix `C`). | | `in_dtype` | `str` | `"bfloat16"` | Unquantized input operand dtype (e.g., `bfloat16`, `float32`). | -| `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized matmul operand dtype (e.g., `float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. | +| `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized LHS (and default RHS) operand dtype (`float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. | +| `rhs_quant_dtype` | `str` | `` `quant_dtype` `` | Quantized RHS (`B`) operand dtype. Defaults to `quant_dtype`; also supports `"int4"` when `quant_dtype` is `float8_e4m3fn` or `float8_e5m2` (`W4A8`). | | `out_dtype` | `str` | `"bfloat16"` | Output tensor dtype (e.g., `bfloat16`, `float32`). | | `scaling` | `str` | `"dynamic"` | Scale calibration: `"dynamic"` (per-row/column `amax` computed on the fly) or `"static"` (fixed clip bound `0.5 * Q_MAX`, i.e., compile-time scale `0.5`). | | `dequantize` | `bool` | `False` | When `True`, multiply the matmul accumulator by the row and column dequantization scales before casting to `out_dtype`. | diff --git a/src/accelerator_microbenchmarks/benchmarks/matmul.py b/src/accelerator_microbenchmarks/benchmarks/matmul.py index d5936538..4906c164 100644 --- a/src/accelerator_microbenchmarks/benchmarks/matmul.py +++ b/src/accelerator_microbenchmarks/benchmarks/matmul.py @@ -771,6 +771,12 @@ class QuantizedGemmParams(MatmulShapeParams): DTYPE_CONSTRAINTS = { "in_dtype": _UNQUANTIZED_FLOAT_DTYPES, "quant_dtype": (jnp.float8_e4m3fn, jnp.float8_e5m2, jnp.int8), + "rhs_quant_dtype": ( + jnp.float8_e4m3fn, + jnp.float8_e5m2, + jnp.int8, + jnp.int4, + ), "out_dtype": _UNQUANTIZED_FLOAT_DTYPES, } @@ -778,8 +784,18 @@ class QuantizedGemmParams(MatmulShapeParams): default=dtypes.DType.FLOAT8_E4M3FN, metadata={ "help": ( - "Quantized operand data type fed to the MXU (e.g." - " 'float8_e4m3fn', 'int8')." + "Quantized LHS (and default RHS) operand data type fed to the" + " MXU (e.g. 'float8_e4m3fn', 'int8')." + ) + }, + ) + rhs_quant_dtype: dtypes.DType = base.dtype_field( + default=dtypes.DType.FLOAT8_E4M3FN, + metadata={ + "help": ( + "Quantized RHS (B) operand data type. Defaults to `quant_dtype`;" + " also supports 'int4' when `quant_dtype` is FP8" + " ('float8_e4m3fn' or 'float8_e5m2')." ) }, ) @@ -802,6 +818,40 @@ class QuantizedGemmParams(MatmulShapeParams): }, ) + def __post_init__(self): + if "rhs_quant_dtype" not in self._explicit_fields: + self.rhs_quant_dtype = self.quant_dtype + super().__post_init__() + if self.rhs_quant_dtype != self.quant_dtype and not ( + self.rhs_quant_dtype == dtypes.DType.INT4 + and self.quant_dtype in ( + dtypes.DType.FLOAT8_E4M3FN, + dtypes.DType.FLOAT8_E5M2, + ) + ): + raise ValueError( + "rhs_quant_dtype must match quant_dtype or be 'int4' paired with an" + " FP8 quant_dtype ('float8_e4m3fn' or 'float8_e5m2'), got" + f" quant_dtype='{self.quant_dtype}'," + f" rhs_quant_dtype='{self.rhs_quant_dtype}'." + ) + + def with_generation_defaults( + self, param_defaults: dict[str, Any] + ) -> base.BaseBenchmarkParams: + if ( + param_defaults + and "quant_dtype" in param_defaults + and "quant_dtype" not in self._explicit_fields + and "rhs_quant_dtype" not in param_defaults + and "rhs_quant_dtype" not in self._explicit_fields + ): + param_defaults = { + **param_defaults, + "rhs_quant_dtype": param_defaults["quant_dtype"], + } + return super().with_generation_defaults(param_defaults) + @registry.benchmark_registry.register("gemm_quantized", is_experimental=True) class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]): @@ -812,6 +862,7 @@ class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]): REPORT_SCHEMA: Sequence[tuple[str, Callable[[Any], str]]] = ( ("in_dtype", report.format_str), ("quant_dtype", report.format_str), + ("rhs_quant_dtype", report.format_str), ("out_dtype", report.format_str), ("m", report.format_str), ("k", report.format_str), @@ -826,7 +877,7 @@ class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]): ) def get_compute_dtype(self) -> str: - """Returns the quantized dtype, which determines the MXU peak TFLOPS.""" + """Returns the LHS quantized dtype, which determines the MXU peak TFLOPS.""" return str(self.config.quant_dtype) def setup(self): @@ -834,10 +885,12 @@ def setup(self): raise ValueError("Mesh not initialized.") cfg = self.config quant_dtype = dtypes.parse_dtype(cfg.quant_dtype) + rhs_quant_dtype = dtypes.parse_dtype(cfg.rhs_quant_dtype) out_dtype = dtypes.parse_dtype(cfg.out_dtype) - dot_acc_dtype = ( - jnp.int32 if jnp.issubdtype(quant_dtype, jnp.integer) else jnp.float32 + both_int = jnp.issubdtype(quant_dtype, jnp.integer) and jnp.issubdtype( + rhs_quant_dtype, jnp.integer ) + dot_acc_dtype = jnp.int32 if both_int else jnp.float32 @jax.jit def quant_gemm_fn(a, b): @@ -851,7 +904,7 @@ def quant_gemm_fn(a, b): qb, sf_b = _quantize_per_channel( b, reduce_axis=0, - quant_dtype=quant_dtype, + quant_dtype=rhs_quant_dtype, scaling=cfg.scaling, ) acc = jax.lax.dot_general( @@ -868,9 +921,14 @@ def quant_gemm_fn(a, b): def get_run_identifier(self) -> str: cfg = self.config + quant_str = ( + f"{cfg.quant_dtype}" + if cfg.rhs_quant_dtype == cfg.quant_dtype + else f"{cfg.quant_dtype}_rhs_{cfg.rhs_quant_dtype}" + ) run_id = ( f"m_{cfg.m}_k_{cfg.k}_n_{cfg.n}_" - f"{cfg.in_dtype}_to_{cfg.quant_dtype}_{cfg.scaling}_out_{cfg.out_dtype}" + f"{cfg.in_dtype}_to_{quant_str}_{cfg.scaling}_out_{cfg.out_dtype}" ) if cfg.dequantize: run_id += "_dequant" diff --git a/tests/test_matmul.py b/tests/test_matmul.py index 2a8e1682..8b746292 100644 --- a/tests/test_matmul.py +++ b/tests/test_matmul.py @@ -774,14 +774,22 @@ def test_benchmark_registered(self): bm_class = registry.benchmark_registry.get_benchmark("gemm_quantized") self.assertEqual(bm_class, matmul.QuantizedGemmBenchmark) - @parameterized.parameters("float8_e4m3fn", "int8") - def test_generate_inputs_and_run_op(self, qd): + @parameterized.parameters( + ("float8_e4m3fn", "float8_e4m3fn"), + ("float8_e5m2", "float8_e5m2"), + ("int8", "int8"), + ("float8_e4m3fn", "int4"), + ("float8_e5m2", "int4"), + ) + def test_generate_inputs_and_run_op(self, qd, rhs_qd): """Tests input generation and numerical output against quantized matmul.""" - self._setup_benchmark(quant_dtype=qd) + self._setup_benchmark(quant_dtype=qd, rhs_quant_dtype=rhs_qd) + expected_quant_str = qd if rhs_qd == qd else f"{qd}_rhs_{rhs_qd}" self.assertEqual( self.bm.get_run_identifier(), - f"m_64_k_128_n_64_bfloat16_to_{qd}_dynamic_out_bfloat16", + f"m_64_k_128_n_64_bfloat16_to_{expected_quant_str}_dynamic_out_bfloat16", ) + self.assertEqual(self.bm.get_compute_dtype(), qd) inputs = self.bm.generate_inputs() self.assertLen(inputs, 2) a, b = inputs @@ -802,7 +810,7 @@ def test_generate_inputs_and_run_op(self, qd): qb, _ = quantize_fn( b, reduce_axis=0, - quant_dtype=qd, + quant_dtype=rhs_qd, scaling=matmul.QuantScaling.DYNAMIC, ) expected = jnp.matmul(qa.astype(jnp.float32), qb.astype(jnp.float32)) @@ -810,35 +818,55 @@ def test_generate_inputs_and_run_op(self, qd): @parameterized.parameters( ( + "float8_e4m3fn", "dynamic", False, "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_dynamic_out_bfloat16", 25690112.0, ), ( + "float8_e4m3fn", "static", False, "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_static_out_bfloat16", 3211264.0, ), ( + "float8_e4m3fn", "dynamic", True, "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_dynamic_out_bfloat16_dequant", 802816.0, ), ( + "float8_e4m3fn", "static", True, "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_static_out_bfloat16_dequant", 802816.0, ), + ( + "int4", + "dynamic", + False, + "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_rhs_int4_dynamic_out_bfloat16", + 401408.0, + ), + ( + "int4", + "dynamic", + True, + "m_64_k_128_n_64_bfloat16_to_float8_e4m3fn_rhs_int4_dynamic_out_bfloat16_dequant", + 802816.0, + ), ) def test_scaling_and_dequantize_variants( - self, scaling, dequantize, expected_run_id, expected_val + self, rhs_qd, scaling, dequantize, expected_run_id, expected_val ): """Tests run IDs and numerical outputs across scaling and dequantize modes.""" - self._setup_benchmark(scaling=scaling, dequantize=dequantize) + self._setup_benchmark( + rhs_quant_dtype=rhs_qd, scaling=scaling, dequantize=dequantize + ) self.assertEqual(self.bm.get_run_identifier(), expected_run_id) inputs = self.bm.generate_inputs() out = self.bm.run_op(*inputs) @@ -876,6 +904,22 @@ def test_int8_quant_dtype_rounds_and_accumulates_int32(self): # Row 0 dot Col 0: q_a=[127, 1], q_b=[127, 127], sf_a=1.0, sf_b=1.0 # -> (127*127 + 1*127) * 1.0 = 16256.0 (would be 16129.0 without round). self.assertEqual(float(out[0, 0]), 16256.0) + x_int4 = jnp.array([[7.0, 3.6], [3.5, -3.5]], dtype=jnp.bfloat16) + q_int4, sf_int4 = matmul._quantize_per_channel( # pylint: disable=protected-access + x_int4, + reduce_axis=1, + quant_dtype=jnp.int4, + scaling=matmul.QuantScaling.DYNAMIC, + ) + self.assertEqual(q_int4.dtype, jnp.int4) + np.testing.assert_array_equal( + np.asarray(q_int4.astype(jnp.int8)), + np.array([[7, 4], [7, -7]], dtype=np.int8), + ) + np.testing.assert_allclose( + np.asarray(sf_int4, dtype=np.float32), + np.array([[1.0], [0.5]], dtype=np.float32), + ) def test_static_scaling_clips_outliers_to_qmax(self): """Tests that static scaling clips values exceeding 0.5 * q_max to q_max.""" @@ -916,12 +960,46 @@ def test_static_scaling_clips_outliers_to_qmax(self): def test_params_validation(self): """Tests that invalid scaling and quant_dtype values raise ValueError.""" + # Default rhs_quant_dtype tracks quant_dtype when omitted, including across + # with_generation_defaults(). + self.assertEqual( + matmul.QuantizedGemmParams(quant_dtype="int8").rhs_quant_dtype, + "int8", + ) + self.assertEqual( + matmul.QuantizedGemmParams(quant_dtype="float8_e5m2").rhs_quant_dtype, + "float8_e5m2", + ) + default_with_int8 = matmul.QuantizedGemmParams().with_generation_defaults( + {"quant_dtype": "int8"} + ) + self.assertEqual(default_with_int8.quant_dtype, "int8") + self.assertEqual(default_with_int8.rhs_quant_dtype, "int8") + explicit_int4_with_e5m2 = matmul.QuantizedGemmParams( + rhs_quant_dtype="int4" + ).with_generation_defaults({"quant_dtype": "float8_e5m2"}) + self.assertEqual(explicit_int4_with_e5m2.quant_dtype, "float8_e5m2") + self.assertEqual(explicit_int4_with_e5m2.rhs_quant_dtype, "int4") with self.assertRaisesRegex(ValueError, "Invalid scaling"): matmul.QuantizedGemmParams(**{"scaling": "invalid"}) with self.assertRaises(ValueError): matmul.QuantizedGemmParams(quant_dtype="uint8") with self.assertRaises(ValueError): matmul.QuantizedGemmParams(quant_dtype="bfloat16") + with self.assertRaises(ValueError): + matmul.QuantizedGemmParams(rhs_quant_dtype="bfloat16") + with self.assertRaises(ValueError): + matmul.QuantizedGemmParams(**{"rhs_quant_dtype": None}) + with self.assertRaisesRegex( + ValueError, "rhs_quant_dtype must match quant_dtype or be 'int4'" + ): + matmul.QuantizedGemmParams(quant_dtype="int8", rhs_quant_dtype="int4") + with self.assertRaisesRegex( + ValueError, "rhs_quant_dtype must match quant_dtype or be 'int4'" + ): + matmul.QuantizedGemmParams( + quant_dtype="float8_e4m3fn", rhs_quant_dtype="int8" + ) with self.assertRaises(ValueError): matmul.QuantizedGemmParams(in_dtype="float64") with self.assertRaises(ValueError):