Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions docs/BENCHMARKS.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ This guide provides the reference for **TPUMS (TPU Microbenchmark Suite)** param
- [2.1.1 GEMM Throttling (`gemm_throttling`) [experimental]](#211-gemm-throttling-gemm_throttling-experimental)
- [2.1.2 Chained Matrix Multiplication & Determinism Check (`gemm_chained`) [experimental]](#212-chained-matrix-multiplication--determinism-check-gemm_chained-experimental)
- [2.1.3 GEMM Pulse (`gemm_pulse`) [experimental]](#213-gemm-pulse-gemm_pulse-experimental)
- [2.1.4 Quantized Matrix Multiplication (`gemm_quantized`) [experimental]](#214-quantized-matrix-multiplication-gemm_quantized-experimental)
- [2.2 HBM Memory Bandwidth (`hbm`)](#22-hbm-memory-bandwidth-hbm)
- [2.3 Host I/O Bandwidth (`host_to_device`, `device_to_host`)](#23-host-io-bandwidth-host_to_device-device_to_host)
- [2.4 Device-to-Device Transfers (`device_to_device`)](#24-inter-chip-interconnect-device_to_device)
Expand Down Expand Up @@ -252,6 +253,24 @@ final cycle ends with an `idle_s` pause followed by traced
`gemm_throttling` soaking and thermal metrics, plus all `gemm_chained` checksum
metrics when `enable_checksum=True`.

<a id="214-quantized-matrix-multiplication-gemm_quantized-experimental"></a>
#### 2.1.4 Quantized Matrix Multiplication (`gemm_quantized`) [experimental]

Evaluates end-to-end quantized matrix multiplication (`bfloat16` or `float32`
inputs `A[M, K]` and `B[K, N]` dynamically quantized row-wise/column-wise to
`float8_e4m3fn`, `float8_e5m2`, or `int8` inside the timed step before
`lax.dot_general`).

| Parameter | Type | Default | Description |
| :--- | :--- | :--- | :--- |
| `m` | `int` | `1024` | Matrix dimension `M` (rows of `A` and output matrix `C`). |
| `k` | `int` | `1024` | Contracting dimension `K` (columns of `A` and rows of `B`). |
| `n` | `int` | `1024` | Matrix dimension `N` (columns of `B` and output matrix `C`). |
| `in_dtype` | `str` | `"bfloat16"` | Unquantized input operand dtype (e.g., `bfloat16`, `float32`). |
| `quant_dtype` | `str` | `"float8_e4m3fn"` | Quantized matmul operand dtype (e.g., `float8_e4m3fn`, `float8_e5m2`, `int8`). Selects the hardware peak TFLOPS ceiling for roofline efficiency. |
| `out_dtype` | `str` | `"bfloat16"` | Output tensor dtype (e.g., `bfloat16`, `float32`). |
| `seed` | `int` | `0` | PRNG seed for initializing input tensors in HBM. |

---

<a id="22-hbm-memory-bandwidth-hbm"></a>
Expand Down
194 changes: 168 additions & 26 deletions src/accelerator_microbenchmarks/benchmarks/matmul.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,16 @@
import jax.numpy as jnp
import numpy as np

# Unquantized floating-point dtypes supported for inputs and outputs.
_UNQUANTIZED_FLOAT_DTYPES = (
jnp.bfloat16,
jnp.float32,
)


@dataclasses.dataclass
class GemmParams(base.BaseBenchmarkParams):
"""Configuration parameters for GEMM benchmark."""
class MatmulShapeParams(base.BaseBenchmarkParams):
"""Shared matrix shape, dtype, and seed parameters for GEMM benchmarks."""

m: int = dataclasses.field(
default=1024,
Expand Down Expand Up @@ -52,11 +58,16 @@ class GemmParams(base.BaseBenchmarkParams):
"help": "Output accumulation/result data type (e.g. bfloat16)."
},
)

seed: int = dataclasses.field(
default=0,
metadata={"min": 0, "help": "Random seed for tensor initialization."},
)


@dataclasses.dataclass
class GemmParams(MatmulShapeParams):
"""Configuration parameters for GEMM benchmark."""

use_scaling_factors: bool = dataclasses.field(
default=False,
metadata={"help": "Apply row-wise/column-wise FP8 scaling factors."},
Expand Down Expand Up @@ -134,6 +145,29 @@ def calculate_throughput_metrics(
f"{prefix}_tflops_per_device": tflops_per_sec,
}

def _replicated_normal(
self, key: jax.Array, shape: tuple[int, ...], dtype: Any
) -> jax.Array:
"""Samples a normal array replicated on every mesh device.

Generating under `jit` with a replicated `out_shardings` (as in `hbm.py`)
produces each shard on its own device, and the key is passed as a host
array, so no data moves between devices. The result equals eager sampling
with the same key.
"""
# Replicated sharding for computation ops to avoid discrepancies
if self.mesh is None:
raise ValueError("Mesh not initialized.")
replicated_sharding = jax.sharding.NamedSharding(
self.mesh, jax.sharding.PartitionSpec(None, None)
)
generate = jax.jit(
_sample_normal,
static_argnums=(1, 2),
out_shardings=replicated_sharding,
)
return generate(np.asarray(key), shape, dtype)


@registry.benchmark_registry.register("gemm", aliases=["gemm_generalized"])
class GeneralizedGemmBenchmark(BaseGemmBenchmark[TGemmParams]):
Expand Down Expand Up @@ -207,29 +241,6 @@ def get_run_identifier(self) -> str:
run_identifier += "_sf"
return run_identifier

def _replicated_normal(
self, key: jax.Array, shape: tuple[int, ...], dtype: Any
) -> jax.Array:
"""Samples a normal array replicated on every mesh device.

Generating under `jit` with a replicated `out_shardings` (as in `hbm.py`)
produces each shard on its own device, and the key is passed as a host
array, so no data moves between devices. The result equals eager sampling
with the same key.
"""
# Replicated sharding for computation ops to avoid discrepancies
if self.mesh is None:
raise ValueError("Mesh not initialized.")
replicated_sharding = jax.sharding.NamedSharding(
self.mesh, jax.sharding.PartitionSpec(None, None)
)
generate = jax.jit(
_sample_normal,
static_argnums=(1, 2),
out_shardings=replicated_sharding,
)
return generate(np.asarray(key), shape, dtype)

def generate_inputs(self) -> tuple[Any, ...]:
(
m,
Expand Down Expand Up @@ -696,3 +707,134 @@ def calculate_metrics(self, times_ms: list[float]) -> dict[str, Any]:
if self.config.enable_checksum:
return super().calculate_metrics(times_ms)
return super(GemmChainedBenchmark, self).calculate_metrics(times_ms)


def _quantize_per_channel_dynamic(
x: jax.Array,
*,
reduce_axis: int,
quant_dtype: jax.typing.DTypeLike,
) -> jax.Array:
"""Dynamically quantizes `x` per channel along `reduce_axis`."""
q_dtype = jnp.dtype(quant_dtype)
q_max = float(dtypes.get_dtype_max(q_dtype))
x_f32 = x.astype(jnp.float32)
amax = jnp.max(jnp.abs(x_f32), axis=reduce_axis, keepdims=True)
inv_scale_f32 = q_max / jnp.maximum(amax, 1e-7)
q = x_f32 * inv_scale_f32
if jnp.issubdtype(q_dtype, jnp.integer):
q = jnp.clip(jnp.round(q), -q_max, q_max)
else:
q = jnp.clip(q, -q_max, q_max)
return q.astype(q_dtype)


@dataclasses.dataclass
class QuantizedGemmParams(MatmulShapeParams):
"""Configuration parameters for the `gemm_quantized` microbenchmark."""

DTYPE_CONSTRAINTS = {
"in_dtype": _UNQUANTIZED_FLOAT_DTYPES,
"quant_dtype": (jnp.float8_e4m3fn, jnp.float8_e5m2, jnp.int8),
"out_dtype": _UNQUANTIZED_FLOAT_DTYPES,
}

quant_dtype: dtypes.DType = base.dtype_field(
default=dtypes.DType.FLOAT8_E4M3FN,
metadata={
"help": (
"Quantized operand data type fed to the MXU (e.g."
" 'float8_e4m3fn', 'int8')."
)
},
)


@registry.benchmark_registry.register("gemm_quantized", is_experimental=True)
class QuantizedGemmBenchmark(BaseGemmBenchmark[QuantizedGemmParams]):
"""[EXPERIMENTAL] GEMM with dynamic per-channel quantization of A and B."""

Config = QuantizedGemmParams # pylint: disable=invalid-name
requires_ici: bool = False
REPORT_SCHEMA: Sequence[tuple[str, Callable[[Any], str]]] = (
("in_dtype", report.format_str),
("quant_dtype", report.format_str),
("out_dtype", report.format_str),
("m", report.format_str),
("k", report.format_str),
("n", report.format_str),
("total_flops", report.format_2f),
("wall_clock_p50_ms", report.format_4f),
("wall_clock_tflops_per_chip", report.format_2f),
("xprof_p50_ms", report.format_4f),
("xprof_tflops_per_chip", report.format_2f),
)

def get_compute_dtype(self) -> str:
"""Returns the quantized dtype, which determines the MXU peak TFLOPS."""
return str(self.config.quant_dtype)

def setup(self):
if self.mesh is None:
raise ValueError("Mesh not initialized.")
cfg = self.config
quant_dtype = dtypes.parse_dtype(cfg.quant_dtype)
out_dtype = dtypes.parse_dtype(cfg.out_dtype)
dot_acc_dtype = (
jnp.int32 if jnp.issubdtype(quant_dtype, jnp.integer) else jnp.float32
)

@jax.jit
def quant_gemm_fn(a, b):
with jax.named_scope(constants.MARKER):
qa = _quantize_per_channel_dynamic(
a,
reduce_axis=1,
quant_dtype=quant_dtype,
)
qb = _quantize_per_channel_dynamic(
b,
reduce_axis=0,
quant_dtype=quant_dtype,
)
acc = jax.lax.dot_general(
qa,
qb,
dimension_numbers=(((1,), (0,)), ((), ())),
preferred_element_type=dot_acc_dtype,
)
return acc.astype(out_dtype)

self._jit_fn = quant_gemm_fn

def get_run_identifier(self) -> str:
cfg = self.config
return (
f"m_{cfg.m}_k_{cfg.k}_n_{cfg.n}_"
f"{cfg.in_dtype}_to_{cfg.quant_dtype}_out_{cfg.out_dtype}"
)

def generate_inputs(self) -> tuple[Any, ...]:
cfg = self.config
in_dtype = dtypes.parse_dtype(cfg.in_dtype)
key_a, key_b = jax.random.split(jax.random.PRNGKey(cfg.seed))
a = self._replicated_normal(key_a, (cfg.m, cfg.k), in_dtype)
b = self._replicated_normal(key_b, (cfg.k, cfg.n), in_dtype)
return (a, b)

def run_op(self, *args) -> jax.Array:
if self._jit_fn is None:
raise ValueError("JIT function not initialized.")
return self._jit_fn(*args)

def get_total_bytes(self) -> float:
"""Returns 1-pass algorithmic HBM bytes (read A, read B, write OUT)."""
cfg = self.config
m, k, n = cfg.m, cfg.k, cfg.n
in_bytes = dtypes.get_dtype_bytes(cfg.in_dtype)
out_bytes = dtypes.get_dtype_bytes(cfg.out_dtype)
return float((m * k + k * n) * in_bytes + m * n * out_bytes)

def get_total_flops(self) -> float:
m, k, n = self.config.m, self.config.k, self.config.n
return float(2 * m * n * k)
14 changes: 1 addition & 13 deletions src/accelerator_microbenchmarks/op_flags.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,6 @@ gemm_flags: &gemm_flags
- "--xla_tpu_vmem_scavenging_mode=NONE"
- "--xla_tpu_dvfs_p_state=7"

gemm_numerics_flags: &gemm_numerics_flags
flags:
- "--xla_tpu_enable_async_collective_fusion=false"
- "--xla_tpu_enable_async_collective_fusion_multiple_steps=true"
- "--xla_tpu_overlap_compute_collective_tc=true"
- "--xla_enable_async_all_gather=true"
- "--xla_enable_async_collective_permute=true"
- "--xla_tpu_enable_all_experimental_scheduler_features=true"
- "--xla_tpu_accumulate_into_mrb=true"
- "--xla_tpu_scoped_vmem_limit_kib=65536"
- "--xla_tpu_allow_conv_input_fusion_with_downcast_convert=true"
- "--xla_tpu_dvfs_p_state=7"

all_reduce:
flags:
- "--xla_jf_debug_level=3"
Expand Down Expand Up @@ -123,6 +110,7 @@ tokamax_splash_attention:
- "--xla_tpu_dvfs_p_state=7"

gemm: *gemm_flags
gemm_quantized: *gemm_flags
gemm_throttling: *gemm_flags
gemm_pulse: *gemm_flags
gemm_chained: *gemm_flags
Expand Down
5 changes: 5 additions & 0 deletions tests/test_benchmark_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def test_load_all_benchmarks_expected_benchmarks(self):
"gemm_throttling",
"gemm_pulse",
"gemm_chained",
"gemm_quantized",
"hbm",
"hbm_bandwidth",
"attention_flashed",
Expand Down Expand Up @@ -93,6 +94,9 @@ def test_experimental_benchmarks(self):
)
self.assertTrue(registry.benchmark_registry.is_experimental("gemm_chained"))
self.assertTrue(registry.benchmark_registry.is_experimental("gemm_pulse"))
self.assertTrue(
registry.benchmark_registry.is_experimental("gemm_quantized")
)

def test_all_benchmarks_have_valid_config(self):
"""Verify that every registered benchmark defines a Config dataclass."""
Expand Down Expand Up @@ -165,6 +169,7 @@ def test_benchmarks_without_ici(self):
"gemm_throttling",
"gemm_pulse",
"gemm_chained",
"gemm_quantized",
"hbm",
"hbm_bandwidth",
"host_to_device",
Expand Down
Loading
Loading