From 2d7a8ac1c7f181f7f7930690c2ef304057ea16f8 Mon Sep 17 00:00:00 2001 From: Gina Sitaraman Date: Fri, 7 Aug 2026 15:38:17 -0500 Subject: [PATCH 1/3] Generalize scripts for different sites with one file to edit for the user, making the process simpler --- MLExamples/PyTorch_Profiling/.gitignore | 9 + .../README_ROCM_NIGHTLY_TESTING.md | 122 +++++++----- .../ROCM_PYTORCH_PIP_VENV_SETUP.md | 108 +++++------ MLExamples/PyTorch_Profiling/env.sh | 116 ++++++++++++ .../PyTorch_Profiling/install_rocm_pytorch.sh | 173 ++++++++++++++++++ .../PyTorch_Profiling/local.env.example | 79 ++++++++ .../modulefiles/rocm-pytorch-pip/TEMPLATE.lua | 34 ++++ .../slurm_single_process_noprofile.sh | 2 + .../slurm_single_process_analyze.sh | 26 ++- .../slurm_single_process_profile.sh | 2 + .../slurm_single_process.sh | 2 + .../rocprofv3/slurm_single_process_kernels.sh | 4 +- .../rocprofv3/slurm_single_process_traces.sh | 4 +- .../slurm_single_process.sh | 28 ++- MLExamples/PyTorch_Profiling/run_all.sh | 69 +++++++ MLExamples/PyTorch_Profiling/setup_rocm.sh | 56 ++++-- 16 files changed, 701 insertions(+), 133 deletions(-) create mode 100644 MLExamples/PyTorch_Profiling/.gitignore create mode 100755 MLExamples/PyTorch_Profiling/env.sh create mode 100755 MLExamples/PyTorch_Profiling/install_rocm_pytorch.sh create mode 100644 MLExamples/PyTorch_Profiling/local.env.example create mode 100644 MLExamples/PyTorch_Profiling/modulefiles/rocm-pytorch-pip/TEMPLATE.lua create mode 100755 MLExamples/PyTorch_Profiling/run_all.sh mode change 100644 => 100755 MLExamples/PyTorch_Profiling/setup_rocm.sh diff --git a/MLExamples/PyTorch_Profiling/.gitignore b/MLExamples/PyTorch_Profiling/.gitignore new file mode 100644 index 00000000..ef782b66 --- /dev/null +++ b/MLExamples/PyTorch_Profiling/.gitignore @@ -0,0 +1,9 @@ +# Site configs hold per-user paths and allocations. +local.env +local.env.* +!local.env.example + +# Job output. +logs/ +*.out +*.err diff --git a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md index 6460fe2c..200dbb8b 100644 --- a/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md +++ b/MLExamples/PyTorch_Profiling/README_ROCM_NIGHTLY_TESTING.md @@ -3,9 +3,9 @@ This guide walks you through testing a **nightly ROCm build** together with PyTorch on an AMD GPU. You will: -1. Build a self-contained Python virtual environment with nightly ROCm, PyTorch, - and the ROCm profilers. -2. Create a `setup_rocm.sh` script that activates that environment. +1. Describe your cluster in a single `local.env` file. +2. Build a self-contained Python virtual environment with nightly ROCm, PyTorch, + and the ROCm profilers, exposed as an Lmod module. 3. Run the CIFAR-100 training workload through each profiling tool using the provided SLURM scripts, and check that everything works end to end. @@ -13,46 +13,65 @@ The workload itself is the same `train_cifar_100.py` used throughout this directory — a small vision model trained on CIFAR-100. It is intentionally short so that a nightly build can be validated quickly. -> This guide targets an AMD MI300A GPU (`gfx942`) on a SLURM cluster. For a -> different GPU, change the architecture (`device-gfx942` / `--arch`) and the -> SLURM `--partition` in the scripts accordingly. +The scripts run unmodified on any SLURM cluster and on any supported GPU: +everything machine-specific lives in `local.env`, and no script needs editing. --- -## Step 1 — Build the virtual environment +## Step 1 — Describe your site in `local.env` -Follow [`ROCM_PYTORCH_PIP_VENV_SETUP.md`](./ROCM_PYTORCH_PIP_VENV_SETUP.md) to -create the `rocm-pytorch-pip` venv. In short, it: +```bash +cp local.env.example local.env +$EDITOR local.env +``` + +`env.sh` sources it, or `$SITE_ENV` if set, and derives every path from it. + +| Key | Meaning | +|-----|---------| +| `PROJECT` / `PARTITION` | SLURM account and partition, exported as `SBATCH_ACCOUNT` / `SBATCH_PARTITION` | +| `VENV_BASE` | Shared directory holding the venvs, modulefiles and examples | +| `GPU_ARCH` / `ROOFLINE_ARCH` | Wheel extra `device-`, and the roofline extractor's counter set | +| `ROCM_VERSION` / `ROCM_INDEX_URL` | Nightly build to validate, and the wheel index | +| `PYTHON_MODULE` / `BASE_PYTHON` | Base interpreter used to create the venv | +| `LMOD_INIT` | Lmod init script, for sites where `module` is undefined in non-login shells | +| `PROXY` | Outbound proxy, for sites whose nodes have no direct internet | +| `MIOPEN_LOCAL_DB` / `MIOPEN_TMP_BASE` | Keep MIOpen's SQLite databases on node-local storage | + +Two worked examples ship in `local.env.example`: OLCF Frontier (MI250X, Lustre, +proxied compute nodes) and AAC6 (MI300A, NFS, direct internet). -- creates a venv under `${VENV_BASE}/venvs/rocm-pytorch-pip` (`VENV_BASE` - defaults to your home directory), -- installs nightly ROCm + PyTorch + profilers from the multi-arch nightly index, -- installs `transformers` (required by the training script), -- runs `rocm-sdk init` to extract the development headers and device code. +`VENV_BASE` must be visible from **both** the login and the compute nodes, since +the login node builds the venv and the jobs read it. Prefer the fastest shared +filesystem available; node-local paths such as `/tmp` or `/dev/shm` do not work +however fast they are. -> **Tip:** Point `VENV_BASE` at node-local/fast storage (not a shared NFS home). -> The venv holds hundreds of MB of ROCm/PyTorch libraries, and loading them from -> NFS on every job noticeably slows startup. +To validate a different nightly later, change `ROCM_VERSION` and rebuild. -To test a **specific nightly**, set the ROCm version pin in that guide, e.g.: +## Step 2 — Build the environment + +Run this **on a login node** (one with internet access): ```bash -ROCM_VERSION=7.15.0a20260721 +bash install_rocm_pytorch.sh ``` -Change this value to the nightly date you want to validate. +It is idempotent, and it: -## Step 2 — Verify `setup_rocm.sh` +- creates the venv under `${VENV_BASE}/venvs/rocm-pytorch-pip` and installs + nightly ROCm + PyTorch + profilers from the multi-arch nightly index, plus + `transformers` (required by the training script), +- runs `rocm-sdk init` to extract the development headers and device code, +- generates an Lmod modulefile at + `${VENV_BASE}/modulefiles/rocm-pytorch-pip/${ROCM_VERSION}.lua`, +- pre-stages everything the jobs need, so compute nodes need no internet: + `rooflineExtractor` and its requirements, the isolated + `rocprof-compute analyze` venv (`numpy==1.26.4`), and the CIFAR-100 dataset. -The SLURM scripts in each sub-directory activate the environment by sourcing -`../setup_rocm.sh` — i.e. the `setup_rocm.sh` shipped in **this** -(`PyTorch_Profiling/`) directory. It is already provided; just verify (and edit -if needed) that its `VENV` points at the venv you built in Step 1. Its full -contents, and a GPU-node sanity check, are covered in steps 5-7 of -[`ROCM_PYTORCH_PIP_VENV_SETUP.md`](./ROCM_PYTORCH_PIP_VENV_SETUP.md). +[`ROCM_PYTORCH_PIP_VENV_SETUP.md`](./ROCM_PYTORCH_PIP_VENV_SETUP.md) walks +through the same venv build by hand, for adapting it or debugging a failure. -Once verified, a quick check that the nightly build runs GPU kernels through -PyTorch: +Confirm the nightly runs GPU kernels through PyTorch before profiling: ```bash source setup_rocm.sh @@ -60,6 +79,8 @@ srun -n1 --gpus=1 python3 -c "import torch; print('torch', torch.__version__); \ x = torch.ones(4, device='cuda:0'); print('device ok:', (x+1).sum().item())" ``` +`setup_rocm.sh` loads the generated module when it exists and otherwise +activates the venv directly; the SLURM scripts source it as `../setup_rocm.sh`. If you see `device ok:`, you are ready to profile. ## Step 3 — Run the SLURM scripts @@ -69,8 +90,13 @@ Each sub-directory contains a single-process SLURM script that sources under one tool. All of them use a single GPU and a short run (`--batch-size 32 --max-steps 5`) so a nightly can be checked quickly. -**Submit each script from its own directory** (the scripts use -`SLURM_SUBMIT_DIR` to locate themselves): +`run_all.sh` submits the whole suite from a login node, each script from its own +directory, chaining the analyze job after its profile job with +`--dependency=afterok`. It prints one ` ` line per submission: + +```bash +bash run_all.sh +``` | Tool | Directory | Script | What it produces | |------|-----------|--------|------------------| @@ -82,14 +108,14 @@ under one tool. All of them use a single GPU and a short run | ROCm Systems Profiler | `rocm-systems-profiler/` | `slurm_single_process.sh` | Sampling profile + trace under `rocprofsys-python3-output/`. | | Roofline Extractor | `roofline-extractor/` | `slurm_single_process.sh` | Per-kernel roofline analysis + interactive HTML plot under `output/`. | -Example (baseline sanity check first, then a profiler): +To submit one tool on its own, do it **from that tool's directory** (the scripts +use `SLURM_SUBMIT_DIR` to locate themselves), having sourced `env.sh` so the +account and partition reach `sbatch`: ```bash +source env.sh cd no-profiling sbatch slurm_single_process_noprofile.sh - -cd ../rocprofv3 -sbatch slurm_single_process_kernels.sh ``` Check job status and output: @@ -102,10 +128,10 @@ squeue --me ### Analyzing the results > **Note — running `rocprof-compute analyze`:** run it only **after** its -> profile job has finished (the counter database must exist). Because it needs -> `numpy==1.26.4` (vs the shared venv's `numpy>=2.0`), the analysis script uses -> its own isolated venv (`~/venvs/rocprof-compute-analyze`) and never touches -> the shared venv. +> profile job has finished (the counter database must exist); `run_all.sh` +> chains it for you. Because it needs `numpy==1.26.4` (vs the shared venv's +> `numpy>=2.0`), the analysis script uses its own isolated venv +> (`${VENV_BASE}/venvs/rocprof-compute-analyze`) and never touches the shared venv. - **ROCm Compute Profiler:** submit the companion analysis job from `rocm-compute-profiler/` (it locates the workload and runs the analysis for you): @@ -126,9 +152,19 @@ sbatch slurm_single_process_analyze.sh ## Notes -- If a job fails to start, check the SLURM `--partition` and time limits in the - script headers match your cluster. +- **Account and partition are not in the `#SBATCH` headers.** SLURM parses those + directives before the script runs, so they cannot expand variables. `env.sh` + exports `SBATCH_ACCOUNT` and `SBATCH_PARTITION` instead, which `sbatch` honours + at submit time. Source `env.sh` (or use `run_all.sh`) before submitting by hand. +- **MIOpen on NFS/Lustre.** MIOpen's SQLite perf and kernel databases need real + POSIX file locking, and `conv2d` fails with + `RuntimeError: miopenStatusInternalError` without it. `setup_rocm.sh` points + `MIOPEN_USER_DB_PATH` and `MIOPEN_CUSTOM_CACHE_DIR` at node-local storage; set + `MIOPEN_LOCAL_DB=0` if your shared filesystem locks correctly. +- **Pre-fetch the dataset with `download_only_nogpus.py`.** + `train_cifar_100.py --download-only` calls `dist.init_process_group("nccl", ...)` + even in download-only mode, which hangs on a GPU-less login node. +- If a job fails to start, check the time limits in the script headers and the + `PARTITION` in `local.env` against your cluster's limits. - The scripts derive a per-job rendezvous port from the SLURM job ID, so multiple jobs can share a node without port collisions. -- To validate a different nightly, rebuild the venv (Step 1) with a new - `ROCM_VERSION` and re-run the scripts. diff --git a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md index d8e41c8b..2d0a37bd 100644 --- a/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md +++ b/MLExamples/PyTorch_Profiling/ROCM_PYTORCH_PIP_VENV_SETUP.md @@ -1,46 +1,47 @@ -# Creating the `rocm-pytorch-pip` venv (nightly ROCm + PyTorch, MI300A / gfx942) - -This guide walks you through building a Python virtual environment with ROCm, -PyTorch, and the ROCm profiling tools, so you can train and profile a model on -an AMD GPU. Follow the steps in order. +# Creating the `rocm-pytorch-pip` venv (nightly ROCm + PyTorch) +This guide builds a Python virtual environment holding ROCm, PyTorch, and the +ROCm profiling tools, so you can train and profile a model on an AMD GPU. Everything installs from pip into one self-contained venv, so PyTorch and the -profilers use the same ROCm. Each step below is a single command block you can -copy and run. +profilers use the same ROCm. + +> `install_rocm_pytorch.sh` performs every step below and pre-stages the test +> jobs. Follow the steps by hand only to adapt the build or debug a failure; see +> [`README_ROCM_NIGHTLY_TESTING.md`](./README_ROCM_NIGHTLY_TESTING.md) for the +> scripted path. -> This guide targets an AMD MI300A GPU (`gfx942`). If you have a different GPU, -> change `device-gfx942` to your architecture. +The commands use the settings from `local.env`, so create one first and let +`env.sh` load it. Nothing below is specific to a cluster or a GPU: + +```bash +cp local.env.example local.env +$EDITOR local.env # VENV_BASE, GPU_ARCH, ROCM_VERSION at minimum +source env.sh +``` --- ## 1. Create and activate the venv -First choose where the venv should live. Set `VENV_BASE` to the directory that -will hold the `venvs` folder (defaults to your home directory). Keep this shell -open for the remaining steps, which reuse the variable. - -> **Tip:** Prefer node-local/fast storage (e.g. a local NVMe scratch path) over -> a shared NFS home directory. The venv holds hundreds of MB of ROCm/PyTorch -> shared libraries, and loading them from NFS on every job noticeably slows -> startup. +`VENV_BASE` holds the `venvs` folder. It must be visible from **both** the login +node that builds the venv and the compute nodes that run the jobs, so use the +fastest *shared* filesystem available (a parallel filesystem where there is one). +Node-local paths such as `/tmp` or `/dev/shm` cannot work here, however fast they +are: the compute node would not see what the login node wrote. ```bash -VENV_BASE=~ mkdir -p "${VENV_BASE}/venvs" -python -m venv "${VENV_BASE}/venvs/rocm-pytorch-pip" -source "${VENV_BASE}/venvs/rocm-pytorch-pip/bin/activate" +python3 -m venv "${VENV}" +source "${VENV}/bin/activate" ``` ## 2. Install ROCm + PyTorch from the nightly multi-arch index ```bash -# Pin the nightly ROCm version once and reuse it everywhere below. -ROCM_VERSION=7.15.0a20260721 - -pip install --index-url https://rocm.nightlies.amd.com/whl-multi-arch/ \ - "rocm[profiler,devel,libraries,device-gfx942]==${ROCM_VERSION}" \ - "torch[device-gfx942]" \ - "torchvision[device-gfx942]" +pip install --index-url "${ROCM_INDEX_URL}" \ + "rocm[profiler,devel,libraries,device-${GPU_ARCH}]==${ROCM_VERSION}" \ + "torch[device-${GPU_ARCH}]" \ + "torchvision[device-${GPU_ARCH}]" ``` The `rocm[...]` extras pull in the pieces this workflow needs: @@ -48,7 +49,12 @@ The `rocm[...]` extras pull in the pieces this workflow needs: `rocprof-sys` (bundled `_rocm_profiler`) - `devel` — development package (headers/device code, extracted in step 4) - `libraries` — math libraries (hipBLAS, rocBLAS, ...) -- `device-gfx942` — the GPU-arch kernels for MI300A +- `device-${GPU_ARCH}` — the GPU-arch kernels (`gfx942` for MI300A, `gfx90a` for + MI250X); see the index for the architectures a given nightly ships + +To validate a different nightly, change `ROCM_VERSION` in `local.env` and +rebuild. Available versions are listed at the index URL itself, +. ## 3. Install `transformers` (required by the training script) @@ -62,7 +68,7 @@ required. ## 4. Extract development headers and device code ```bash -"${VENV_BASE}/venvs/rocm-pytorch-pip/bin/rocm-sdk" init +"${VENV}/bin/rocm-sdk" init ``` `rocm-sdk init` unpacks the `devel` payload (headers, LLVM device bitcode) into @@ -71,44 +77,24 @@ and the paths `setup_rocm.sh` points at next. --- -## 5. Verify `setup_rocm.sh` +## 5. Activate with `setup_rocm.sh` -The repo already ships `setup_rocm.sh` in `MLExamples/PyTorch_Profiling/` (the -SLURM scripts source it as `../setup_rocm.sh`). It activates the venv and points -the ROCm environment at the extracted `_rocm_sdk_devel` tree. It defaults -`VENV_BASE` to your home directory; if you used a different `VENV_BASE` in -step 1, export it before sourcing (or edit the default here). Its contents are: +The repo ships `setup_rocm.sh` in `MLExamples/PyTorch_Profiling/` (the SLURM +scripts source it as `../setup_rocm.sh`). It reads `local.env` through `env.sh`, +activates the venv, points the ROCm environment at the extracted +`_rocm_sdk_devel` tree, and keeps MIOpen's databases on node-local storage. +There is nothing in it to edit: ```bash -#!/usr/bin/env bash -# Source this to activate the ROCm venv and set ROCm env vars: -# source setup_rocm.sh -VENV_BASE="${VENV_BASE:-$HOME}" -VENV="$VENV_BASE/venvs/rocm-pytorch-pip" -source "$VENV/bin/activate" -DEVEL="$(python3 -c 'import site; print(site.getsitepackages()[0])')/_rocm_sdk_devel" -export ROCM_PATH="$DEVEL" -export HIP_PATH="$DEVEL" -export HIP_DEVICE_LIB_PATH="$DEVEL/lib/llvm/amdgcn/bitcode" -export PATH="$DEVEL/bin:$PATH" -export LD_LIBRARY_PATH="$DEVEL/lib:$DEVEL/lib/rocm_sysdeps/lib:$LD_LIBRARY_PATH" -echo "ROCm venv active: $VENV" -``` - -## 6. Re-source to pick up the ROCm env vars - -If the venv is already active from step 1, deactivate and source the script so -the `ROCM_PATH` / `LD_LIBRARY_PATH` exports take effect (run from -`MLExamples/PyTorch_Profiling/`): - -```bash -deactivate +deactivate # if the venv is still active from step 1 source setup_rocm.sh ``` ---- +Where `install_rocm_pytorch.sh` has generated an Lmod modulefile under +`${VENV_BASE}/modulefiles`, `setup_rocm.sh` loads that module instead of +activating the venv directly. Either way the resulting environment is the same. -## 7. Verify (on a GPU node) +## 6. Verify (on a GPU node) ```bash source setup_rocm.sh @@ -119,7 +105,7 @@ x = torch.ones(4, device='cuda:0'); print('device ok:', (x+1).sum().item())" Expected output resembles: ``` -ROCm venv active: /.../rocm-pytorch-pip +setup_rocm.sh: ROCm venv active: /.../rocm-pytorch-pip torch 2.12.0+rocm7.15.0a20260721 device ok: 8.0 ``` diff --git a/MLExamples/PyTorch_Profiling/env.sh b/MLExamples/PyTorch_Profiling/env.sh new file mode 100755 index 00000000..41403412 --- /dev/null +++ b/MLExamples/PyTorch_Profiling/env.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Central configuration for running the PyTorch_Profiling suite on a SLURM +# cluster. Nothing here is site-specific: every machine-dependent value comes +# from a site config file, so the same scripts run unmodified on any cluster. +# +# Site config resolution order: +# 1. $SITE_ENV, if set (lets one checkout drive several clusters) +# 2. local.env next to this file (copy local.env.example and edit) +# +# Sourced by setup_rocm.sh, install_rocm_pytorch.sh, run_all.sh and the +# rocprof-compute analyze script. + +_env_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +SITE_ENV="${SITE_ENV:-${_env_here}/local.env}" +if [[ -f "${SITE_ENV}" ]]; then + # shellcheck disable=SC1090 + source "${SITE_ENV}" +else + echo "env.sh: WARNING: no site config at ${SITE_ENV}" >&2 + echo "env.sh: copy local.env.example to local.env and edit it for this cluster." >&2 +fi + +# --- scheduler ------------------------------------------------------------- +# SLURM charge account and partition. Left empty means "use the cluster default". +PROJECT="${PROJECT:-}" +PARTITION="${PARTITION:-}" + +# sbatch reads these at submit time. The slurm_*.sh headers carry no +# --account/--partition, since #SBATCH directives cannot expand variables. +if [[ -n "${PROJECT}" ]]; then + export SBATCH_ACCOUNT="${PROJECT}" +fi +if [[ -n "${PARTITION}" ]]; then + export SBATCH_PARTITION="${PARTITION}" +fi + +# --- storage --------------------------------------------------------------- +# Base directory holding the venvs, modulefiles and the cloned examples. +# Prefer the fastest storage that is shared by login and compute nodes. +# Defaults to $HOME, matching the upstream setup instructions. +VENV_BASE="${VENV_BASE:-${HOME}}" + +# --- target hardware ------------------------------------------------------- +# GPU_ARCH drives the pip wheel extras (device-); ROOFLINE_ARCH selects +# the counter set passed to the roofline extractor's --arch. +GPU_ARCH="${GPU_ARCH:-}" +ROOFLINE_ARCH="${ROOFLINE_ARCH:-}" + +# --- ROCm build ------------------------------------------------------------ +# Nightly build to validate; see the index below for available versions. +ROCM_VERSION="${ROCM_VERSION:-}" +ROCM_INDEX_URL="${ROCM_INDEX_URL:-https://rocm.nightlies.amd.com/whl-multi-arch/}" + +# --- base python used to create the venv ----------------------------------- +# PYTHON_MODULE is loaded first when set (e.g. cray-python on Cray systems); +# BASE_PYTHON overrides the interpreter outright. +PYTHON_MODULE="${PYTHON_MODULE:-}" +BASE_PYTHON="${BASE_PYTHON:-}" + +# --- Lmod module exposing the venv ---------------------------------------- +MODULE_NAME="${MODULE_NAME:-rocm-pytorch-pip}" +MODULE_VERSION="${MODULE_VERSION:-${ROCM_VERSION}}" +MODULEROOT="${MODULEROOT:-${VENV_BASE}/modulefiles}" + +# Lmod init script to source when `module` is not already defined (common in +# non-login shells and over `ssh host command`). Empty where Lmod is preloaded. +LMOD_INIT="${LMOD_INIT:-}" + +# --- derived paths --------------------------------------------------------- +VENV="${VENV:-${VENV_BASE}/venvs/rocm-pytorch-pip}" +ANALYZE_VENV="${ANALYZE_VENV:-${VENV_BASE}/venvs/rocprof-compute-analyze}" + +# Two layouts are supported. When this file sits inside the examples tree (the +# normal case: MLExamples/PyTorch_Profiling), the tests run from that tree in +# place. When it is kept in a separate working copy, install_rocm_pytorch.sh +# clones the examples under VENV_BASE and overlays these scripts onto them. +if [[ -f "${_env_here}/train_cifar_100.py" ]]; then + EXAMPLES_TOP="${EXAMPLES_TOP:-${_env_here}}" + EXAMPLES_REPO="${EXAMPLES_REPO:-$(cd "${_env_here}/../.." 2>/dev/null && pwd)}" +else + EXAMPLES_REPO="${EXAMPLES_REPO:-${VENV_BASE}/HPCTrainingExamples}" + EXAMPLES_TOP="${EXAMPLES_TOP:-${EXAMPLES_REPO}/MLExamples/PyTorch_Profiling}" +fi + +# --- network --------------------------------------------------------------- +# Outbound proxy for sites whose nodes have no direct internet. Empty elsewhere; +# it is only exported when set, so it never breaks direct-access clusters. +PROXY="${PROXY:-}" + +# --- MIOpen ---------------------------------------------------------------- +# MIOpen keeps its perf/kernel databases as SQLite files. The default location +# is $HOME/.config/miopen, and SQLite file locking fails on NFS and Lustre, +# surfacing as "RuntimeError: miopenStatusInternalError" during conv2d. Keeping +# these databases on node-local storage avoids it. Set MIOPEN_LOCAL_DB=0 on +# sites where the shared filesystem supports real POSIX locks. +MIOPEN_LOCAL_DB="${MIOPEN_LOCAL_DB:-1}" +MIOPEN_TMP_BASE="${MIOPEN_TMP_BASE:-${TMPDIR:-/tmp}}" + +# Make `module` usable when the shell did not inherit Lmod's shell function. +ensure_module_cmd() { + if ! command -v module >/dev/null 2>&1; then + if [[ -n "${LMOD_INIT}" && -r "${LMOD_INIT}" ]]; then + # shellcheck disable=SC1090 + source "${LMOD_INIT}" + fi + fi +} + +# Export the proxy only when the site defines one. +export_proxy() { + if [[ -n "${PROXY}" ]]; then + export http_proxy="${http_proxy:-${PROXY}}" + export https_proxy="${https_proxy:-${PROXY}}" + fi +} diff --git a/MLExamples/PyTorch_Profiling/install_rocm_pytorch.sh b/MLExamples/PyTorch_Profiling/install_rocm_pytorch.sh new file mode 100755 index 00000000..ea90f9ed --- /dev/null +++ b/MLExamples/PyTorch_Profiling/install_rocm_pytorch.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# --------------------------------------------------------------------------- +# install_rocm_pytorch.sh -- run this ON A LOGIN NODE (one with internet). +# +# Builds a self-contained nightly ROCm + PyTorch venv under VENV_BASE, exposes +# it as an Lmod module, and pre-stages everything the compute-node test jobs +# need (many clusters give compute nodes no direct internet): +# 1. venv + pip install nightly rocm/torch/torchvision/transformers +# 2. rocm-sdk init (extract ROCm SDK) +# 3. generate the Lmod modulefile with resolved absolute paths +# 4. clone amd/HPCTrainingExamples and overlay these scripts +# 5. pre-stage rooflineExtractor + its pip deps +# 6. pre-create the rocprof-compute analyze venv (numpy 1.26.4) +# 7. pre-download the CIFAR-100 dataset +# +# Idempotent: re-running reuses existing pieces. All site-specific settings come +# from PyTorch_Profiling/local.env (copy local.env.example and edit it). +# --------------------------------------------------------------------------- +set -euo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Works both from inside MLExamples/PyTorch_Profiling and from a separate +# working copy that keeps the scripts in a PyTorch_Profiling/ subdirectory. +if [[ -d "${SELF_DIR}/PyTorch_Profiling" ]]; then + OVERLAY_DIR="${SELF_DIR}/PyTorch_Profiling" +else + OVERLAY_DIR="${SELF_DIR}" +fi +# shellcheck source=PyTorch_Profiling/env.sh +source "${OVERLAY_DIR}/env.sh" + +for _required in GPU_ARCH ROCM_VERSION VENV_BASE; do + if [[ -z "${!_required}" ]]; then + echo "ERROR: ${_required} is not set. Configure it in ${SITE_ENV}." >&2 + exit 1 + fi +done + +echo "==========================================================================" +echo " ROCm nightly build" +echo " PROJECT = ${PROJECT:-}" +echo " VENV = ${VENV}" +echo " GPU_ARCH = ${GPU_ARCH}" +echo " ROCM_VERSION = ${ROCM_VERSION}" +echo " MODULE = ${MODULE_NAME}/${MODULE_VERSION}" +echo " MODULEROOT = ${MODULEROOT}" +echo " EXAMPLES_TOP = ${EXAMPLES_TOP}" +echo " host = $(hostname)" +echo "==========================================================================" + +# Only takes effect on sites that define PROXY. +export_proxy + +# --- 0. base python to create the venv ------------------------------------ +# PYTHON_MODULE lets Cray-style sites pull in a newer interpreter (cray-python); +# BASE_PYTHON overrides the choice outright. Otherwise system python3 is used. +ensure_module_cmd +if [[ -n "${PYTHON_MODULE}" ]]; then + module load "${PYTHON_MODULE}" 2>/dev/null \ + || echo "WARN: could not 'module load ${PYTHON_MODULE}'; using system python3" +fi +BASE_PY="${BASE_PYTHON:-$(command -v python3)}" +echo "Base python: ${BASE_PY} ($(${BASE_PY} --version 2>&1))" + +# --- 1. venv + nightly ROCm/PyTorch --------------------------------------- +mkdir -p "$(dirname "${VENV}")" +if [[ ! -x "${VENV}/bin/python3" ]]; then + echo "Creating venv at ${VENV}" + "${BASE_PY}" -m venv "${VENV}" +fi +# shellcheck disable=SC1091 +source "${VENV}/bin/activate" +python3 -m pip install --upgrade pip + +echo "Installing nightly ROCm + PyTorch (${ROCM_VERSION}, ${GPU_ARCH}) ..." +python3 -m pip install \ + --index-url "${ROCM_INDEX_URL}" \ + "rocm[profiler,devel,libraries,device-${GPU_ARCH}]==${ROCM_VERSION}" \ + "torch[device-${GPU_ARCH}]" \ + "torchvision[device-${GPU_ARCH}]" + +# transformers is required by train_cifar_100.py. +python3 -m pip install transformers + +# --- 2. extract the ROCm SDK ---------------------------------------------- +echo "Running rocm-sdk init ..." +rocm-sdk init + +# Resolve the extracted SDK path (python minor version independent). +SITE_PACKAGES="$(python3 -c 'import site; print(site.getsitepackages()[0])')" +DEVEL="${SITE_PACKAGES}/_rocm_sdk_devel" +if [[ ! -d "${DEVEL}" ]]; then + echo "ERROR: expected ROCm SDK not found at ${DEVEL}" >&2 + exit 1 +fi +echo "ROCm SDK: ${DEVEL}" + +# --- 3. generate the Lmod modulefile -------------------------------------- +MODDIR="${MODULEROOT}/${MODULE_NAME}" +mkdir -p "${MODDIR}" +MODFILE="${MODDIR}/${MODULE_VERSION}.lua" +sed -e "s#@MODULE_NAME@#${MODULE_NAME}#g" \ + -e "s#@VERSION@#${MODULE_VERSION}#g" \ + -e "s#@GPU_ARCH@#${GPU_ARCH}#g" \ + -e "s#@VENV@#${VENV}#g" \ + -e "s#@DEVEL@#${DEVEL}#g" \ + "${OVERLAY_DIR}/modulefiles/${MODULE_NAME}/TEMPLATE.lua" > "${MODFILE}" +echo "Wrote modulefile: ${MODFILE}" +echo " ( module use ${MODULEROOT} && module load ${MODULE_NAME}/${MODULE_VERSION} )" + +# --- 4. make sure the examples tree is in place --------------------------- +if [[ "${OVERLAY_DIR}" == "${EXAMPLES_TOP}" ]]; then + echo "Running inside the examples tree (${EXAMPLES_TOP}); no clone needed." +else + if [[ ! -d "${EXAMPLES_REPO}/.git" ]]; then + echo "Cloning amd/HPCTrainingExamples -> ${EXAMPLES_REPO}" + git clone https://github.com/amd/HPCTrainingExamples.git "${EXAMPLES_REPO}" + else + echo "Examples repo present; pulling latest" + git -C "${EXAMPLES_REPO}" pull --ff-only || echo "WARN: git pull skipped" + fi + echo "Overlaying scripts into ${EXAMPLES_TOP}" + mkdir -p "${EXAMPLES_TOP}" + cp -rv "${OVERLAY_DIR}/." "${EXAMPLES_TOP}/" + chmod +x "${EXAMPLES_TOP}"/*/*.sh "${EXAMPLES_TOP}"/setup_rocm.sh 2>/dev/null || true + + # The job scripts source env.sh from the deployed tree, where it resolves + # local.env by default. Install the site config actually used for this build + # under that name so jobs cannot pick up another cluster's settings. + if [[ -f "${SITE_ENV}" ]]; then + cp -v "${SITE_ENV}" "${EXAMPLES_TOP}/local.env" + fi +fi + +# --- 5. pre-stage rooflineExtractor + deps -------------------------------- +RE_DIR="${EXAMPLES_TOP}/roofline-extractor/rooflineExtractor" +if [[ ! -d "${RE_DIR}" ]]; then + echo "Cloning rooflineExtractor -> ${RE_DIR}" + git clone https://github.com/AMD-HPC/rooflineExtractor.git "${RE_DIR}" +fi +echo "Installing rooflineExtractor requirements into the shared venv" +python3 -m pip install -r "${RE_DIR}/requirements.txt" + +# --- 6. pre-create the rocprof-compute analyze venv (numpy 1.26.4) -------- +ROCM_PROFILER="${SITE_PACKAGES}/_rocm_profiler" +REQ_FILE="${ROCM_PROFILER}/libexec/rocprofiler-compute/requirements.txt" +if [[ -f "${REQ_FILE}" ]]; then + if [[ ! -x "${ANALYZE_VENV}/bin/python3" ]]; then + echo "Creating analyze venv at ${ANALYZE_VENV}" + "${VENV}/bin/python3" -m venv "${ANALYZE_VENV}" + fi + "${ANALYZE_VENV}/bin/python3" -m pip install --upgrade pip + "${ANALYZE_VENV}/bin/python3" -m pip install -r "${REQ_FILE}" + cp "${REQ_FILE}" "${ANALYZE_VENV}/.rocprof_compute_reqs_installed" + echo "Analyze venv ready: ${ANALYZE_VENV}" +else + echo "WARN: rocprof-compute requirements.txt not found at ${REQ_FILE}; skipping analyze venv" +fi + +# --- 7. pre-download the CIFAR-100 dataset -------------------------------- +# Use download_only_nogpus.py: unlike train_cifar_100.py --download-only, it does +# NOT call dist.init_process_group("nccl", ...), which would hang on a login node +# that has no GPU. This runs purely on torchvision on the login node. +echo "Pre-downloading CIFAR-100 dataset" +python3 "${EXAMPLES_TOP}/download_only_nogpus.py" \ + --data-path "${EXAMPLES_TOP}/data" + +echo "==========================================================================" +echo " Build complete." +echo " Load with : module use ${MODULEROOT} && module load ${MODULE_NAME}/${MODULE_VERSION}" +echo " Tests in : ${EXAMPLES_TOP}" +echo " Submit : bash ${SELF_DIR}/run_all.sh" +echo "==========================================================================" diff --git a/MLExamples/PyTorch_Profiling/local.env.example b/MLExamples/PyTorch_Profiling/local.env.example new file mode 100644 index 00000000..e17e80b1 --- /dev/null +++ b/MLExamples/PyTorch_Profiling/local.env.example @@ -0,0 +1,79 @@ +# Site configuration for the PyTorch_Profiling suite. +# +# cp local.env.example local.env # then edit for your cluster +# +# env.sh sources this file (or $SITE_ENV, if you keep one config per cluster in +# a single checkout). Everything machine-specific lives here; no other script +# needs editing. local.env is git-ignored. + +# --- scheduler ------------------------------------------------------------- +# SLURM charge account and partition. Leave a value empty to use the cluster +# default. These are exported as SBATCH_ACCOUNT / SBATCH_PARTITION, so the +# slurm_*.sh headers stay site-independent. +PROJECT=your_account +PARTITION= + +# --- storage --------------------------------------------------------------- +# Holds the venvs, the generated modulefiles and the cloned examples. Use the +# fastest storage visible from BOTH login and compute nodes (a parallel +# filesystem where available; node-local paths such as /dev/shm or /tmp will +# NOT work, since compute nodes cannot see what the login node wrote there). +VENV_BASE=/path/to/your/scratch/dir + +# --- target hardware ------------------------------------------------------- +# GPU_ARCH selects the pip wheel extras (device-); ROOFLINE_ARCH selects +# the roofline extractor's counter set. +GPU_ARCH=gfx90a +ROOFLINE_ARCH=MI250X + +# --- ROCm build ------------------------------------------------------------ +# Nightly build to validate; see https://rocm.nightlies.amd.com/whl-multi-arch/ +ROCM_VERSION=10.1.0a20260805 + +# --- base python ----------------------------------------------------------- +# Optional module providing a recent python3 used to create the venv, and/or an +# explicit interpreter path. Leave empty to use whatever python3 is on PATH. +PYTHON_MODULE= +BASE_PYTHON= + +# --- Lmod ------------------------------------------------------------------ +# Set LMOD_INIT when `module` is not defined in non-login shells (check with +# `ssh 'type module'`); env.sh sources it before using Lmod. +LMOD_INIT= + +# --- network --------------------------------------------------------------- +# Outbound proxy for clusters whose nodes have no direct internet. Leave empty +# where nodes reach the internet directly. +PROXY= + +# --- MIOpen ---------------------------------------------------------------- +# MIOpen's SQLite perf/kernel databases need real POSIX file locking, which +# NFS and Lustre do not provide (conv2d then fails with +# miopenStatusInternalError). Keep them on node-local storage; set to 0 only if +# your shared filesystem locks correctly. +MIOPEN_LOCAL_DB=1 +MIOPEN_TMP_BASE= + +# --------------------------------------------------------------------------- +# Worked examples +# --------------------------------------------------------------------------- +# +# Frontier (OLCF; MI250X, Lustre, compute nodes behind a proxy): +# PROJECT=abc123 +# PARTITION=partition_name +# VENV_BASE=/lustre/orion/abc123/scratch/$USER +# GPU_ARCH=gfx90a +# ROOFLINE_ARCH=MI250X +# PYTHON_MODULE=cray-python +# LMOD_INIT= +# PROXY=http://proxy.ccs.ornl.gov:3128 +# +# AAC6 (AMD Accelerator Cloud; MI300A, NFS home, direct internet): +# PROJECT=amd +# PARTITION=partition_name +# VENV_BASE=$HOME/pytorch-profiling +# GPU_ARCH=gfx942 +# ROOFLINE_ARCH=MI300A +# PYTHON_MODULE= +# LMOD_INIT=/etc/profile.d/lmod.sh +# PROXY= diff --git a/MLExamples/PyTorch_Profiling/modulefiles/rocm-pytorch-pip/TEMPLATE.lua b/MLExamples/PyTorch_Profiling/modulefiles/rocm-pytorch-pip/TEMPLATE.lua new file mode 100644 index 00000000..b5136096 --- /dev/null +++ b/MLExamples/PyTorch_Profiling/modulefiles/rocm-pytorch-pip/TEMPLATE.lua @@ -0,0 +1,34 @@ +-- @MODULE_NAME@/@VERSION@ +-- Self-contained nightly ROCm + PyTorch virtual environment (GPU arch @GPU_ARCH@). +-- Generated by install_rocm_pytorch.sh from this template; @VENV@ / @DEVEL@ are +-- substituted with the absolute paths resolved after the venv is built. +-- +-- Loading this module reproduces the upstream setup_rocm.sh environment without +-- needing to `source .../bin/activate`: it puts the venv (and the extracted +-- ROCm SDK) on PATH/LD_LIBRARY_PATH and exports the ROCm/HIP variables. + +whatis("Nightly ROCm + PyTorch venv (arch @GPU_ARCH@), self-contained pip install") +help([[ +Nightly ROCm + PyTorch venv built by install_rocm_pytorch.sh. +Provides: python3 (venv), torch/torchvision (ROCm), rocprofv3, rocprof-compute, +rocprof-sys, and the ROCm SDK. Load, then run the PyTorch_Profiling scripts. +]]) + +-- Only one ROCm-providing environment should be loaded at a time. +family("rocm") + +local venv = "@VENV@" +local devel = "@DEVEL@" + +-- Virtualenv activation, expressed as env vars (no activate script needed). +setenv("VIRTUAL_ENV", venv) +prepend_path("PATH", pathJoin(venv, "bin")) +setenv("PYTHONNOUSERSITE", "1") + +-- Extracted ROCm SDK (from `rocm-sdk init`). +setenv("ROCM_PATH", devel) +setenv("HIP_PATH", devel) +setenv("HIP_DEVICE_LIB_PATH", pathJoin(devel, "lib/llvm/amdgcn/bitcode")) +prepend_path("PATH", pathJoin(devel, "bin")) +prepend_path("LD_LIBRARY_PATH", pathJoin(devel, "lib")) +prepend_path("LD_LIBRARY_PATH", pathJoin(devel, "lib/rocm_sysdeps/lib")) diff --git a/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh b/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh index 11fef41b..f013010f 100755 --- a/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh +++ b/MLExamples/PyTorch_Profiling/no-profiling/slurm_single_process_noprofile.sh @@ -1,5 +1,7 @@ #!/bin/bash #SBATCH --job-name=rpc-single-noprofile +# Charge account and partition come from SBATCH_ACCOUNT / SBATCH_PARTITION, +# which env.sh exports from local.env (#SBATCH lines cannot expand variables). #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=8 diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh index e587d8ae..c7cb7e4d 100755 --- a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh +++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_analyze.sh @@ -1,5 +1,7 @@ #!/bin/bash #SBATCH --job-name=rpc-single-analyze +# Charge account and partition come from SBATCH_ACCOUNT / SBATCH_PARTITION, +# which env.sh exports from local.env (#SBATCH lines cannot expand variables). #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=8 @@ -11,9 +13,12 @@ # Analyze the profiled workload with rocprof-compute analyze. # # rocprof-compute analyze needs numpy 1.26.x, which conflicts with the shared -# venv's numpy 2.x. So we use a separate venv holding only rocprof-compute's -# pinned requirements, while reusing the tool bundled in the shared venv (the -# shared venv is never modified). CPU-only; run after the profile job finishes. +# venv's numpy 2.x, so it runs from a separate venv holding only its pinned +# requirements, reusing the tool bundled in the shared venv without modifying +# it. CPU-only; run after the profile job finishes. +# +# Both venvs live under VENV_BASE (see local.env). install_rocm_pytorch.sh +# populates the analysis venv on a login node, so this job needs no network. # --------------------------------------------------------------------------- set -e @@ -29,9 +34,13 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")" echo "SCRIPT_DIR=${SCRIPT_DIR}" echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}" -# Locate the shared ROCm venv WITHOUT activating it; we only need the ROCm -# install inside it (bundled rocprof-compute launcher + ROCm libraries). -MAIN_VENV="${HOME}/venvs/rocm-pytorch-pip" +# Site config (VENV_BASE, PROXY, ...). +source ${PROFILER_TOP_DIR}/env.sh +export_proxy + +# Locate the shared ROCm venv WITHOUT activating it; only the ROCm install +# inside it is needed (bundled rocprof-compute launcher + ROCm libraries). +MAIN_VENV="${VENV:-${VENV_BASE}/venvs/rocm-pytorch-pip}" if [[ ! -x "${MAIN_VENV}/bin/python3" ]]; then echo "ERROR: shared ROCm venv not found at ${MAIN_VENV}" >&2 exit 1 @@ -53,8 +62,9 @@ for p in "${RPC}" "${REQ_FILE}" "${ROCM_CORE}" "${ROCM_DEVEL}"; do done # Create/refresh the dedicated analysis venv with rocprof-compute's pinned -# requirements (numpy 1.26.x). Reinstalled only when requirements.txt changes. -ANALYZE_VENV="${HOME}/venvs/rocprof-compute-analyze" +# requirements (numpy 1.26.x). Reinstalled only when requirements.txt changes, +# so this is a no-op where install_rocm_pytorch.sh already created it. +ANALYZE_VENV="${ANALYZE_VENV:-${VENV_BASE}/venvs/rocprof-compute-analyze}" if [[ ! -x "${ANALYZE_VENV}/bin/python3" ]]; then echo "Creating analysis venv at ${ANALYZE_VENV}" # A venv stays isolated from its creator's site-packages, so using the diff --git a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh index be3974ed..337bf9c1 100755 --- a/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh +++ b/MLExamples/PyTorch_Profiling/rocm-compute-profiler/slurm_single_process_profile.sh @@ -1,5 +1,7 @@ #!/bin/bash #SBATCH --job-name=rpc-single-profile +# Charge account and partition come from SBATCH_ACCOUNT / SBATCH_PARTITION, +# which env.sh exports from local.env (#SBATCH lines cannot expand variables). #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=8 diff --git a/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh b/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh index be71a9f5..60dd759b 100755 --- a/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh +++ b/MLExamples/PyTorch_Profiling/rocm-systems-profiler/slurm_single_process.sh @@ -1,5 +1,7 @@ #!/bin/bash #SBATCH --job-name=rps-single-run +# Charge account and partition come from SBATCH_ACCOUNT / SBATCH_PARTITION, +# which env.sh exports from local.env (#SBATCH lines cannot expand variables). #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=8 diff --git a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh index e4cb75a4..a923e956 100755 --- a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh +++ b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_kernels.sh @@ -1,5 +1,7 @@ #!/bin/bash #SBATCH --job-name=rpv3-single-kernels +# Charge account and partition come from SBATCH_ACCOUNT / SBATCH_PARTITION, +# which env.sh exports from local.env (#SBATCH lines cannot expand variables). #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=8 @@ -43,7 +45,7 @@ if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then fi OUT_DIR=${SCRIPT_DIR}/single_process -rm -rf ${OUT_DIR} +rm -f ${OUT_DIR}/kernels* cd ${SCRIPT_DIR} # Profile kernels with rocprofv3. diff --git a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh index 910890ab..9d694346 100755 --- a/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh +++ b/MLExamples/PyTorch_Profiling/rocprofv3/slurm_single_process_traces.sh @@ -1,5 +1,7 @@ #!/bin/bash #SBATCH --job-name=rpv3-single-traces +# Charge account and partition come from SBATCH_ACCOUNT / SBATCH_PARTITION, +# which env.sh exports from local.env (#SBATCH lines cannot expand variables). #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=8 @@ -43,7 +45,7 @@ if [ ! -d ${PROFILER_TOP_DIR}/data/cifar-100-python ]; then fi OUT_DIR=${SCRIPT_DIR}/single_process -rm -rf ${OUT_DIR} +rm -f ${OUT_DIR}/traces* cd ${SCRIPT_DIR} # Collect GPU timeline traces with rocprofv3. diff --git a/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh b/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh index b98a1691..5d992cef 100755 --- a/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh +++ b/MLExamples/PyTorch_Profiling/roofline-extractor/slurm_single_process.sh @@ -1,5 +1,7 @@ #!/bin/bash #SBATCH --job-name=roofline-single +# Charge account and partition come from SBATCH_ACCOUNT / SBATCH_PARTITION, +# which env.sh exports from local.env (#SBATCH lines cannot expand variables). #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=8 @@ -14,6 +16,12 @@ # Its profile_app.py runs rocprofv3 several times to collect counters + a kernel # trace, then produces the per-kernel roofline analysis and an HTML plot. # Sources ../setup_rocm.sh to activate the ROCm venv. +# +# Notes: +# * --arch comes from ROOFLINE_ARCH in local.env and selects the counter set +# matching this cluster's GPU (e.g. MI250X for gfx90a, MI300A for gfx942). +# * rooflineExtractor and its pip deps are PRE-STAGED by install_rocm_pytorch.sh +# on a login node, for sites whose compute nodes have no direct internet. # --------------------------------------------------------------------------- set -e @@ -29,16 +37,24 @@ PROFILER_TOP_DIR="$(dirname "${SCRIPT_DIR}")" echo "SCRIPT_DIR=${SCRIPT_DIR}" echo "PROFILER_TOP_DIR=${PROFILER_TOP_DIR}" -# ../setup_rocm.sh activates the ROCm PyTorch venv and exports ROCm env vars. +# ../setup_rocm.sh activates the ROCm PyTorch venv and exports ROCm env vars +# (and, via env.sh, sets ROOFLINE_ARCH from local.env). source ${PROFILER_TOP_DIR}/setup_rocm.sh rocprofv3 --version -# Fetch rooflineExtractor and install its Python dependencies (into the venv). +if [[ -z "${ROOFLINE_ARCH}" ]]; then + echo "ERROR: ROOFLINE_ARCH is not set; define it in local.env." >&2 + exit 1 +fi + +# Use the copy pre-staged by install_rocm_pytorch.sh on a login node; fall back +# to cloning here, which requires internet on the compute node. RE_DIR=${SCRIPT_DIR}/rooflineExtractor if [ ! -d ${RE_DIR} ]; then + echo "rooflineExtractor not pre-staged; cloning (requires internet on this node)" git clone https://github.com/AMD-HPC/rooflineExtractor.git ${RE_DIR} + python3 -m pip install -r ${RE_DIR}/requirements.txt fi -python3 -m pip install -r ${RE_DIR}/requirements.txt # Distributed bootstrap variables expected by train_cifar_100.py (single rank). export NPROCS=1 @@ -57,15 +73,15 @@ OUT_DIR=${SCRIPT_DIR}/output rm -rf ${OUT_DIR} cd ${SCRIPT_DIR} -# Collect roofline data and generate plots. --arch MI300A selects the gfx942 +# Collect roofline data and generate plots. --arch selects this cluster's # counter set; profile_app.py runs rocprofv3 itself, so it is the target. echo echo "===================================================================" -echo "profile_app.py --arch MI300A -- python3 train_cifar_100.py" +echo "profile_app.py --arch ${ROOFLINE_ARCH} -- python3 train_cifar_100.py" echo "===================================================================" srun -n 1 --gpus=1 --cpus-per-task=8 \ python3 ${RE_DIR}/profile_app.py \ - --arch MI300A \ + --arch ${ROOFLINE_ARCH} \ -o ${OUT_DIR} \ -- \ python3 ${PROFILER_TOP_DIR}/train_cifar_100.py \ diff --git a/MLExamples/PyTorch_Profiling/run_all.sh b/MLExamples/PyTorch_Profiling/run_all.sh new file mode 100755 index 00000000..a98f89b8 --- /dev/null +++ b/MLExamples/PyTorch_Profiling/run_all.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# --------------------------------------------------------------------------- +# run_all.sh -- submit the whole PyTorch_Profiling suite. +# +# Run this on a LOGIN node after install_rocm_pytorch.sh has built the +# venv/module and pre-staged everything. Each script is submitted from its own +# directory (the scripts rely on SLURM_SUBMIT_DIR). The rocprof-compute analyze +# job is chained after its profile job with --dependency=afterok. +# +# Account and partition are taken from local.env and passed to sbatch through +# the SBATCH_ACCOUNT / SBATCH_PARTITION environment variables exported by env.sh. +# +# Prints one " " line per submission. +# --------------------------------------------------------------------------- +set -euo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Works both from inside MLExamples/PyTorch_Profiling and from a separate +# working copy that keeps the scripts in a PyTorch_Profiling/ subdirectory. +if [[ -d "${SELF_DIR}/PyTorch_Profiling" ]]; then + # shellcheck source=PyTorch_Profiling/env.sh + source "${SELF_DIR}/PyTorch_Profiling/env.sh" +else + # shellcheck source=env.sh + source "${SELF_DIR}/env.sh" +fi + +if [[ ! -d "${EXAMPLES_TOP}" ]]; then + echo "ERROR: ${EXAMPLES_TOP} not found. Run install_rocm_pytorch.sh first." >&2 + exit 1 +fi + +submit() { + # submit