From 743c9f1267ac550fb81280aee44c5e7fc9272062 Mon Sep 17 00:00:00 2001 From: yh-noh Date: Mon, 2 Nov 2026 13:32:37 +0900 Subject: [PATCH 1/3] =?UTF-8?q?fix(iam-nginx):=20Web=20Console=20=ED=94=84?= =?UTF-8?q?=EB=A1=9D=EC=8B=9C=20=EC=9D=91=EB=8B=B5=20=EB=8C=80=EA=B8=B0=20?= =?UTF-8?q?=EC=8B=9C=EA=B0=84=EC=9D=84=2090=EC=B4=88=EB=A1=9C=20=EB=8A=98?= =?UTF-8?q?=EB=A6=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 노드 삭제·전원 제어 등 mc-infra-manager 동기 API 는 CSP 작업이 끝난 뒤 응답한다(GCP 노드 삭제 약 62초). Web Console location 의 proxy_read_timeout 60s 보다 길어 작업은 성공하는데 브라우저는 504 를 받았다. - Web Console(:3001) location proxy_read_timeout 60s → 90s (prod/local 템플릿) - README/README_kr Known Issues 에 원인과 120s 상향 절차 기록 --- README.md | 21 +++++++++++++++++++ README_kr.md | 21 +++++++++++++++++++ .../conf/mc-iam-manager/nginx.template.conf | 4 +++- .../mc-iam-manager/nginx.template.local.conf | 4 +++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8a285ce..c014098 100644 --- a/README.md +++ b/README.md @@ -354,6 +354,27 @@ When you switch from Mode A (self-signed cert) to Mode B (Let's Encrypt) using t - **Firefox**: use a new browser profile or delete `SiteSecurityServiceState.txt` from your profile folder +### Web Console 504 on long infra operations (nginx read timeout) + +Some infra operations — node delete, node power control — are synchronous in mc-infra-manager (CB-Tumblebug): the response comes back only after the CSP finishes. On GCP a node delete took about 62 seconds. If that is longer than the nginx `proxy_read_timeout` of the Web Console location, the browser gets **504** even though the operation completes successfully afterwards. + +The Web Console location (`:3001`) in `conf/docker/conf/mc-iam-manager/nginx.template.conf` (and `nginx.template.local.conf`) uses `proxy_read_timeout 90s`. + +**If 504 still occurs**, raise it to `120s`. Keep it as short as practical — a longer value only makes the browser wait longer. + +```bash +# 1. Edit the Web Console location in the template +# proxy_read_timeout 90s; → proxy_read_timeout 120s; +vi conf/docker/conf/mc-iam-manager/nginx.template.conf + +# 2. Apply to a running installation (generated file) +sed -i '/upstream_console/,/^ }/ s/proxy_read_timeout 90s;/proxy_read_timeout 120s;/' \ + conf/docker/container-volume/mc-iam-manager/nginx/nginx.conf +docker exec mc-iam-manager-nginx nginx -t && docker exec mc-iam-manager-nginx nginx -s reload +``` + +To confirm the cause, look for `upstream timed out ... while reading response header from upstream` in `docker logs mc-iam-manager-nginx` at the time of the 504. + ## TLS Certificate Auto-Renewal (Mode B) Let's Encrypt certificates are valid for 90 days. The certificate is stored in the container volume `conf/docker/container-volume/mc-iam-manager/certs` (mounted into `mc-iam-manager-nginx`), **not** in the host's `/etc/letsencrypt`, so a host certbot or `certbot.timer` does not renew it. diff --git a/README_kr.md b/README_kr.md index 56bdf0c..afcbd50 100644 --- a/README_kr.md +++ b/README_kr.md @@ -265,6 +265,27 @@ Mode A(자체 서명 인증서) → Mode B(Let's Encrypt)로 같은 도메인을 - **Firefox**: 새 브라우저 프로파일 사용 또는 프로파일 폴더의 `SiteSecurityServiceState.txt` 삭제 +### Web Console 장시간 Infra 작업 504 (nginx 응답 대기 시간) + +노드 삭제·노드 전원 제어 등 일부 Infra 작업은 mc-infra-manager(CB-Tumblebug)에서 동기로 처리되어, CSP 작업이 끝난 뒤에야 응답합니다. GCP 노드 삭제는 약 62초가 걸렸습니다. 이 시간이 Web Console location 의 nginx `proxy_read_timeout` 보다 길면, 작업은 이후 정상 완료되는데도 브라우저는 **504** 를 받습니다. + +`conf/docker/conf/mc-iam-manager/nginx.template.conf`(및 `nginx.template.local.conf`)의 Web Console location(`:3001`)은 `proxy_read_timeout 90s` 를 사용합니다. + +**504 가 계속 발생하면** `120s` 로 늘립니다. 값을 늘리면 브라우저가 그만큼 오래 기다리게 되므로 필요한 만큼만 늘립니다. + +```bash +# 1. 템플릿의 Web Console location 수정 +# proxy_read_timeout 90s; → proxy_read_timeout 120s; +vi conf/docker/conf/mc-iam-manager/nginx.template.conf + +# 2. 기동 중인 설치에 반영(생성된 파일) +sed -i '/upstream_console/,/^ }/ s/proxy_read_timeout 90s;/proxy_read_timeout 120s;/' \ + conf/docker/container-volume/mc-iam-manager/nginx/nginx.conf +docker exec mc-iam-manager-nginx nginx -t && docker exec mc-iam-manager-nginx nginx -s reload +``` + +원인 확인: 504 발생 시각의 `docker logs mc-iam-manager-nginx` 에 `upstream timed out ... while reading response header from upstream` 이 있는지 봅니다. + ## TLS 인증서 자동갱신 (Mode B) Let's Encrypt 인증서의 유효기간은 90일입니다. 인증서는 호스트의 `/etc/letsencrypt`가 **아니라** 컨테이너 볼륨 `conf/docker/container-volume/mc-iam-manager/certs`(`mc-iam-manager-nginx`에 마운트)에 저장되므로, 호스트 certbot이나 `certbot.timer`로는 갱신되지 않습니다. diff --git a/conf/docker/conf/mc-iam-manager/nginx.template.conf b/conf/docker/conf/mc-iam-manager/nginx.template.conf index f7dc69d..c95e198 100644 --- a/conf/docker/conf/mc-iam-manager/nginx.template.conf +++ b/conf/docker/conf/mc-iam-manager/nginx.template.conf @@ -181,7 +181,9 @@ http { proxy_connect_timeout 60s; proxy_send_timeout 60s; - proxy_read_timeout 60s; + # Infra node delete/control calls wait for the CSP and can take over 60s. + # 90s first; raise to 120s if 504 still occurs (see README Known Issues). + proxy_read_timeout 90s; } } diff --git a/conf/docker/conf/mc-iam-manager/nginx.template.local.conf b/conf/docker/conf/mc-iam-manager/nginx.template.local.conf index 401872a..f8bcdb4 100644 --- a/conf/docker/conf/mc-iam-manager/nginx.template.local.conf +++ b/conf/docker/conf/mc-iam-manager/nginx.template.local.conf @@ -113,7 +113,9 @@ http { proxy_connect_timeout 60s; proxy_send_timeout 60s; - proxy_read_timeout 60s; + # Infra node delete/control calls wait for the CSP and can take over 60s. + # 90s first; raise to 120s if 504 still occurs (see README Known Issues). + proxy_read_timeout 90s; } } From 71fc941f03e3113e3304f90c54ff5af9f72a5923 Mon Sep 17 00:00:00 2001 From: yh-noh Date: Mon, 2 Nov 2026 16:15:08 +0900 Subject: [PATCH 2/3] =?UTF-8?q?fix(iam-nginx):=20Web=20Console=20=ED=94=84?= =?UTF-8?q?=EB=A1=9D=EC=8B=9C=20=EC=9D=91=EB=8B=B5=20=EB=8C=80=EA=B8=B0=20?= =?UTF-8?q?=EC=8B=9C=EA=B0=84=EC=9D=84=20120=EC=B4=88=EB=A1=9C=20=EB=8A=98?= =?UTF-8?q?=EB=A6=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stage 실측에서 AWS EKS NodeGroup 생성이 99초 걸려 90초로도 504 가 났다. - Web Console(:3001) location proxy_read_timeout 90s → 120s (prod/local 템플릿) - README/README_kr Known Issues 실측값(노드 삭제 62초, NodeGroup 생성 99초)과 상향 절차 갱신 --- README.md | 10 +++++----- README_kr.md | 10 +++++----- conf/docker/conf/mc-iam-manager/nginx.template.conf | 6 +++--- .../conf/mc-iam-manager/nginx.template.local.conf | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index c014098..f3b46ba 100644 --- a/README.md +++ b/README.md @@ -356,19 +356,19 @@ When you switch from Mode A (self-signed cert) to Mode B (Let's Encrypt) using t ### Web Console 504 on long infra operations (nginx read timeout) -Some infra operations — node delete, node power control — are synchronous in mc-infra-manager (CB-Tumblebug): the response comes back only after the CSP finishes. On GCP a node delete took about 62 seconds. If that is longer than the nginx `proxy_read_timeout` of the Web Console location, the browser gets **504** even though the operation completes successfully afterwards. +Some infra operations are synchronous in mc-infra-manager (CB-Tumblebug): the response comes back only after the CSP finishes. Measured on stage: GCP node delete 62 seconds, AWS EKS NodeGroup create 99 seconds. If that is longer than the nginx `proxy_read_timeout` of the Web Console location, the browser gets **504** (and a failure toast) even though the operation completes successfully afterwards. -The Web Console location (`:3001`) in `conf/docker/conf/mc-iam-manager/nginx.template.conf` (and `nginx.template.local.conf`) uses `proxy_read_timeout 90s`. +The Web Console location (`:3001`) in `conf/docker/conf/mc-iam-manager/nginx.template.conf` (and `nginx.template.local.conf`) uses `proxy_read_timeout 120s` (default was 60s). -**If 504 still occurs**, raise it to `120s`. Keep it as short as practical — a longer value only makes the browser wait longer. +**If 504 still occurs**, raise it further. Keep it as short as practical — a longer value only makes the browser wait longer. ```bash # 1. Edit the Web Console location in the template -# proxy_read_timeout 90s; → proxy_read_timeout 120s; +# proxy_read_timeout 120s; → e.g. proxy_read_timeout 180s; vi conf/docker/conf/mc-iam-manager/nginx.template.conf # 2. Apply to a running installation (generated file) -sed -i '/upstream_console/,/^ }/ s/proxy_read_timeout 90s;/proxy_read_timeout 120s;/' \ +sed -i '/upstream_console/,/^ }/ s/proxy_read_timeout 120s;/proxy_read_timeout 180s;/' \ conf/docker/container-volume/mc-iam-manager/nginx/nginx.conf docker exec mc-iam-manager-nginx nginx -t && docker exec mc-iam-manager-nginx nginx -s reload ``` diff --git a/README_kr.md b/README_kr.md index afcbd50..36fbf4a 100644 --- a/README_kr.md +++ b/README_kr.md @@ -267,19 +267,19 @@ Mode A(자체 서명 인증서) → Mode B(Let's Encrypt)로 같은 도메인을 ### Web Console 장시간 Infra 작업 504 (nginx 응답 대기 시간) -노드 삭제·노드 전원 제어 등 일부 Infra 작업은 mc-infra-manager(CB-Tumblebug)에서 동기로 처리되어, CSP 작업이 끝난 뒤에야 응답합니다. GCP 노드 삭제는 약 62초가 걸렸습니다. 이 시간이 Web Console location 의 nginx `proxy_read_timeout` 보다 길면, 작업은 이후 정상 완료되는데도 브라우저는 **504** 를 받습니다. +일부 Infra 작업은 mc-infra-manager(CB-Tumblebug)에서 동기로 처리되어, CSP 작업이 끝난 뒤에야 응답합니다. stage 실측: GCP 노드 삭제 62초, AWS EKS NodeGroup 생성 99초. 이 시간이 Web Console location 의 nginx `proxy_read_timeout` 보다 길면, 작업은 이후 정상 완료되는데도 브라우저는 **504**(와 실패 토스트)를 받습니다. -`conf/docker/conf/mc-iam-manager/nginx.template.conf`(및 `nginx.template.local.conf`)의 Web Console location(`:3001`)은 `proxy_read_timeout 90s` 를 사용합니다. +`conf/docker/conf/mc-iam-manager/nginx.template.conf`(및 `nginx.template.local.conf`)의 Web Console location(`:3001`)은 `proxy_read_timeout 120s` 를 사용합니다(기존 기본값 60s). -**504 가 계속 발생하면** `120s` 로 늘립니다. 값을 늘리면 브라우저가 그만큼 오래 기다리게 되므로 필요한 만큼만 늘립니다. +**504 가 계속 발생하면** 값을 더 늘립니다. 값을 늘리면 브라우저가 그만큼 오래 기다리게 되므로 필요한 만큼만 늘립니다. ```bash # 1. 템플릿의 Web Console location 수정 -# proxy_read_timeout 90s; → proxy_read_timeout 120s; +# proxy_read_timeout 120s; → 예: proxy_read_timeout 180s; vi conf/docker/conf/mc-iam-manager/nginx.template.conf # 2. 기동 중인 설치에 반영(생성된 파일) -sed -i '/upstream_console/,/^ }/ s/proxy_read_timeout 90s;/proxy_read_timeout 120s;/' \ +sed -i '/upstream_console/,/^ }/ s/proxy_read_timeout 120s;/proxy_read_timeout 180s;/' \ conf/docker/container-volume/mc-iam-manager/nginx/nginx.conf docker exec mc-iam-manager-nginx nginx -t && docker exec mc-iam-manager-nginx nginx -s reload ``` diff --git a/conf/docker/conf/mc-iam-manager/nginx.template.conf b/conf/docker/conf/mc-iam-manager/nginx.template.conf index c95e198..4501ea5 100644 --- a/conf/docker/conf/mc-iam-manager/nginx.template.conf +++ b/conf/docker/conf/mc-iam-manager/nginx.template.conf @@ -181,9 +181,9 @@ http { proxy_connect_timeout 60s; proxy_send_timeout 60s; - # Infra node delete/control calls wait for the CSP and can take over 60s. - # 90s first; raise to 120s if 504 still occurs (see README Known Issues). - proxy_read_timeout 90s; + # Infra/K8s calls such as node delete and NodeGroup create wait for the CSP + # (measured 62s / 99s). Raise further if 504 still occurs (see README Known Issues). + proxy_read_timeout 120s; } } diff --git a/conf/docker/conf/mc-iam-manager/nginx.template.local.conf b/conf/docker/conf/mc-iam-manager/nginx.template.local.conf index f8bcdb4..2703127 100644 --- a/conf/docker/conf/mc-iam-manager/nginx.template.local.conf +++ b/conf/docker/conf/mc-iam-manager/nginx.template.local.conf @@ -113,9 +113,9 @@ http { proxy_connect_timeout 60s; proxy_send_timeout 60s; - # Infra node delete/control calls wait for the CSP and can take over 60s. - # 90s first; raise to 120s if 504 still occurs (see README Known Issues). - proxy_read_timeout 90s; + # Infra/K8s calls such as node delete and NodeGroup create wait for the CSP + # (measured 62s / 99s). Raise further if 504 still occurs (see README Known Issues). + proxy_read_timeout 120s; } } From 59734ea8d0a00454b1408a388fe7350b643741ea Mon Sep 17 00:00:00 2001 From: yh-noh Date: Mon, 9 Nov 2026 14:32:03 +0900 Subject: [PATCH 3/3] =?UTF-8?q?chore:=20mc-web-console=20=EC=9D=B4?= =?UTF-8?q?=EB=AF=B8=EC=A7=80=C2=B7api.yaml=20=EB=B2=84=EC=A0=84=EC=9D=84?= =?UTF-8?q?=200.7.4=EB=A1=9C=20=EA=B3=A0=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/api.yaml | 2 +- conf/docker/api.yaml | 2 +- conf/docker/conf/mc-web-console/api/conf/api.yaml | 2 +- conf/docker/docker-compose.mini.yaml | 4 ++-- conf/docker/docker-compose.yaml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/api.yaml b/conf/api.yaml index 4301a28..1a5cd05 100644 --- a/conf/api.yaml +++ b/conf/api.yaml @@ -25,7 +25,7 @@ services: password: spider mc-web-console: - version: 0.7.3 + version: 0.7.4 baseurl: http://mc-web-console:3000 auth: type: bearer diff --git a/conf/docker/api.yaml b/conf/docker/api.yaml index 4301a28..1a5cd05 100644 --- a/conf/docker/api.yaml +++ b/conf/docker/api.yaml @@ -25,7 +25,7 @@ services: password: spider mc-web-console: - version: 0.7.3 + version: 0.7.4 baseurl: http://mc-web-console:3000 auth: type: bearer diff --git a/conf/docker/conf/mc-web-console/api/conf/api.yaml b/conf/docker/conf/mc-web-console/api/conf/api.yaml index 32bece1..31dd75d 100644 --- a/conf/docker/conf/mc-web-console/api/conf/api.yaml +++ b/conf/docker/conf/mc-web-console/api/conf/api.yaml @@ -25,7 +25,7 @@ services: password: default mc-web-console: - version: 0.7.3 + version: 0.7.4 baseurl: http://mc-web-console-api:3000 auth: type: bearer diff --git a/conf/docker/docker-compose.mini.yaml b/conf/docker/docker-compose.mini.yaml index d60eead..b7eb93b 100644 --- a/conf/docker/docker-compose.mini.yaml +++ b/conf/docker/docker-compose.mini.yaml @@ -389,7 +389,7 @@ services: exec docker-entrypoint.sh postgres" mc-web-console-api: - image: cloudbaristaorg/mc-web-console-api:0.7.3 + image: cloudbaristaorg/mc-web-console-api:0.7.4 pull_policy: always container_name: mc-web-console-api platform: linux/amd64 @@ -438,7 +438,7 @@ services: <<: *default-health-check mc-web-console-front: - image: cloudbaristaorg/mc-web-console-front:0.7.3 + image: cloudbaristaorg/mc-web-console-front:0.7.4 pull_policy: always container_name: mc-web-console-front platform: linux/amd64 diff --git a/conf/docker/docker-compose.yaml b/conf/docker/docker-compose.yaml index fd9e4b1..9a62f8c 100644 --- a/conf/docker/docker-compose.yaml +++ b/conf/docker/docker-compose.yaml @@ -1187,7 +1187,7 @@ services: exec docker-entrypoint.sh postgres" mc-web-console-api: - image: cloudbaristaorg/mc-web-console-api:0.7.3 + image: cloudbaristaorg/mc-web-console-api:0.7.4 pull_policy: always container_name: mc-web-console-api platform: linux/amd64 @@ -1231,7 +1231,7 @@ services: <<: *default-health-check mc-web-console-front: - image: cloudbaristaorg/mc-web-console-front:0.7.3 + image: cloudbaristaorg/mc-web-console-front:0.7.4 pull_policy: always container_name: mc-web-console-front platform: linux/amd64