Skip to content

cli: run kubeadm reset when removing control-plane nodes #157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alicefr merged 1 commit into from
Oct 23, 2026
Merged

cli: run kubeadm reset when removing control-plane nodes #157

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions internal/cli/node/remove.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,19 @@ func runRemove(ctx context.Context, nodeName string, force bool, logger *logrus.
isControlPlane := role == "control-plane"
logger.Info("")

// Step 2: Drain and delete from Kubernetes via SSH to a control-plane node
logger.Info("Step 2: Removing node from Kubernetes...")
// Step 2: For control-plane nodes, run kubeadm reset on the node to
// cleanly remove it from the etcd cluster. Without this, etcd loses
// quorum when the node goes down.
if isControlPlane {
logger.Info("Step 2: Resetting control-plane node (removes etcd member)...")
targetSSH := ssh.NewClientForNode(clusterName, nodeName, logger)
if err := resetControlPlane(ctx, targetSSH, logger); err != nil {
Comment thread
alicefr marked this conversation as resolved.
return fmt.Errorf("kubeadm reset failed: %w", err)
}
}

// Step 3: Drain and delete from Kubernetes via SSH to a control-plane node
logger.Info("Step 3: Removing node from Kubernetes...")
cpNode, err := findControlPlaneNode(ctx, podmanClient, clusterName, nodeName)
if err != nil {
logger.Warnf("Could not find control-plane node: %v", err)
Expand All @@ -84,8 +95,8 @@ func runRemove(ctx context.Context, nodeName string, force bool, logger *logrus.
}
logger.Info("")

// Step 3: Remove DNS entry
logger.Info("Step 3: Removing DNS entry...")
// Step 4: Remove DNS entry
logger.Info("Step 4: Removing DNS entry...")
dnsMgr, err := dns.NewManager(clusterName)
if err != nil {
logger.Warnf("Failed to create DNS manager: %v", err)
Expand All @@ -94,8 +105,8 @@ func runRemove(ctx context.Context, nodeName string, force bool, logger *logrus.
}
logger.Info("")

// Step 4: Stop and remove the container
logger.Info("Step 4: Stopping and removing container...")
// Step 5: Stop and remove the container
logger.Info("Step 5: Stopping and removing container...")
if err := podmanClient.ContainerStop(ctx, containerName); err != nil {
logger.Warnf("Failed to stop container: %v", err)
}
Expand All @@ -105,9 +116,9 @@ func runRemove(ctx context.Context, nodeName string, force bool, logger *logrus.
logger.Infof("Container %s removed", containerName)
logger.Info("")

// Step 5: Update HAProxy if this was a control-plane node
// Step 6: Update HAProxy if this was a control-plane node
if isControlPlane {
logger.Info("Step 5: Updating HAProxy load balancer...")
logger.Info("Step 6: Updating HAProxy load balancer...")
haproxyMgr, err := haproxy.NewManager(clusterName)
if err != nil {
logger.Warnf("Failed to create HAProxy manager: %v", err)
Expand Down Expand Up @@ -148,6 +159,15 @@ func findControlPlaneNode(ctx context.Context, podmanClient *podman.Client, clus
return "", fmt.Errorf("no control-plane node found in cluster %s", clusterName)
}

func resetControlPlane(ctx context.Context, sshClient *ssh.Client, logger *logrus.Logger) error {
logger.Info("Running kubeadm reset on control-plane node...")
if _, err := sshClient.Exec(ctx, "sudo kubeadm reset --force"); err != nil {
return fmt.Errorf("kubeadm reset: %w", err)
}
logger.Info("kubeadm reset completed")
return nil
}

func removeFromKubernetes(ctx context.Context, sshClient *ssh.Client, nodeName string, force bool, logger *logrus.Logger) error {
kubectl := "sudo kubectl --kubeconfig=/etc/kubernetes/admin.conf"

Expand Down
Loading