Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -256,12 +256,20 @@ rc admin info cluster local
rc admin info server local
rc admin info disk local --offline

# Heal operations
# Aggregate background heal status
rc admin heal status local

# Root recursive manual heal
rc admin heal start local --scan-mode deep
rc admin heal status local --client-token <TOKEN_FROM_START>
rc admin heal stop local --client-token <TOKEN_FROM_START>

# Bucket manual heal
rc admin heal start local --bucket mybucket --scan-mode deep
rc admin heal status local --bucket mybucket --client-token <TOKEN_FROM_START>
rc admin heal start local --dry-run
rc admin heal stop local --bucket mybucket --client-token <TOKEN_FROM_START>

# Global force stop
rc admin heal stop local

# Pool expansion and decommission workflows
Expand Down
39 changes: 31 additions & 8 deletions crates/cli/src/commands/admin/heal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -491,19 +491,15 @@ fn heal_task_request(

match (has_target, has_token) {
(false, false) => Ok(None),
(true, true) => Ok(Some(HealTaskRequest {
bucket: bucket.expect("bucket is present"),
(_, true) => Ok(Some(HealTaskRequest {
bucket: bucket.unwrap_or_default(),
prefix,
client_token: client_token.expect("client token is present"),
})),
(true, false) => {
formatter.error("Heal task request requires --client-token when --bucket is set.");
Err(ExitCode::UsageError)
}
(false, true) => {
formatter.error("Heal task request requires --bucket when --client-token is set.");
Err(ExitCode::UsageError)
}
}
}

Expand Down Expand Up @@ -611,10 +607,37 @@ mod tests {
}

#[test]
fn test_heal_task_request_rejects_token_without_bucket() {
fn test_heal_task_request_accepts_token_without_bucket() {
let formatter = Formatter::default();

let request = heal_task_request(None, None, Some("root-token".to_string()), &formatter)
.expect("root token request should be valid")
.expect("root token request should be task scoped");

assert!(request.bucket.is_empty());
assert!(request.prefix.is_none());
assert_eq!(request.client_token, "root-token");
}

#[test]
fn test_heal_task_request_rejects_bucket_without_token() {
let formatter = Formatter::default();

let result = heal_task_request(Some("logs".to_string()), None, None, &formatter);

assert!(matches!(result, Err(ExitCode::UsageError)));
}

#[test]
fn test_heal_task_request_rejects_prefix_without_bucket() {
let formatter = Formatter::default();

let result = heal_task_request(None, None, Some("root-token".to_string()), &formatter);
let result = heal_task_request(
None,
Some("2026/".to_string()),
Some("root-token".to_string()),
&formatter,
);

assert!(matches!(result, Err(ExitCode::UsageError)));
}
Expand Down
110 changes: 100 additions & 10 deletions crates/s3/src/admin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -573,24 +573,23 @@ fn rustfs_heal_path(request: &HealStartRequest) -> Result<String> {
}

fn rustfs_heal_task_path(request: &HealTaskRequest) -> Result<String> {
if request.bucket.is_empty() {
return Err(Error::InvalidPath(
"heal task status requires a bucket target".to_string(),
));
}

let bucket = (!request.bucket.is_empty()).then_some(request.bucket.as_str());
let prefix = request
.prefix
.as_deref()
.filter(|prefix| !prefix.is_empty());

match prefix {
Some(prefix) => Ok(format!(
match (bucket, prefix) {
(None, None) => Ok("/heal/".to_string()),
(Some(bucket), None) => Ok(format!("/heal/{}", urlencoding::encode(bucket))),
(Some(bucket), Some(prefix)) => Ok(format!(
"/heal/{}/{}",
urlencoding::encode(&request.bucket),
urlencoding::encode(bucket),
urlencoding::encode(prefix)
)),
None => Ok(format!("/heal/{}", urlencoding::encode(&request.bucket))),
(None, Some(_)) => Err(Error::InvalidPath(
"heal task prefix requires a bucket target".to_string(),
)),
}
}

Expand Down Expand Up @@ -1465,6 +1464,34 @@ mod tests {
));
}

#[test]
fn test_rustfs_heal_task_path_supports_root_target() {
let request = HealTaskRequest {
bucket: String::new(),
prefix: None,
client_token: "root-token".to_string(),
};

assert_eq!(
rustfs_heal_task_path(&request).expect("root task path"),
"/heal/"
);
}

#[test]
fn test_rustfs_heal_task_path_rejects_root_prefix() {
let request = HealTaskRequest {
bucket: String::new(),
prefix: Some("2026/".to_string()),
client_token: "root-token".to_string(),
};

assert!(matches!(
rustfs_heal_task_path(&request),
Err(Error::InvalidPath(_))
));
}

#[test]
fn test_rustfs_heal_body_matches_server_heal_options() {
let request = HealStartRequest {
Expand Down Expand Up @@ -1794,6 +1821,38 @@ mod tests {
handle.join().expect("server thread should finish");
}

#[tokio::test]
async fn test_heal_task_status_queries_root_route_with_client_token() {
let (endpoint, receiver, handle) = start_admin_test_server(
"200 OK",
r#"{"summary":"running","detail":"","startTime":"2026-07-15T00:38:07Z","settings":{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":1,"updateParity":false,"nolock":false},"items":[]}"#,
);
let client = admin_client_for_endpoint(&endpoint);

let status = client
.heal_task_status(HealTaskRequest {
bucket: String::new(),
prefix: None,
client_token: "root-token".to_string(),
})
.await
.expect("root heal task status request");

assert_eq!(status.heal_id, "root-token");
assert!(status.healing);
assert!(status.bucket.is_empty());
assert!(status.object.is_empty());

let request = receiver.recv().expect("captured request");
assert_eq!(request.method, "POST");
assert_eq!(
request.target,
"/rustfs/admin/v3/heal/?clientToken=root-token"
);
assert!(request.body.is_empty());
handle.join().expect("server thread should finish");
}

#[tokio::test]
async fn test_heal_task_status_queries_bucket_route_with_client_token() {
let (endpoint, receiver, handle) = start_admin_test_server(
Expand Down Expand Up @@ -1866,6 +1925,37 @@ mod tests {
handle.join().expect("server thread should finish");
}

#[tokio::test]
async fn test_heal_task_stop_posts_root_force_stop_with_client_token() {
let (endpoint, receiver, handle) = start_admin_test_server(
"200 OK",
r#"{"summary":"stopped","detail":"heal task cancelled","startTime":"2026-07-15T00:38:07Z","settings":{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":1,"updateParity":false,"nolock":false},"items":[]}"#,
);
let client = admin_client_for_endpoint(&endpoint);

let status = client
.heal_task_stop(HealTaskRequest {
bucket: String::new(),
prefix: None,
client_token: "root-token".to_string(),
})
.await
.expect("root heal task stop request");

assert_eq!(status.heal_id, "root-token");
assert!(!status.healing);
assert!(status.bucket.is_empty());

let request = receiver.recv().expect("captured request");
assert_eq!(request.method, "POST");
assert_eq!(
request.target,
"/rustfs/admin/v3/heal/?clientToken=root-token&forceStop=true"
);
assert!(request.body.is_empty());
handle.join().expect("server thread should finish");
}

#[tokio::test]
async fn test_heal_task_stop_posts_force_stop_with_client_token() {
let (endpoint, receiver, handle) = start_admin_test_server(
Expand Down
14 changes: 12 additions & 2 deletions docs/reference/rc/admin.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,14 @@ Check global background heal status:
rc admin heal status local
```

Start and inspect a root recursive manual heal:

```bash
rc admin heal start local
rc admin heal status local --client-token <TOKEN_FROM_START>
rc admin heal stop local --client-token <TOKEN_FROM_START>
```

Check a manual bucket heal task using the client token returned by `start`:

```bash
Expand Down Expand Up @@ -136,7 +144,7 @@ rc admin service restart local

Admin operations use the configured alias to create a RustFS admin client. The credentials behind the alias must have permissions for the requested administrative API. The command accepts aliases with or without a trailing slash.

`rc admin heal status <ALIAS>` reports aggregate background heal status. Manual bucket or prefix heals started with `rc admin heal start` are token-scoped tasks; the start output includes a client token, and subsequent task status or stop requests must pass that token with `--bucket`, optional `--prefix`, and `--client-token`.
`rc admin heal status <ALIAS>` reports aggregate background heal status. Manual heals started with `rc admin heal start` are token-scoped tasks; the start output includes a client token. Root recursive tasks are inspected or stopped with `--client-token`, while bucket or prefix tasks additionally pass `--bucket` and optional `--prefix`.

## Heal Workflow

Expand All @@ -145,9 +153,11 @@ Admin operations use the configured alias to create a RustFS admin client. The c
| Command | Description |
| --- | --- |
| `rc admin heal status <ALIAS>` | Show aggregate background heal status. |
| `rc admin heal status <ALIAS> --client-token <TOKEN>` | Show a token-scoped root recursive manual heal task. |
| `rc admin heal status <ALIAS> --bucket <BUCKET> [--prefix <PREFIX>] --client-token <TOKEN>` | Show a token-scoped manual heal task. |
| `rc admin heal start <ALIAS> [OPTIONS]` | Start a manual heal operation. |
| `rc admin heal stop <ALIAS>` | Stop the global background heal operation. |
| `rc admin heal stop <ALIAS> --client-token <TOKEN>` | Stop a token-scoped root recursive manual heal task. |
| `rc admin heal stop <ALIAS> --bucket <BUCKET> [--prefix <PREFIX>] --client-token <TOKEN>` | Stop a token-scoped manual heal task. |

`heal start` accepts these operation options:
Expand All @@ -161,7 +171,7 @@ Admin operations use the configured alias to create a RustFS admin client. The c
| `--recreate` | Recreate missing data. |
| `--dry-run` | Report what would be healed without applying changes. |

Manual bucket and prefix heals are token-scoped. Save the `clientToken` returned by `heal start`; the token is required to inspect or stop that manual task.
All manual heals are token-scoped. Save the `clientToken` returned by `heal start`; the token is required to inspect or stop the task. Root recursive tasks use the token alone, while bucket and prefix tasks also require their original target options.

## Decommission Workflow

Expand Down
Loading
Loading