fran/rustfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement list_parts API for S3 multipart upload compatibility (#209)

Browse Source 
* feat: add list_parts api
This commit is contained in:
weisd
2025-07-15 16:04:03 +08:00
committed by GitHub
parent 38cdc87e93
commit a9d77a618f
21 changed files with 810 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/test.json
-232
View File
@@ -1,232 +0,0 @@
{
"mode": "online",
"domain": null,
"region": null,
"sqsARN": null,
"deploymentID": null,
"buckets": {
"count": 0,
"error": null
},
"objects": {
"count": 0,
"error": null
},
"versions": {
"count": 0,
"error": null
},
"deletemarkers": {
"count": 0,
"error": null
},
"usage": {
"size": 0,
"error": null
},
"services": {
"kms": null,
"kmsStatus": null,
"ldap": null,
"logger": null,
"audit": null,
"notifications": null
},
"backend": {
"backendType": "Erasure",
"onlineDisks": 5,
"offlineDisks": 0,
"standardSCParity": 2,
"rrSCParity": 1,
"totalSets": [
1
],
"totalDrivesPerSet": [
5
]
},
"servers": [
{
"state": "online",
"endpoint": "127.0.0.1:9000",
"scheme": "",
"uptime": 1736146443,
"version": "",
"commitID": "",
"network": {
"127.0.0.1:9000": "online"
},
"drives": [
{
"endpoint": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test0",
"root_disk": true,
"drive_path": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test0",
"healing": false,
"scanning": true,
"state": "ok",
"uuid": "",
"major": 0,
"minor": 0,
"model": null,
"total_space": 494384795648,
"used_space": 283710812160,
"available_space": 210673983488,
"read_throughput": 0.0,
"write_throughput": 0.0,
"read_latency": 0.0,
"write_latency": 0.0,
"utilization": 57.386637828967736,
"metrics": null,
"heal_info": null,
"used_inodes": 2353357,
"free_inodes": 2057363120,
"local": true,
"pool_index": 0,
"set_index": 0,
"disk_index": 0
},
{
"endpoint": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test1",
"root_disk": true,
"drive_path": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test1",
"healing": false,
"scanning": true,
"state": "ok",
"uuid": "",
"major": 0,
"minor": 0,
"model": null,
"total_space": 494384795648,
"used_space": 283710812160,
"available_space": 210673983488,
"read_throughput": 0.0,
"write_throughput": 0.0,
"read_latency": 0.0,
"write_latency": 0.0,
"utilization": 57.386637828967736,
"metrics": null,
"heal_info": null,
"used_inodes": 2353357,
"free_inodes": 2057363120,
"local": true,
"pool_index": 0,
"set_index": 0,
"disk_index": 1
},
{
"endpoint": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test2",
"root_disk": true,
"drive_path": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test2",
"healing": false,
"scanning": false,
"state": "ok",
"uuid": "",
"major": 0,
"minor": 0,
"model": null,
"total_space": 494384795648,
"used_space": 283710812160,
"available_space": 210673983488,
"read_throughput": 0.0,
"write_throughput": 0.0,
"read_latency": 0.0,
"write_latency": 0.0,
"utilization": 57.386637828967736,
"metrics": null,
"heal_info": null,
"used_inodes": 2353357,
"free_inodes": 2057363120,
"local": true,
"pool_index": 0,
"set_index": 0,
"disk_index": 2
},
{
"endpoint": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test3",
"root_disk": true,
"drive_path": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test3",
"healing": false,
"scanning": false,
"state": "ok",
"uuid": "",
"major": 0,
"minor": 0,
"model": null,
"total_space": 494384795648,
"used_space": 283710812160,
"available_space": 210673983488,
"read_throughput": 0.0,
"write_throughput": 0.0,
"read_latency": 0.0,
"write_latency": 0.0,
"utilization": 57.386637828967736,
"metrics": null,
"heal_info": null,
"used_inodes": 2353357,
"free_inodes": 2057363120,
"local": true,
"pool_index": 0,
"set_index": 0,
"disk_index": 3
},
{
"endpoint": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test4",
"root_disk": true,
"drive_path": "/Users/weisd/project/weisd/s3-rustfs/target/volume/test4",
"healing": false,
"scanning": false,
"state": "ok",
"uuid": "",
"major": 0,
"minor": 0,
"model": null,
"total_space": 494384795648,
"used_space": 283710812160,
"available_space": 210673983488,
"read_throughput": 0.0,
"write_throughput": 0.0,
"read_latency": 0.0,
"write_latency": 0.0,
"utilization": 57.386637828967736,
"metrics": null,
"heal_info": null,
"used_inodes": 2353357,
"free_inodes": 2057363120,
"local": true,
"pool_index": 0,
"set_index": 0,
"disk_index": 4
}
],
"poolNumber": 1,
"poolNumbers": [
1
],
"mem_stats": {
"alloc": 0,
"total_alloc": 0,
"mallocs": 0,
"frees": 0,
"heap_alloc": 0
},
"max_procs": 0,
"num_cpu": 0,
"runtime_version": "",
"rustfs_env_vars": {}
}
],
"pools": {
"0": {
"0": {
"id": 0,
"rawUsage": 1418554060800,
"rawCapacity": 2471923978240,
"usage": 0,
"objectsCount": 0,
"versionsCount": 0,
"deleteMarkersCount": 0,
"healDisks": 0
}
}
}
}
rustfs/src/storage/ecfs.rs
+101 -4
View File
@@ -28,6 +28,7 @@ use chrono::Utc;
use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder;
use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder;
use datafusion::arrow::json::writer::JsonArray;
use rustfs_ecstore::set_disk::MAX_PARTS_COUNT;
use rustfs_s3select_api::object_store::bytes_stream;
use rustfs_s3select_api::query::Context;
use rustfs_s3select_api::query::Query;
@@ -1489,18 +1490,114 @@ impl S3 for FS {
#[tracing::instrument(level = "debug", skip(self, req))]
async fn list_parts(&self, req: S3Request<ListPartsInput>) -> S3Result<S3Response<ListPartsOutput>> {
let ListPartsInput {
bucket, key, upload_id, ..
bucket,
key,
upload_id,
part_number_marker,
max_parts,
..
} = req.input;
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let part_number_marker = part_number_marker.map(|x| x as usize);
let max_parts = max_parts.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT);
let res = store
.list_object_parts(&bucket, &key, &upload_id, part_number_marker, max_parts, &ObjectOptions::default())
.await
.map_err(ApiError::from)?;
let output = ListPartsOutput {
bucket: Some(bucket),
key: Some(key),
upload_id: Some(upload_id),
bucket: Some(res.bucket),
key: Some(res.object),
upload_id: Some(res.upload_id),
parts: Some(
res.parts
.into_iter()
.map(|p| Part {
e_tag: p.etag,
last_modified: p.last_mod.map(Timestamp::from),
part_number: Some(p.part_num as i32),
size: Some(p.size as i64),
..Default::default()
})
.collect(),
),
..Default::default()
};
Ok(S3Response::new(output))
}
async fn list_multipart_uploads(
&self,
req: S3Request<ListMultipartUploadsInput>,
) -> S3Result<S3Response<ListMultipartUploadsOutput>> {
let ListMultipartUploadsInput {
bucket,
prefix,
delimiter,
key_marker,
upload_id_marker,
max_uploads,
..
} = req.input;
let Some(store) = new_object_layer_fn() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let prefix = prefix.unwrap_or_default();
let max_uploads = max_uploads.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT);
if let Some(key_marker) = &key_marker {
if !key_marker.starts_with(prefix.as_str()) {
return Err(s3_error!(NotImplemented, "Invalid key marker"));
}
}
let result = store
.list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads)
.await
.map_err(ApiError::from)?;
let output = ListMultipartUploadsOutput {
bucket: Some(bucket),
prefix: Some(prefix),
delimiter: result.delimiter,
key_marker: result.key_marker,
upload_id_marker: result.upload_id_marker,
max_uploads: Some(result.max_uploads as i32),
is_truncated: Some(result.is_truncated),
uploads: Some(
result
.uploads
.into_iter()
.map(|u| MultipartUpload {
key: Some(u.object),
upload_id: Some(u.upload_id),
initiated: u.initiated.map(Timestamp::from),
..Default::default()
})
.collect(),
),
common_prefixes: Some(
result
.common_prefixes
.into_iter()
.map(|c| CommonPrefix { prefix: Some(c) })
.collect(),
),
..Default::default()
};
Ok(S3Response::new(output))
}
#[tracing::instrument(level = "debug", skip(self, req))]
async fn complete_multipart_upload(
&self,