From 1483c05abaf8ab394a30a6407496413667257744 Mon Sep 17 00:00:00 2001 From: Dongcheng Lin Date: Sat, 24 Oct 2026 22:07:39 +0900 Subject: [PATCH 1/2] feat(mcp): import xlsx workbooks with data.import --- crates/agent/src/capabilities.rs | 2 +- crates/cli/src/tool.rs | 6 +- crates/core/src/automation/mod.rs | 1 + crates/core/src/automation/native_import.rs | 12 + crates/core/src/automation/table_import.rs | 139 ++++++++--- .../core/src/automation/table_import_xlsx.rs | 156 ++++++++++++ .../src/automation/table_import_xlsx_tests.rs | 236 ++++++++++++++++++ crates/io/src/xlsx.rs | 163 +++++++++--- docs/src/content/docs/reference/cli.md | 2 +- docs/src/content/docs/reference/mcp.md | 27 +- docs/src/content/docs/zh-cn/reference/cli.md | 2 +- docs/src/content/docs/zh-cn/reference/mcp.md | 18 +- 12 files changed, 685 insertions(+), 79 deletions(-) create mode 100644 crates/core/src/automation/table_import_xlsx.rs create mode 100644 crates/core/src/automation/table_import_xlsx_tests.rs diff --git a/crates/agent/src/capabilities.rs b/crates/agent/src/capabilities.rs index 4395f68c..7cc4ef88 100644 --- a/crates/agent/src/capabilities.rs +++ b/crates/agent/src/capabilities.rs @@ -95,7 +95,7 @@ const PURPOSES: [(&str, &str); 28] = [ ), ( "data.import", - "Import CSV/TSV tables or native acquisitions", + "Import CSV/TSV/XLSX tables or native acquisitions", ), ( "data.transform", diff --git a/crates/cli/src/tool.rs b/crates/cli/src/tool.rs index 758223fd..cbc6e4a5 100644 --- a/crates/cli/src/tool.rs +++ b/crates/cli/src/tool.rs @@ -155,7 +155,7 @@ fn run_operation( Some(path) => { eprintln!("plotium-cli: loading {}", path.display()); let loaded = plotium_core::workflow::load_dataset(path).map_err(|e| { - if path.is_file() && is_delimited_table(path) { + if path.is_file() && is_table_file(path) { format!( "{e}. Tables are imported with data.import: omit the input and \ list the file's absolute path in parameters.paths with mode \ @@ -282,10 +282,10 @@ fn run_operation( serde_json::to_value(result).map_err(|e| e.to_string()) } -fn is_delimited_table(path: &std::path::Path) -> bool { +fn is_table_file(path: &std::path::Path) -> bool { path.extension() .and_then(|e| e.to_str()) - .is_some_and(|e| ["csv", "tsv", "txt"].contains(&e.to_ascii_lowercase().as_str())) + .is_some_and(|e| ["csv", "tsv", "txt", "xlsx"].contains(&e.to_ascii_lowercase().as_str())) } /// The empty selector for tools whose targets travel inside their parameters. diff --git a/crates/core/src/automation/mod.rs b/crates/core/src/automation/mod.rs index 921f8124..541467d0 100644 --- a/crates/core/src/automation/mod.rs +++ b/crates/core/src/automation/mod.rs @@ -5,6 +5,7 @@ mod native_import; pub use native_import::*; mod table_import; +mod table_import_xlsx; mod table_provenance; mod table_read; mod table_transform; diff --git a/crates/core/src/automation/native_import.rs b/crates/core/src/automation/native_import.rs index 411d80b7..f23ac09c 100644 --- a/crates/core/src/automation/native_import.rs +++ b/crates/core/src/automation/native_import.rs @@ -35,6 +35,18 @@ impl NativeImportParams { "Native import requires 1–16 absolute explicit data files or atomic acquisition directories.", )); } + if let Some(table) = self.paths.iter().find(|path| { + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| { + ["csv", "tsv", "xlsx"].contains(&ext.to_ascii_lowercase().as_str()) + }) + }) { + return Err(invalid(format!( + "{} is a table; import CSV, TSV and XLSX files with mode table_data_only.", + table.display() + ))); + } Ok(()) } } diff --git a/crates/core/src/automation/table_import.rs b/crates/core/src/automation/table_import.rs index 84d8c5ad..8dfcaae3 100644 --- a/crates/core/src/automation/table_import.rs +++ b/crates/core/src/automation/table_import.rs @@ -20,6 +20,9 @@ pub const TABLE_IMPORT_FILE_BYTES: u64 = 16 * 1024 * 1024; pub const TABLE_IMPORT_TOTAL_BYTES: u64 = 64 * 1024 * 1024; pub const TABLE_IMPORT_COLUMNS: usize = 256; pub const TABLE_IMPORT_ROWS: usize = 1_000_000; +/// Worksheet ranges are read densely, so XLSX is bounded by cells read rather +/// than by its compressed size. +pub const TABLE_IMPORT_XLSX_CELLS: usize = 8_000_000; #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] @@ -39,6 +42,7 @@ pub enum ImportHeader { pub enum ImportFormat { Csv, Tsv, + Xlsx, } #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -48,6 +52,8 @@ pub struct TableImportParams { #[serde(default)] pub header: ImportHeader, pub format: Option, + /// XLSX worksheets to import by name; every visible worksheet when absent. + pub sheets: Option>, /// An explicit ordered schema, not display-name-based resource addressing. pub schema: Option, pub schema_path: Option, @@ -58,9 +64,10 @@ pub fn is_table_import(value: &Value) -> bool { pub fn table_import_schema() -> Value { json!({"type":"object","additionalProperties":false,"required":["mode","paths"],"properties":{ "mode":{"const":"table_data_only"},"paths":{"type":"array","minItems":1,"maxItems":TABLE_IMPORT_FILES,"items":{"type":"string"}}, - "header":{"enum":["auto","present","absent"],"default":"auto"},"format":{"enum":["csv","tsv"]}, + "header":{"enum":["auto","present","absent"],"default":"auto","description":"CSV/TSV only."},"format":{"enum":["csv","tsv","xlsx"]}, + "sheets":{"type":"array","minItems":1,"items":{"type":"string"},"description":"XLSX worksheet names, each imported as its own table. Default: every visible worksheet."}, "schema":table_declaration_schema(), - "schema_path":{"type":"string","description":"Explicit PlotiumDelimitedSchemaV1 JSON file. Never discovered automatically."}}, + "schema_path":{"type":"string","description":"Explicit PlotiumDelimitedSchemaV1 JSON file for CSV/TSV. Never discovered automatically."}}, "not":{"required":["schema","schema_path"]}}) } pub fn table_declaration_schema() -> Value { @@ -75,7 +82,7 @@ pub fn table_declaration_schema() -> Value { "business_keys":{"type":"array","maxItems":0},"metadata":{"type":"object"}}}) } pub fn table_import_limits() -> Value { - json!({"files":TABLE_IMPORT_FILES,"file_bytes":TABLE_IMPORT_FILE_BYTES,"total_bytes":TABLE_IMPORT_TOTAL_BYTES,"columns":TABLE_IMPORT_COLUMNS,"rows_per_file":TABLE_IMPORT_ROWS,"prepared_plans":4,"in_flight":1,"schema_file_bytes":TABLE_RESPONSE_BYTES,"prepared_summary_bytes":TABLE_RESPONSE_BYTES/2,"preflight_response_bytes":TABLE_RESPONSE_BYTES}) + json!({"files":TABLE_IMPORT_FILES,"file_bytes":TABLE_IMPORT_FILE_BYTES,"total_bytes":TABLE_IMPORT_TOTAL_BYTES,"columns":TABLE_IMPORT_COLUMNS,"rows_per_file":TABLE_IMPORT_ROWS,"xlsx_cells_per_file":TABLE_IMPORT_XLSX_CELLS,"prepared_plans":4,"in_flight":1,"schema_file_bytes":TABLE_RESPONSE_BYTES,"prepared_summary_bytes":TABLE_RESPONSE_BYTES/2,"preflight_response_bytes":TABLE_RESPONSE_BYTES}) } fn invalid(message: impl Into) -> AutomationError { AutomationError::InvalidParameters { @@ -96,6 +103,9 @@ impl TableImportParams { if self.paths.is_empty() { return Err(invalid("Import requires 1–16 explicit files.")); } + if self.sheets.as_ref().is_some_and(Vec::is_empty) { + return Err(invalid("sheets must name at least one worksheet.")); + } if self.schema.is_some() && self.schema_path.is_some() { return Err(invalid("Supply schema or schema_path, not both.")); } @@ -115,10 +125,22 @@ impl TableImportParams { } struct ImportItem { path: PathBuf, + /// The source worksheet of an XLSX table. + sheet: Option, dataset: Option, summary: Value, error: Option, } +impl ImportItem { + fn json(&self, status: &str) -> Value { + let mut value = + json!({"path":self.path,"status":status,"metadata":self.summary,"error":self.error}); + if let Some(sheet) = &self.sheet { + value["sheet"] = json!(sheet); + } + value + } +} pub struct PreparedTableImport { items: Vec, params: TableImportParams, @@ -160,20 +182,30 @@ impl PreparedTableImport { if cancelled() { return Err(AutomationError::PreviewCancelled); } - let result = prepare_file(path, ¶ms, schema.clone(), source, &mut budget); - let (dataset, summary, error) = match result { - Ok(dataset) => { - let summary = table_metadata(&dataset)?; - (Some(dataset), summary, None) + match prepare_file(path, ¶ms, schema.clone(), source, &mut budget) { + Ok(tables) => { + for (sheet, dataset) in tables { + let (summary, dataset, error) = match dataset { + Ok(dataset) => (table_metadata(&dataset)?, Some(dataset), None), + Err(error) => (Value::Null, None, Some(error)), + }; + items.push(ImportItem { + path: path.clone(), + sheet, + summary, + dataset, + error, + }); + } } - Err(error) => (None, Value::Null, Some(error)), - }; - items.push(ImportItem { - path: path.clone(), - dataset, - summary, - error, - }); + Err(error) => items.push(ImportItem { + path: path.clone(), + sheet: None, + dataset: None, + summary: Value::Null, + error: Some(error), + }), + } } let prepared = Self { items, params }; if serde_json::to_vec(&prepared.preflight()) @@ -181,12 +213,41 @@ impl PreparedTableImport { .len() > TABLE_RESPONSE_BYTES / 2 { - return Err(AutomationError::ResponseTooLarge); + let mut workbooks: Vec<(&Path, Vec)> = Vec::new(); + for item in &prepared.items { + let Some(sheet) = &item.sheet else { continue }; + let name = format!("'{sheet}'"); + match workbooks.iter_mut().find(|(path, _)| *path == item.path) { + Some((_, names)) => names.push(name), + None => workbooks.push((&item.path, vec![name])), + } + } + if workbooks.is_empty() { + return Err(AutomationError::ResponseTooLarge); + } + let listed = workbooks + .iter() + .map(|(path, names)| format!("{}: {}", path.display(), names.join(", "))) + .collect::>() + .join("; "); + return Err(invalid(format!( + "The import summary exceeds 128 KiB; import one workbook at a time and name fewer worksheets in sheets. Worksheets by file: {listed}." + ))); } Ok(prepared) } pub fn preflight(&self) -> Value { - json!({"phase":"preflight","prepared_files":self.items.iter().filter(|i|i.dataset.is_some()).count(),"failed_files":self.items.iter().filter(|i|i.error.is_some()).count(),"status":if self.status()=="succeeded"{"prepared"}else{self.status()},"items":self.items.iter().map(|item| json!({"path":item.path,"status":if item.error.is_some(){"failed"}else{"prepared"},"metadata":item.summary,"error":item.error})).collect::>()}) + json!({"phase":"preflight","prepared_files":self.files(|i|i.dataset.is_some()),"failed_files":self.files(|i|i.error.is_some()),"status":if self.status()=="succeeded"{"prepared"}else{self.status()},"items":self.items.iter().map(|item| item.json(if item.error.is_some(){"failed"}else{"prepared"})).collect::>()}) + } + /// Counts distinct files with a matching item; a workbook has an item per table. + fn files(&self, matches: impl Fn(&ImportItem) -> bool) -> usize { + let mut paths = Vec::new(); + for item in self.items.iter().filter(|item| matches(item)) { + if !paths.contains(&&item.path) { + paths.push(&item.path); + } + } + paths.len() } fn status(&self) -> &'static str { let success = self.items.iter().filter(|i| i.dataset.is_some()).count(); @@ -218,9 +279,9 @@ impl PreparedTableImport { ); let mut actions = Vec::new(); let mut values = Vec::new(); - for item in self.items { + for mut item in self.items { let success = item.dataset.is_some(); - if let Some(dataset) = item.dataset { + if let Some(dataset) = item.dataset.take() { result.produced.push(dataset.resource_id().into()); actions.push(Action::InsertTableData { dataset: Box::new(dataset), @@ -238,14 +299,14 @@ impl PreparedTableImport { } else { TargetOutcome::Failed }, - message: item - .error - .clone() - .unwrap_or_else(|| "Imported native table data.".into()), + message: item.error.clone().unwrap_or_else(|| match &item.sheet { + Some(sheet) => format!("Imported worksheet '{sheet}' as native table data."), + None => "Imported native table data.".into(), + }), skip_reason: None, fingerprints: Vec::new(), }); - values.push(json!({"path":item.path,"status":if success{"succeeded"}else{"failed"},"metadata":item.summary,"error":item.error})); + values.push(item.json(if success { "succeeded" } else { "failed" })); } if !actions.is_empty() { app.try_execute_action(Action::Composite(actions)) @@ -309,26 +370,47 @@ fn read_selected(path: &Path, budget: &mut u64, limit: u64) -> Result, S } Ok(bytes) } +/// Prepares every table in one file, paired with its XLSX worksheet name. fn prepare_file( path: &Path, params: &TableImportParams, schema: Option, source: &str, budget: &mut u64, -) -> Result { +) -> Result, String> { let format = params .format .or_else(|| match path.extension().and_then(|s| s.to_str()) { Some(ext) if ext.eq_ignore_ascii_case("csv") => Some(ImportFormat::Csv), Some(ext) if ext.eq_ignore_ascii_case("tsv") => Some(ImportFormat::Tsv), + Some(ext) if ext.eq_ignore_ascii_case("xlsx") => Some(ImportFormat::Xlsx), _ => None, }) - .ok_or("Specify CSV or TSV format for this file.")?; + .ok_or("Specify CSV, TSV or XLSX format for this file.")?; + if format == ImportFormat::Xlsx { + if params.header != ImportHeader::Auto { + return Err( + "header applies to CSV/TSV; XLSX detects each worksheet's header row.".into(), + ); + } + if schema.is_some() { + return Err( + "A declared schema applies to CSV/TSV; XLSX keeps the column types stored in the workbook." + .into(), + ); + } + let bytes = read_selected(path, budget, TABLE_IMPORT_FILE_BYTES)?; + return super::table_import_xlsx::prepare(path, bytes, params.sheets.as_deref()); + } + if params.sheets.is_some() { + return Err("sheets applies to XLSX workbooks only.".into()); + } let bytes = read_selected(path, budget, TABLE_IMPORT_FILE_BYTES)?; let input = std::str::from_utf8(&bytes).map_err(|e| format!("CSV/TSV must be UTF-8: {e}"))?; let delimiter = match format { ImportFormat::Csv => Delimiter::Comma, ImportFormat::Tsv => Delimiter::Tab, + ImportFormat::Xlsx => unreachable!("XLSX returned above"), }; let parsed = crate::delimited::parse_delimited_columns( input, @@ -375,11 +457,12 @@ fn prepare_file( match format { ImportFormat::Csv => "text/csv", ImportFormat::Tsv => "text/tab-separated-values", + ImportFormat::Xlsx => unreachable!("XLSX returned above"), }, ); retained.name = path.file_name().map(|s| s.to_string_lossy().into_owned()); table.import_sources.push(retained); - Ok(Dataset::Table(Box::new(table))) + Ok(vec![(None, Ok(Dataset::Table(Box::new(table))))]) } #[cfg(test)] diff --git a/crates/core/src/automation/table_import_xlsx.rs b/crates/core/src/automation/table_import_xlsx.rs new file mode 100644 index 00000000..1eca778e --- /dev/null +++ b/crates/core/src/automation/table_import_xlsx.rs @@ -0,0 +1,156 @@ +//! XLSX workbooks import one native table per worksheet, as the desktop does. +use super::table_import::{TABLE_IMPORT_COLUMNS, TABLE_IMPORT_ROWS, TABLE_IMPORT_XLSX_CELLS}; +use crate::{ + state::{Dataset, TableDataset, TableImportSource, TypedTableState}, + xlsx::ImportedXlsxSheet, +}; +use plotium_data::{CodecRegistry, MemoryBlockStore}; +use plotium_io::xlsx::{XlsxIoError, XlsxReader, XlsxWorkbook}; +use std::{path::Path, sync::Arc}; + +const XLSX_MEDIA_TYPE: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + +/// A prepared table, or why it failed, with the worksheet it came from. +pub(super) type PreparedTable = (Option, Result); + +/// Parses the retained bytes worksheet by worksheet, so one worksheet that +/// fails or exceeds the cell budget leaves the others importable. `sheets` +/// selects worksheets by name; otherwise every visible worksheet with cells is +/// imported. +pub(super) fn prepare( + path: &Path, + bytes: Vec, + sheets: Option<&[String]>, +) -> Result, String> { + let bytes = Arc::<[u8]>::from(bytes); + let mut reader = + XlsxReader::from_bytes(&bytes, TABLE_IMPORT_XLSX_CELLS).map_err(|e| e.to_string())?; + let visible = reader + .sheets + .iter() + .filter(|sheet| !sheet.hidden) + .map(|sheet| sheet.name.clone()) + .collect::>(); + let selected = match sheets { + Some(names) => { + if let Some(missing) = names.iter().find(|name| !visible.contains(name)) { + let available = visible + .iter() + .map(|name| format!("'{name}'")) + .collect::>() + .join(", "); + return Err(format!( + "Worksheet '{missing}' is not a visible worksheet of this workbook. Visible worksheets: {available}." + )); + } + visible + .iter() + .filter(|name| names.contains(name)) + .cloned() + .collect() + } + None => visible, + }; + let mut remaining = TABLE_IMPORT_XLSX_CELLS; + let mut read = Vec::new(); + for name in selected { + match reader.read_sheet(&name, remaining) { + Ok(sheet) => { + let width = sheet.rows.iter().map(Vec::len).max().unwrap_or(0); + remaining = remaining.saturating_sub(sheet.rows.len() * width); + // Workbooks often keep blank default sheets, which hold no table. + if sheets.is_some() || !sheet.rows.is_empty() { + read.push((name, Ok(sheet))); + } + } + Err(XlsxIoError::SheetOverBudget { .. }) => read.push(( + name, + Err(format!( + "The worksheet exceeds the cell budget: one XLSX import reads at most {TABLE_IMPORT_XLSX_CELLS} cells across its worksheets. Name fewer worksheets in sheets." + )), + )), + Err(error) => read.push((name, Err(error.to_string()))), + } + } + if read.is_empty() { + return Err("The workbook has no visible worksheet with cells.".into()); + } + let shared = reader.into_workbook(Vec::new()); + let file_stem = (sheets.is_none() && read.len() == 1).then(|| { + path.file_stem() + .unwrap_or_default() + .to_string_lossy() + .into_owned() + }); + let retained = Retained { + bytes, + file_name: path.file_name().map(|s| s.to_string_lossy().into_owned()), + store: Arc::new(MemoryBlockStore::default()), + }; + let codecs = CodecRegistry::with_arrow_ipc(); + let mut prepared = Vec::new(); + for (name, sheet) in read { + let tables = sheet.and_then(|sheet| { + let workbook = XlsxWorkbook { + sheets: vec![sheet], + ..shared.clone() + }; + crate::xlsx::import_xlsx_workbook(&workbook, retained.store.as_ref(), &codecs) + .map_err(|e| e.to_string()) + }); + match tables { + Ok(tables) => prepared.extend(tables.into_iter().map(|table| { + let dataset_name = file_stem.clone().unwrap_or_else(|| table.name.clone()); + ( + Some(table.name.clone()), + retained.dataset(table, dataset_name), + ) + })), + Err(error) => prepared.push((Some(name), Err(error))), + } + } + Ok(prepared) +} + +/// What every table of one workbook shares. +struct Retained { + bytes: Arc<[u8]>, + file_name: Option, + store: Arc, +} + +impl Retained { + fn dataset(&self, sheet: ImportedXlsxSheet, name: String) -> Result { + if sheet.snapshot.schema.columns.len() > TABLE_IMPORT_COLUMNS + || sheet.snapshot.row_count > TABLE_IMPORT_ROWS as u64 + { + return Err(format!( + "The worksheet exceeds the budget of {TABLE_IMPORT_COLUMNS} columns and {TABLE_IMPORT_ROWS} rows per table." + )); + } + let typed = TypedTableState::imported_with_operation( + sheet.snapshot, + Arc::clone(&self.store), + "plotium.import.xlsx.v1", + ) + .map_err(|e| e.to_string())?; + let mut table = TableDataset::from_typed(typed); + table.name = Some(name); + let mut source = TableImportSource::new(Arc::clone(&self.bytes), XLSX_MEDIA_TYPE); + source.name = self.file_name.clone(); + source.metadata.insert( + "space.nmrtist.plotium.import.worksheet".into(), + serde_json::Value::String(sheet.name), + ); + source.metadata.insert( + "space.nmrtist.plotium.import.formula_policy".into(), + serde_json::Value::String("cached-values-only".into()), + ); + table.import_sources.push(source); + Ok(Dataset::Table(Box::new(table))) + } +} + +#[cfg(test)] +#[path = "table_import_xlsx_tests.rs"] +mod tests; diff --git a/crates/core/src/automation/table_import_xlsx_tests.rs b/crates/core/src/automation/table_import_xlsx_tests.rs new file mode 100644 index 00000000..307363c2 --- /dev/null +++ b/crates/core/src/automation/table_import_xlsx_tests.rs @@ -0,0 +1,236 @@ +use super::super::*; +use crate::state::PlotiumApp; +use plotium_io::xlsx::{XlsxCell, XlsxSheet, XlsxValue, XlsxWorkbook, write_xlsx}; +use serde_json::{Value, json}; +use std::path::PathBuf; + +fn text(value: &str) -> XlsxCell { + XlsxCell::value(XlsxValue::Utf8(value.into())) +} + +fn number(value: f64) -> XlsxCell { + XlsxCell::value(XlsxValue::Float64(value)) +} + +fn sheet(name: &str, rows: Vec>) -> XlsxSheet { + XlsxSheet { + name: name.into(), + hidden: false, + rows, + } +} + +fn workbook(sheets: Vec) -> PathBuf { + let path = + std::env::temp_dir().join(format!("plotium-xlsx-import-{}.xlsx", uuid::Uuid::new_v4())); + write_xlsx( + &path, + &XlsxWorkbook { + sheets, + uses_1904_date_system: false, + plotium_schema: None, + diagnostics: Vec::new(), + }, + ) + .unwrap(); + path +} + +fn curve(name: &str, scale: f64) -> XlsxSheet { + sheet( + name, + vec![ + vec![text("Voltage (V)"), text("Current (mA)")], + vec![number(0.0), number(scale)], + vec![number(1.0), number(2.0 * scale)], + ], + ) +} + +fn prepare(params: Value) -> PreparedTableImport { + PreparedTableImport::prepare(serde_json::from_value(params).unwrap(), || false).unwrap() +} + +fn request(app: &PlotiumApp, params: Value) -> ToolRequest { + ToolRequest { + tool_id: "data.import".into(), + tool_version: 1, + parameters: params, + targets: TargetSelector::Explicit { ids: Vec::new() }, + expected_revision: DocumentRevision(app.doc.automation_revision), + caller: CallerType::Agent, + } +} + +#[test] +fn xlsx_imports_each_worksheet_with_cells_as_a_named_table() { + let path = workbook(vec![ + curve("Figure 1", 1.0), + sheet("Blank", Vec::new()), + curve("Figure 2", 5.0), + ]); + let params = json!({"mode":"table_data_only","paths":[path]}); + let prepared = prepare(params.clone()); + let preflight = prepared.preflight(); + assert_eq!(preflight["status"], "prepared", "{preflight}"); + let sheets = preflight["items"] + .as_array() + .unwrap() + .iter() + .map(|item| item["sheet"].as_str().unwrap()) + .collect::>(); + assert_eq!(sheets, ["Figure 1", "Figure 2"]); + assert_eq!(preflight["items"][0]["metadata"]["total_rows"], 2); + + let mut app = PlotiumApp::new(); + let plan = plan_tool(&app, request(&app, params)).unwrap(); + let result = execute_prepared_table_import( + &mut app, + plan, + prepared, + ExecutionAuthority::ReversibleModify, + ) + .unwrap(); + std::fs::remove_file(path).unwrap(); + assert_eq!(result.value["status"], "succeeded"); + let names = app + .doc + .datasets + .iter() + .map(|dataset| dataset.name().unwrap()) + .collect::>(); + assert_eq!(names, ["Figure 1", "Figure 2"]); + let table = app.doc.datasets[1].as_table().unwrap(); + let columns = &table.typed_state.envelope.revision.snapshot.schema.columns; + assert_eq!(columns[1].name, "Current (mA)"); + assert_eq!( + table.import_sources[0].metadata["space.nmrtist.plotium.import.worksheet"], + "Figure 2" + ); +} + +#[test] +fn xlsx_sheets_selects_worksheets_and_names_the_visible_ones_when_missing() { + let path = workbook(vec![curve("Figure 1", 1.0), curve("Figure 2", 5.0)]); + let selected = prepare(json!({"mode":"table_data_only","paths":[path],"sheets":["Figure 2"]})); + let preflight = selected.preflight(); + assert_eq!(preflight["items"].as_array().unwrap().len(), 1); + assert_eq!(preflight["items"][0]["sheet"], "Figure 2"); + + let missing = prepare(json!({"mode":"table_data_only","paths":[path],"sheets":["Figure 3"]})); + std::fs::remove_file(&path).unwrap(); + let error = missing.preflight()["items"][0]["error"] + .as_str() + .unwrap() + .to_owned(); + assert!(error.contains("'Figure 3'"), "{error}"); + assert!(error.contains("'Figure 1', 'Figure 2'"), "{error}"); +} + +#[test] +fn xlsx_rejects_csv_only_options_and_csv_rejects_sheets() { + let path = workbook(vec![curve("Figure 1", 1.0)]); + let header = prepare(json!({"mode":"table_data_only","paths":[path],"header":"present"})); + std::fs::remove_file(&path).unwrap(); + assert!( + header.preflight()["items"][0]["error"] + .as_str() + .unwrap() + .contains("header applies to CSV/TSV") + ); + + let csv = + std::env::temp_dir().join(format!("plotium-xlsx-import-{}.csv", uuid::Uuid::new_v4())); + std::fs::write(&csv, "x,y\n0,1\n").unwrap(); + let sheets = prepare(json!({"mode":"table_data_only","paths":[csv],"sheets":["Data"]})); + std::fs::remove_file(&csv).unwrap(); + assert!( + sheets.preflight()["items"][0]["error"] + .as_str() + .unwrap() + .contains("XLSX workbooks only") + ); +} + +#[test] +fn native_import_points_tables_at_table_data_only() { + let params: NativeImportParams = + serde_json::from_value(json!({"paths":["/data/Source Data.xlsx"]})).unwrap(); + let message = params.validate().unwrap_err().to_string(); + assert!(message.contains("mode table_data_only"), "{message}"); +} + +#[test] +fn xlsx_worksheet_failures_leave_the_other_worksheets_importable() { + let path = workbook(vec![ + curve("Figure 1", 1.0), + sheet("Notes", vec![vec![text("Source data for Fig. 1")]]), + ]); + let preflight = prepare(json!({"mode":"table_data_only","paths":[path]})).preflight(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(preflight["status"], "partial", "{preflight}"); + assert_eq!(preflight["items"][0]["status"], "prepared"); + assert_eq!(preflight["items"][1]["sheet"], "Notes"); + assert_eq!(preflight["items"][1]["status"], "failed"); + assert_eq!(preflight["prepared_files"], 1); +} + +#[test] +fn xlsx_worksheets_over_the_cell_budget_fail_before_they_are_read() { + // A few kilobytes that would expand to more than the cell budget. + let mut rows = vec![vec![text("x")]]; + rows.extend((0..400_000).map(|_| Vec::new())); + rows.push( + (0..25) + .map(|_| XlsxCell::value(XlsxValue::Empty)) + .chain([number(1.0)]) + .collect(), + ); + let path = workbook(vec![sheet("Sparse", rows), curve("Figure 1", 1.0)]); + let preflight = prepare(json!({"mode":"table_data_only","paths":[path]})).preflight(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(preflight["items"][0]["sheet"], "Sparse"); + let error = preflight["items"][0]["error"].as_str().unwrap(); + assert!(error.contains("cell budget"), "{error}"); + assert_eq!(preflight["items"][1]["status"], "prepared"); +} + +#[test] +fn xlsx_with_one_table_is_named_after_the_file() { + let path = workbook(vec![curve("Data", 1.0), sheet("Sheet2", Vec::new())]); + let params = json!({"mode":"table_data_only","paths":[path],"format":"xlsx"}); + let prepared = prepare(params.clone()); + let mut app = PlotiumApp::new(); + let plan = plan_tool(&app, request(&app, params)).unwrap(); + execute_prepared_table_import( + &mut app, + plan, + prepared, + ExecutionAuthority::ReversibleModify, + ) + .unwrap(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(app.doc.datasets.len(), 1); + assert_eq!( + app.doc.datasets[0].name().as_deref(), + path.file_stem().and_then(|stem| stem.to_str()) + ); +} + +#[test] +fn xlsx_rejects_a_declared_schema() { + let path = workbook(vec![curve("Figure 1", 1.0)]); + let schema = plotium_data::TableSchema::new(vec![plotium_data::ColumnSchema::new( + "x", + plotium_data::LogicalType::Float64, + )]) + .unwrap(); + let preflight = + prepare(json!({"mode":"table_data_only","paths":[path],"schema":schema})).preflight(); + std::fs::remove_file(&path).unwrap(); + let error = preflight["items"][0]["error"].as_str().unwrap(); + assert!( + error.contains("declared schema applies to CSV/TSV"), + "{error}" + ); +} diff --git a/crates/io/src/xlsx.rs b/crates/io/src/xlsx.rs index 97d06b0a..6b393d2d 100644 --- a/crates/io/src/xlsx.rs +++ b/crates/io/src/xlsx.rs @@ -7,7 +7,10 @@ use calamine::{Data, HeaderRow, Reader, SheetVisible, Xlsx, open_workbook}; use rust_xlsxwriter::{Workbook, XlsxError as WriteError}; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; -use std::path::Path; +use std::{ + io::{Cursor, Read, Seek}, + path::Path, +}; pub const PLOTIUM_SCHEMA_SHEET: &str = "_Plotium_schema_v1"; const PLOTIUM_SCHEMA_MARKER: &str = "plotium-schema-v1"; @@ -90,50 +93,146 @@ pub enum XlsxIoError { Unsupported1904DateSystem, #[error("worksheet row or column exceeds the XLSX format limit")] SheetTooLarge, + #[error("worksheet '{sheet}' spans more than {max_cells} cells")] + SheetOverBudget { sheet: String, max_cells: usize }, } pub fn read_xlsx(path: impl AsRef) -> Result { - let mut reader: Xlsx<_> = open_workbook(path)?; - reader.with_header_row(HeaderRow::Row(0)); - let uses_1904_date_system = reader.has_1904_epoch(); - let metadata = reader.sheets_metadata().to_vec(); - let mut sheets = Vec::new(); - let mut plotium_schema = None; - let mut diagnostics = Vec::new(); - - for metadata in metadata { - let values = reader.worksheet_range(&metadata.name)?; - let formulas = reader.worksheet_formula(&metadata.name)?; - if metadata.name == PLOTIUM_SCHEMA_SHEET { + let mut reader = XlsxReader::new(open_workbook(path)?, usize::MAX)?; + let names = reader.sheets.clone(); + let sheets = names + .iter() + .map(|sheet| reader.read_sheet(&sheet.name, usize::MAX)) + .collect::, _>>()?; + Ok(reader.into_workbook(sheets)) +} + +/// A worksheet's name and visibility, known before its cells are read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct XlsxSheetInfo { + pub name: String, + pub hidden: bool, +} + +/// Reads worksheets one at a time, so callers can select worksheets and bound +/// each one's size before its cells are materialized. +pub struct XlsxReader { + reader: Xlsx, + /// Data worksheets in workbook order, without the hidden schema sheet. + pub sheets: Vec, + pub uses_1904_date_system: bool, + pub plotium_schema: Option, + pub diagnostics: Vec, +} + +impl<'a> XlsxReader> { + /// Opens a workbook already held in memory, so callers that budget and + /// retain the source bytes parse exactly what they retain. `max_cells` + /// bounds the hidden schema sheet. + pub fn from_bytes(bytes: &'a [u8], max_cells: usize) -> Result { + Self::new(Xlsx::new(Cursor::new(bytes))?, max_cells) + } +} + +impl XlsxReader { + fn new(mut reader: Xlsx, max_cells: usize) -> Result { + reader.with_header_row(HeaderRow::Row(0)); + let uses_1904_date_system = reader.has_1904_epoch(); + let metadata = reader.sheets_metadata().to_vec(); + let mut diagnostics = Vec::new(); + let mut plotium_schema = None; + if metadata + .iter() + .any(|sheet| sheet.name == PLOTIUM_SCHEMA_SHEET) + { + check_extent(&mut reader, PLOTIUM_SCHEMA_SHEET, max_cells)?; + let values = reader.worksheet_range(PLOTIUM_SCHEMA_SHEET)?; plotium_schema = read_schema(&values, &mut diagnostics); - continue; } + let sheets = metadata + .into_iter() + .filter(|sheet| sheet.name != PLOTIUM_SCHEMA_SHEET) + .map(|sheet| XlsxSheetInfo { + hidden: sheet.visible != SheetVisible::Visible, + name: sheet.name, + }) + .collect(); + Ok(Self { + reader, + sheets, + uses_1904_date_system, + plotium_schema, + diagnostics, + }) + } + + /// Reads one worksheet, failing before materializing it when its used + /// range spans more than `max_cells` cells. Cell diagnostics accumulate + /// in `diagnostics`. + pub fn read_sheet(&mut self, name: &str, max_cells: usize) -> Result { + let hidden = self + .sheets + .iter() + .find(|sheet| sheet.name == name) + .ok_or_else(|| calamine::XlsxError::WorksheetNotFound(name.to_owned()))? + .hidden; + check_extent(&mut self.reader, name, max_cells)?; + let values = self.reader.worksheet_range(name)?; + let formulas = self.reader.worksheet_formula(name)?; let start = union_start(values.start(), formulas.start()); let end = union_end(values.end(), formulas.end()); let rows = match (start, end) { - (Some(start), Some(end)) => read_rows( - &metadata.name, - start, - end, - &values, - &formulas, - &mut diagnostics, - ), + (Some(start), Some(end)) => { + read_rows(name, start, end, &values, &formulas, &mut self.diagnostics) + } _ => Vec::new(), }; - sheets.push(XlsxSheet { - name: metadata.name, - hidden: metadata.visible != SheetVisible::Visible, + Ok(XlsxSheet { + name: name.to_owned(), + hidden, rows, - }); + }) } - Ok(XlsxWorkbook { - sheets, - uses_1904_date_system, - plotium_schema, - diagnostics, - }) + pub fn into_workbook(self, sheets: Vec) -> XlsxWorkbook { + XlsxWorkbook { + sheets, + uses_1904_date_system: self.uses_1904_date_system, + plotium_schema: self.plotium_schema, + diagnostics: self.diagnostics, + } + } +} + +/// Streams a worksheet's cell positions and fails when the used range they +/// span exceeds `max_cells`, since reading a range allocates it densely. +fn check_extent( + reader: &mut Xlsx, + name: &str, + max_cells: usize, +) -> Result<(), XlsxIoError> { + if max_cells == usize::MAX { + return Ok(()); + } + let mut cells = reader.worksheet_cells_reader(name)?; + let mut extent: Option<((u32, u32), (u32, u32))> = None; + while let Some(cell) = cells.next_cell_with_formula()? { + let (row, column) = cell.pos; + let ((top, left), (bottom, right)) = extent.get_or_insert(((row, column), (row, column))); + *top = (*top).min(row); + *left = (*left).min(column); + *bottom = (*bottom).max(row); + *right = (*right).max(column); + let rows = (*bottom - *top) as usize + 1; + let columns = (*right - *left) as usize + 1; + if rows.saturating_mul(columns) > max_cells { + return Err(XlsxIoError::SheetOverBudget { + sheet: name.to_owned(), + max_cells, + }); + } + } + Ok(()) } pub fn write_xlsx(path: impl AsRef, workbook: &XlsxWorkbook) -> Result<(), XlsxIoError> { diff --git a/docs/src/content/docs/reference/cli.md b/docs/src/content/docs/reference/cli.md index 27cbdf84..ef6a884a 100644 --- a/docs/src/content/docs/reference/cli.md +++ b/docs/src/content/docs/reference/cli.md @@ -250,7 +250,7 @@ nonzero exit code. Optional `--output` saves the native project. `process` also uses the shared scheme service before exporting its default plot. Without an input, `tool run` starts from an empty project. Use this to begin -from CSV or TSV tables, which are imported with `data.import` rather than +from CSV, TSV or XLSX tables, which are imported with `data.import` rather than loaded as an input. Import paths must be absolute. ```sh diff --git a/docs/src/content/docs/reference/mcp.md b/docs/src/content/docs/reference/mcp.md index df0a62a9..0b4bf935 100644 --- a/docs/src/content/docs/reference/mcp.md +++ b/docs/src/content/docs/reference/mcp.md @@ -70,7 +70,7 @@ including query inputs needed to reproduce those operations. Handshakes, tool li the static property catalog and save coordination (planning, submission and `plotium_save_status`), and export planning/submission/status do not add audit records. -The connection supports native acquisition import and CSV/TSV data-only import, native table queries, canvas previews, figure geometry diagnostics, +The connection supports native acquisition import and CSV/TSV/XLSX data-only import, native table queries, canvas previews, figure geometry diagnostics, document structure, native plot creation and page/panel/text/local-image authoring, explicit numeric curve bindings, supported table transforms, cell patches, explicit saved-recipe refreshes and resource queries, renaming datasets, canvases and canvas objects, applying built-in themes or saved canvas styles to canvases, and these display properties only: @@ -377,7 +377,8 @@ reservations total at most 512 MiB across retained plans and the active prefligh oversized batches fail with `import_budget_exceeded`. A call with more than 16 paths fails with a message that gives the number received; import the rest in further calls. Preflight responses are limited to 256 KiB. Scientific processing and DOSY use the separate shared scientific operations. -CSV/TSV `mode: "table_data_only"` retains its existing data-only behavior and limits. +CSV, TSV and XLSX files are tables: import them with `mode: "table_data_only"`. +Native import rejects them with that hint. ## Discover structure without relying on names @@ -475,9 +476,9 @@ Use `plotium_properties_inspect` for target-specific values, defaults, bounds an applicability; an axis accepts only codes compatible with its coordinate unit. Static applicability does not grant connection write permission. -## Import CSV/TSV data without creating a plot +## Import CSV/TSV/XLSX data without creating a plot -Ask the agent to import an explicitly named UTF-8 CSV or TSV, inspect its columns, +Ask the agent to import an explicitly named UTF-8 CSV or TSV, or an XLSX workbook, inspect its columns, and read the whole table in pages. Start with `plotium_get_state {}`, then call `plotium_plan`: @@ -497,13 +498,25 @@ and read the whole table in pages. Start with `plotium_get_state {}`, then call Do not supply `ids` or `targets` for import. `header` accepts `auto` (default), `present` or `absent`; ambiguous auto detection fails. The `.csv` or `.tsv` extension selects comma or tab; `format: "csv" | "tsv"` overrides it explicitly. +An `.xlsx` file (or `format: "xlsx"`) imports each visible worksheet that has +cells as its own table, named after the worksheet, or after the file when the +workbook holds a single table, the same way the desktop imports it. +`sheets: ["Figure 3", …]` imports only the named worksheets; an unknown name +fails that file and lists the visible worksheets. Each worksheet is prepared on +its own: one that holds no table or exceeds a limit fails as its own item and +the others still import. One file reads at most 8,000,000 worksheet cells, +counted over each worksheet's used range. `header`, `schema` and `schema_path` +apply to CSV/TSV only. If the summary of a batch with XLSX files would exceed +128 KiB, the preflight fails and lists each workbook's worksheets, so you can +import one workbook at a time and name fewer worksheets. Only named regular files are read. There is no directory discovery, adjacent sidecar lookup, source writeback, implicit plotting or X/Y binding. Preflight runs in the background and returns only when preparation finishes. -`import.items` reports each file as `prepared` or `failed`, with proposed native -IDs, metadata and diagnostics. No dataset exists yet. After reviewing the result, -call `plotium_execute {"plan_id":"RETURNED_PLAN_ID"}`. Execution uses those frozen +`import.items` reports each file, or each XLSX worksheet with its `sheet`, as +`prepared` or `failed`, with proposed native IDs, metadata and diagnostics. No +dataset exists yet. After reviewing the result, call +`plotium_execute {"plan_id":"RETURNED_PLAN_ID"}`. Execution uses those frozen bytes and typed snapshots, even if a source file changed after preflight. It rechecks revision and permission before one atomic commit of all prepared items. `value.status` is `succeeded`, `partial` or `failed`; inspect every item and its diff --git a/docs/src/content/docs/zh-cn/reference/cli.md b/docs/src/content/docs/zh-cn/reference/cli.md index 22967535..cdaf7cc8 100644 --- a/docs/src/content/docs/zh-cn/reference/cli.md +++ b/docs/src/content/docs/zh-cn/reference/cli.md @@ -218,7 +218,7 @@ plotium-cli tool run data.jdf --request operation.json --output result.plotium 失败终态返回非零退出码；可选 `--output` 保存原生项目。`process` 同样通过 共享处理方案服务执行后导出绘图。 -不给输入时，`tool run` 从空项目开始。CSV 和 TSV 表格不能作为输入加载，要用 +不给输入时，`tool run` 从空项目开始。CSV、TSV 和 XLSX 表格不能作为输入加载，要用 `data.import` 导入，所以可以这样从表格起步（导入路径必须是绝对路径）： ```sh diff --git a/docs/src/content/docs/zh-cn/reference/mcp.md b/docs/src/content/docs/zh-cn/reference/mcp.md index 64f11ec4..108c6a73 100644 --- a/docs/src/content/docs/zh-cn/reference/mcp.md +++ b/docs/src/content/docs/zh-cn/reference/mcp.md @@ -59,7 +59,7 @@ dirty、edit generation 或 automation revision。纯查询工作流同样不改 已认证状态查询、每五秒心跳、静态属性目录以及保存协调（预检、提交与 `plotium_save_status`），以及导出预检／提交／状态查询均不生成审计记录。 -连接支持原生采集导入、CSV/TSV 仅数据导入、原生表格查询、画布预览、图面几何诊断、文档结构、原生建图及页面／面板／文字／本地图片编辑、显式数值曲线绑定、受支持的表格处理、单元格补丁、已存配方显式刷新和 +连接支持原生采集导入、CSV/TSV/XLSX 仅数据导入、原生表格查询、画布预览、图面几何诊断、文档结构、原生建图及页面／面板／文字／本地图片编辑、显式数值曲线绑定、受支持的表格处理、单元格补丁、已存配方显式刷新和 资源查询、数据集／画布／画布对象重命名、为画布套用内置主题或已保存的画布风格，以及以下显示属性： | 属性 ID | 目标 | 取值 | @@ -311,7 +311,7 @@ stale/context 错误或结果不确定时，仍须读取状态。执行继续检 所有保留计划及在途预检的原生冻结负载保守预留合计最多 512 MiB；超限返回 `import_budget_exceeded`。超过 16 个路径时调用失败，报错写明收到的数量；其余文件请分批 导入。预检响应最多 256 KiB。科学处理与 DOSY 使用独立的共享科学操作。 -CSV/TSV 的 `mode: "table_data_only"` 保持既有仅创建数据的行为和限制。 +CSV、TSV 和 XLSX 文件是表格，用 `mode: "table_data_only"` 导入；原生导入收到它们会报错并给出这一提示。 ## 不依赖名称发现结构 @@ -391,9 +391,9 @@ Series 详情包含可直接用于属性查询的完整 target、编码、可见 颜色格式 `#rrggbb` 和全部已注册单位代码。用 `plotium_properties_inspect` 查询目标值、 默认值、范围和适用性；坐标轴只接受与其坐标单位兼容的代码。静态适用能力不授予连接修改权限。 -## 导入 CSV/TSV 数据，不创建图 +## 导入 CSV/TSV/XLSX 数据，不创建图 -让 agent 导入明确指定的 UTF-8 CSV/TSV，检查列信息，再分页读完整张表。先调用 +让 agent 导入明确指定的 UTF-8 CSV/TSV 或 XLSX 工作簿，检查列信息，再分页读完整张表。先调用 `plotium_get_state {}`，然后调用 `plotium_plan`： ```json @@ -411,10 +411,16 @@ Series 详情包含可直接用于属性查询的完整 target、编码、可见 导入不提供 `ids` 或 `targets`。`header` 可为默认的 `auto`、`present` 或 `absent`； 自动判断有歧义则失败。`.csv` 与 `.tsv` 分别选择逗号和制表符；可显式用 -`format: "csv" | "tsv"` 指定。只读所列普通文件，不扫描目录、不自动寻找邻接 +`format: "csv" | "tsv"` 指定。`.xlsx` 文件（或 `format: "xlsx"`）把每个有内容的可见 +工作表各导入成一张表，表名取工作表名，工作簿只有一张表时取文件名，与桌面端导入一致。 +`sheets: ["Figure 3", …]` 只导入列出的工作表；名字不存在时该文件失败，并列出可见工作表。 +每个工作表单独准备：没有表格或超出限制的工作表作为单独一项失败，其余照常导入。一个文件 +最多读取 8,000,000 个工作表单元格，按各工作表的已用范围计算。`header`、`schema` 和 +`schema_path` 只用于 CSV/TSV。含 XLSX 的批次摘要超过 128 KiB 时预检失败，并按文件列出 +工作表，可每次导入一个工作簿并减少 `sheets` 中的工作表。只读所列普通文件，不扫描目录、不自动寻找邻接 sidecar、不回写源文件、不创建图或 X/Y 绑定。 -预检在后台完成准备后才返回。`import.items` 逐文件报告 `prepared` 或 `failed`， +预检在后台完成准备后才返回。`import.items` 逐文件（XLSX 逐工作表，带 `sheet`）报告 `prepared` 或 `failed`， 包含拟建原生 ID、元数据和诊断，此时项目尚未新增数据集。确认结果后调用 `plotium_execute {"plan_id":"RETURNED_PLAN_ID"}`。执行使用预检冻结的字节和 typed snapshot，不重新读取可能已变化的文件；提交前重验 revision 和权限，将全部已准备 From 3aae77a0698e1ae8d90c5e0f898b3ef44843132b Mon Sep 17 00:00:00 2001 From: Dongcheng Lin Date: Sun, 25 Oct 2026 11:16:54 +0900 Subject: [PATCH 2/2] test(mcp): use a platform-absolute path in the native import hint test --- crates/core/src/automation/table_import_xlsx_tests.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/core/src/automation/table_import_xlsx_tests.rs b/crates/core/src/automation/table_import_xlsx_tests.rs index 307363c2..484182e3 100644 --- a/crates/core/src/automation/table_import_xlsx_tests.rs +++ b/crates/core/src/automation/table_import_xlsx_tests.rs @@ -154,8 +154,10 @@ fn xlsx_rejects_csv_only_options_and_csv_rejects_sheets() { #[test] fn native_import_points_tables_at_table_data_only() { - let params: NativeImportParams = - serde_json::from_value(json!({"paths":["/data/Source Data.xlsx"]})).unwrap(); + // A path that is absolute on every platform, so validation reaches the + // extension check. + let path = std::env::temp_dir().join("Source Data.xlsx"); + let params: NativeImportParams = serde_json::from_value(json!({"paths":[path]})).unwrap(); let message = params.validate().unwrap_err().to_string(); assert!(message.contains("mode table_data_only"), "{message}"); }