diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index d2a2c359..803371b0 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -35,6 +35,13 @@ jobs: with: save-if: ${{ github.ref_name == 'main' }} + # The bun runtime tests run bun from PATH. They're unix-only, like the runtime's bun support. + - name: Set up bun + if: runner.os == 'Linux' + uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.4 + - name: Compile tests run: > cargo test --all-features --no-run diff --git a/Cargo.lock b/Cargo.lock index feb86471..12d5a13c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -841,7 +841,7 @@ dependencies = [ [[package]] name = "config" -version = "0.3.75" +version = "0.3.76" dependencies = [ "base64", "chrono", @@ -1033,7 +1033,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto" -version = "0.3.75" +version = "0.3.76" dependencies = [ "aes-gcm", "base64", @@ -4456,7 +4456,7 @@ dependencies = [ [[package]] name = "testutils" -version = "0.3.75" +version = "0.3.76" dependencies = [ "pem", "rsa", @@ -4765,7 +4765,7 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "tower" -version = "0.3.75" +version = "0.3.76" dependencies = [ "config", "pyo3", @@ -4793,7 +4793,7 @@ dependencies = [ [[package]] name = "tower-api" -version = "0.3.75" +version = "0.3.76" dependencies = [ "reqwest", "serde", @@ -4805,7 +4805,7 @@ dependencies = [ [[package]] name = "tower-cmd" -version = "0.3.75" +version = "0.3.76" dependencies = [ "axum", "bytes", @@ -4854,7 +4854,7 @@ dependencies = [ [[package]] name = "tower-duckdb" -version = "0.3.75" +version = "0.3.76" dependencies = [ "chrono", "duckdb", @@ -4889,7 +4889,7 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-package" -version = "0.3.75" +version = "0.3.76" dependencies = [ "async-compression", "flate2", @@ -4914,7 +4914,7 @@ dependencies = [ [[package]] name = "tower-runtime" -version = "0.3.75" +version = "0.3.76" dependencies = [ "async-trait", "chrono", @@ -4938,7 +4938,7 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-telemetry" -version = "0.3.75" +version = "0.3.76" dependencies = [ "tracing", "tracing-appender", @@ -4947,7 +4947,7 @@ dependencies = [ [[package]] name = "tower-uv" -version = "0.3.75" +version = "0.3.76" dependencies = [ "async-compression", "async_zip", @@ -4967,7 +4967,7 @@ dependencies = [ [[package]] name = "tower-version" -version = "0.3.75" +version = "0.3.76" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 07df3a0c..74e98363 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.3.75" +version = "0.3.76" description = "Tower is the best way to host Python data apps in production" # Matches rust-toolchain.toml. The two had drifted: the toolchain has been 1.88 # for a while, and the dependency tree (testcontainers and its transitive deps, diff --git a/crates/tower-cmd/src/catalogs.rs b/crates/tower-cmd/src/catalogs.rs index c53b30c9..d3427aa3 100644 --- a/crates/tower-cmd/src/catalogs.rs +++ b/crates/tower-cmd/src/catalogs.rs @@ -591,18 +591,19 @@ async fn execute_catalog_query( // legitimate analytical scan over a large catalog can take minutes and a // person is driving; the agent path uses `Limits::agent()`, which adds one. let harden = !write; - let result = tokio::task::spawn_blocking(move || -> Result { - let session = Session::open()?; - session.run_setup(&setup)?; - if harden { - // `Hardening::agent()` adds an engine memory ceiling on top of the - // lockdown. The result `limits` bound what comes back; only the engine - // can bound what a query spends producing it. - session.harden(&Hardening::agent())?; - } - session.query(&sql, [], &limits) - }) - .await; + let result = + tokio::task::spawn_blocking(move || -> Result { + let session = Session::open()?; + session.run_setup(&setup)?; + if harden { + // `Hardening::agent()` adds an engine memory ceiling on top of the + // lockdown. The result `limits` bound what comes back; only the engine + // can bound what a query spends producing it. + session.harden(&Hardening::agent())?; + } + session.query(&sql, [], &limits) + }) + .await; match result { Ok(Ok(query_result)) => { @@ -687,13 +688,14 @@ pub(crate) async fn query_catalog_for_agent( &response.credentials, vend_catalog_credentials_body::Mode::Read, ); - let result = tokio::task::spawn_blocking(move || -> Result { - let session = Session::open()?; - session.run_setup(&setup)?; - session.harden(&Hardening::agent())?; - session.query(&sql, [], &Limits::agent()) - }) - .await; + let result = + tokio::task::spawn_blocking(move || -> Result { + let session = Session::open()?; + session.run_setup(&setup)?; + session.harden(&Hardening::agent())?; + session.query(&sql, [], &Limits::agent()) + }) + .await; match result { Ok(Ok(query_result)) => Ok(query_result), @@ -844,7 +846,10 @@ async fn fetch_catalog_columns_via_rest( rows.sort_by(|a, b| { let key = |row: &[serde_json::Value]| { ( - row.first().and_then(|v| v.as_str()).unwrap_or("").to_owned(), + row.first() + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_owned(), row.get(1).and_then(|v| v.as_str()).unwrap_or("").to_owned(), ) }; @@ -1088,7 +1093,10 @@ fn iceberg_type_to_display(ty: &serde_json::Value) -> String { format!("LIST({})", element) } Some("map") => { - let key = obj.get("key").map(iceberg_type_to_display).unwrap_or_default(); + let key = obj + .get("key") + .map(iceberg_type_to_display) + .unwrap_or_default(); let value = obj .get("value") .map(iceberg_type_to_display) @@ -1592,9 +1600,7 @@ pub async fn do_knowledge_show(out: &output::Out, config: Config, args: &ArgMatc let catalog = args .get_one::("catalog_name") .expect("catalog_name is required"); - let name = args - .get_one::("name") - .expect("name is required"); + let name = args.get_one::("name").expect("name is required"); let env = cmd::get_string_flag(args, "environment"); let response = out @@ -1614,9 +1620,7 @@ pub async fn do_knowledge_set(out: &output::Out, config: Config, args: &ArgMatch let catalog = args .get_one::("catalog_name") .expect("catalog_name is required"); - let name = args - .get_one::("name") - .expect("name is required"); + let name = args.get_one::("name").expect("name is required"); let env = cmd::get_string_flag(args, "environment"); let statement = cmd::get_string_flag(args, "statement"); let scope = cmd::get_string_flag(args, "scope"); @@ -1662,9 +1666,7 @@ pub async fn do_knowledge_delete(out: &output::Out, config: Config, args: &ArgMa let catalog = args .get_one::("catalog_name") .expect("catalog_name is required"); - let name = args - .get_one::("name") - .expect("name is required"); + let name = args.get_one::("name").expect("name is required"); let env = cmd::get_string_flag(args, "environment"); out.with_spinner( @@ -2129,7 +2131,10 @@ mod tests { ]) .expect("query --max-rows should parse"); let (_, query_args) = matches.subcommand().expect("expected query subcommand"); - assert_eq!(query_args.get_one::("max_rows").copied(), Some(50_000)); + assert_eq!( + query_args.get_one::("max_rows").copied(), + Some(50_000) + ); assert!( catalogs_cmd() @@ -2298,15 +2303,14 @@ mod tests { .expect("knowledge delete should parse"); let (_, knowledge_args) = matches.subcommand().expect("expected knowledge subcommand"); - let (_, delete_args) = knowledge_args.subcommand().expect("expected delete subcommand"); + let (_, delete_args) = knowledge_args + .subcommand() + .expect("expected delete subcommand"); assert_eq!( delete_args.get_one::("catalog_name").unwrap(), "my-catalog" ); - assert_eq!( - delete_args.get_one::("name").unwrap(), - "my-entry" - ); + assert_eq!(delete_args.get_one::("name").unwrap(), "my-entry"); } #[test] @@ -2365,7 +2369,9 @@ mod tests { .expect("knowledge list should parse"); let (_, knowledge_args) = matches.subcommand().expect("expected knowledge subcommand"); - let (_, list_args) = knowledge_args.subcommand().expect("expected list subcommand"); + let (_, list_args) = knowledge_args + .subcommand() + .expect("expected list subcommand"); assert_eq!( list_args.get_one::("catalog_name").unwrap(), @@ -2453,7 +2459,9 @@ mod tests { .expect("knowledge set with all flags should parse"); let (_, knowledge_args) = matches.subcommand().expect("expected knowledge subcommand"); - let (_, set_args) = knowledge_args.subcommand().expect("expected set subcommand"); + let (_, set_args) = knowledge_args + .subcommand() + .expect("expected set subcommand"); assert_eq!(set_args.get_one::("scope").unwrap(), "column"); assert_eq!( @@ -2486,7 +2494,9 @@ mod tests { .expect("knowledge set should parse"); let (_, knowledge_args) = matches.subcommand().expect("expected knowledge subcommand"); - let (_, set_args) = knowledge_args.subcommand().expect("expected set subcommand"); + let (_, set_args) = knowledge_args + .subcommand() + .expect("expected set subcommand"); assert_eq!(set_args.get_one::("scope").unwrap(), "catalog"); assert_eq!( @@ -2500,8 +2510,13 @@ mod tests { #[test] fn knowledge_set_requires_statement() { - let result = - catalogs_cmd().try_get_matches_from(["catalogs", "knowledge", "set", "my-catalog", "f"]); + let result = catalogs_cmd().try_get_matches_from([ + "catalogs", + "knowledge", + "set", + "my-catalog", + "f", + ]); assert!(result.is_err()); } @@ -2515,7 +2530,9 @@ mod tests { .try_get_matches_from(["catalogs", "knowledge", "show", "my-catalog", "my-entry"]) .expect("knowledge show should parse"); let (_, knowledge_args) = matches.subcommand().expect("expected knowledge subcommand"); - let (_, show_args) = knowledge_args.subcommand().expect("expected show subcommand"); + let (_, show_args) = knowledge_args + .subcommand() + .expect("expected show subcommand"); assert_eq!(show_args.get_one::("name").unwrap(), "my-entry"); assert_eq!( show_args.get_one::("environment").unwrap(), @@ -2531,5 +2548,4 @@ mod tests { assert_eq!(truncated.chars().count(), 80); assert!(truncated.ends_with('…')); } - } diff --git a/crates/tower-cmd/src/error.rs b/crates/tower-cmd/src/error.rs index 55461df5..7740f278 100644 --- a/crates/tower-cmd/src/error.rs +++ b/crates/tower-cmd/src/error.rs @@ -63,6 +63,15 @@ pub enum Error { #[snafu(display("No [project] section found in pyproject.toml"))] MissingProjectSection, + #[snafu(display("package.json not found at {}", path))] + PackageJsonNotFound { path: String }, + + #[snafu(display("Failed to parse package.json: {}", source))] + InvalidPackageJson { source: serde_json::Error }, + + #[snafu(display("No pyproject.toml or package.json found in {}", dir))] + ProjectFileNotFound { dir: String }, + // File I/O errors #[snafu(display("File I/O error: {}", source))] IoError { source: std::io::Error }, diff --git a/crates/tower-cmd/src/lib.rs b/crates/tower-cmd/src/lib.rs index 0bb92756..d1af559d 100644 --- a/crates/tower-cmd/src/lib.rs +++ b/crates/tower-cmd/src/lib.rs @@ -154,24 +154,26 @@ impl App { Some(("query", args)) => { catalogs::do_query(&out, sessionized_config, args).await } - Some(("knowledge", knowledge_matches)) => match knowledge_matches.subcommand() { - Some(("list", args)) => { - catalogs::do_knowledge_list(&out, sessionized_config, args).await + Some(("knowledge", knowledge_matches)) => { + match knowledge_matches.subcommand() { + Some(("list", args)) => { + catalogs::do_knowledge_list(&out, sessionized_config, args).await + } + Some(("show", args)) => { + catalogs::do_knowledge_show(&out, sessionized_config, args).await + } + Some(("set", args)) => { + catalogs::do_knowledge_set(&out, sessionized_config, args).await + } + Some(("delete", args)) => { + catalogs::do_knowledge_delete(&out, sessionized_config, args).await + } + _ => { + catalogs::catalogs_cmd().print_help().unwrap(); + std::process::exit(2); + } } - Some(("show", args)) => { - catalogs::do_knowledge_show(&out, sessionized_config, args).await - } - Some(("set", args)) => { - catalogs::do_knowledge_set(&out, sessionized_config, args).await - } - Some(("delete", args)) => { - catalogs::do_knowledge_delete(&out, sessionized_config, args).await - } - _ => { - catalogs::catalogs_cmd().print_help().unwrap(); - std::process::exit(2); - } - }, + } _ => { catalogs::catalogs_cmd().print_help().unwrap(); std::process::exit(2); diff --git a/crates/tower-cmd/src/mcp.rs b/crates/tower-cmd/src/mcp.rs index 7ccbce56..18dbbb5e 100644 --- a/crates/tower-cmd/src/mcp.rs +++ b/crates/tower-cmd/src/mcp.rs @@ -80,6 +80,11 @@ struct UpdateTowerfileRequest { script: Option, description: Option, source: Option>, + /// Compute size for each run: one of basic.xsmall, basic.small, + /// basic.medium, basic.large. Omit to let Tower apply its default. + run_size: Option, + /// What starts the script: python or bun. Omit for Python. + runtime: Option, } #[derive(Debug, Deserialize, JsonSchema)] @@ -1020,7 +1025,7 @@ impl TowerService { } #[tool( - description = "Update Towerfile config (app name, script, description, source). Use this instead of editing TOML. Optional: working_directory." + description = "Update Towerfile config (app name, script, description, source, run_size, runtime). Use this instead of editing TOML. Optional: working_directory." )] async fn tower_file_update( &self, @@ -1039,6 +1044,12 @@ impl TowerService { if let Some(source) = request.source { tf.app.source = source; } + if let Some(run_size) = request.run_size { + tf.app.run_size = Some(run_size); + } + if let Some(runtime) = request.runtime { + tf.app.runtime = Some(runtime); + } Ok("Towerfile updated".into()) }) } @@ -1133,21 +1144,18 @@ impl TowerService { } #[tool( - description = "Generate Towerfile from pyproject.toml (first step). Don't add build systems to pyproject.toml. Optional: working_directory." + description = "Generate Towerfile from pyproject.toml, or from package.json for a bun app (first step). Don't add build systems to pyproject.toml. Optional: working_directory." )] async fn tower_file_generate( &self, Parameters(request): Parameters, ) -> Result { let working_dir = Self::resolve_working_directory(&request.common); - let pyproject_path = working_dir.join("pyproject.toml"); - let content = match TowerfileGenerator::from_pyproject( - Some(pyproject_path.to_str().unwrap()), - request.script_path.as_deref(), - ) { - Ok(content) => content, - Err(e) => return Self::error_result("Failed to generate Towerfile", e), - }; + let content = + match TowerfileGenerator::from_dir(&working_dir, request.script_path.as_deref()) { + Ok(content) => content, + Err(e) => return Self::error_result("Failed to generate Towerfile", e), + }; let towerfile_path = working_dir.join("Towerfile"); match std::fs::write(&towerfile_path, &content) { @@ -1185,6 +1193,11 @@ All tools accept optional working_directory parameter to specify which project t DO NOT add [build-system], [tool.hatchling], [tool.setuptools] or similar Skip this step if project with pyproject.toml already exists + JAVASCRIPT OR TYPESCRIPT PROJECT (if creating new): + Use 'bun init -y' in project directory to create package.json and index.ts + Skip this step if project with package.json already exists + tower_file_generate then writes a Towerfile with runtime = "bun" + 1. TOWERFILE (required for all Tower operations): tower_file_generate → tower_file_update → tower_file_add/edit/remove_parameter → tower_file_validate CRITICAL: Always use tower_file_update or tower_file_add/edit/remove_parameter to modify @@ -1212,8 +1225,8 @@ All tools accept optional working_directory parameter to specify which project t tower_secrets_create, tower_secrets_list - store credentials, API keys, database passwords TYPICAL WORKFLOW: -1. Check if pyproject.toml exists (or create with uv init) -2. tower_file_generate - creates Towerfile from pyproject.toml +1. Check if pyproject.toml or package.json exists (or create with uv init / bun init -y) +2. tower_file_generate - creates Towerfile from pyproject.toml or package.json 3. tower_run_local - test locally (can access tower secrets) 4. tower_apps_create - create app in Tower cloud 5. tower_deploy - deploy code to Tower @@ -1325,7 +1338,7 @@ Rules: - MCP tools are the authoritative Tower interface (not wrappers) - Use tower_file_update/add/edit/remove_parameter to modify Towerfiles (never edit TOML directly) - DO NOT add hatchling/setuptools to pyproject.toml - Tower handles deployment -- Tower apps need: pyproject.toml (deps only), Python code, Towerfile +- Tower apps need: pyproject.toml (deps only), Python code, Towerfile; or, for bun apps, package.json, JavaScript or TypeScript code, Towerfile - Pass run parameters as a JSON object {\"key\": \"value\"}, not as CLI flags - There is no --param or --params flag diff --git a/crates/tower-cmd/src/skill.rs b/crates/tower-cmd/src/skill.rs index c575293c..c8e88d31 100644 --- a/crates/tower-cmd/src/skill.rs +++ b/crates/tower-cmd/src/skill.rs @@ -182,6 +182,8 @@ uv init Creates `pyproject.toml`, `main.py`, `README.md`. Keep `pyproject.toml` minimal — `[project]` metadata and dependencies only. No `[build-system]`, `[tool.hatchling]`, or similar. Skip if a `pyproject.toml` already exists. +For a JavaScript or TypeScript app, run `bun init -y` instead, which creates `package.json` and `index.ts`. Skip it if a `package.json` already exists. Without a `pyproject.toml`, `tower_file_generate` reads `package.json` and sets `runtime = "bun"`. + ### 1. Towerfile ``` @@ -190,6 +192,10 @@ tower_file_generate → tower_file_update → tower_file_add/edit/remove_paramet Always use `tower_file_update` or `tower_file_add/edit/remove_parameter` to modify. Never edit the TOML directly. +To give an app more compute, set `run_size` via `tower_file_update`: one of `basic.xsmall`, `basic.small`, `basic.medium` (the default), `basic.large`. It is versioned with the code, and self-hosted runners ignore it. + +To run JavaScript or TypeScript, set `runtime` to `bun` via `tower_file_update` and point `script` at a `.ts`, `.tsx`, `.js`, `.jsx`, `.mjs` or `.cjs` file. Dependencies come from `package.json` (and `bun.lock`). Leave `runtime` unset for Python apps. + ### 2. Local development (preferred) ``` diff --git a/crates/tower-cmd/src/towerfile_gen.rs b/crates/tower-cmd/src/towerfile_gen.rs index 58525a45..58c972b4 100644 --- a/crates/tower-cmd/src/towerfile_gen.rs +++ b/crates/tower-cmd/src/towerfile_gen.rs @@ -1,10 +1,129 @@ use crate::Error; use std::fs; use std::path::Path; +use tower_package::is_bun_script; pub struct TowerfileGenerator; impl TowerfileGenerator { + /// Generates a Towerfile from the project file in `dir`: pyproject.toml when there is one, + /// otherwise package.json for a bun app. + pub fn from_dir(dir: &Path, script_path: Option<&str>) -> Result { + let pyproject_path = dir.join("pyproject.toml"); + if pyproject_path.exists() { + return Self::from_pyproject(pyproject_path.to_str(), script_path); + } + + let package_json_path = dir.join("package.json"); + if package_json_path.exists() { + return Self::from_package_json(package_json_path.to_str(), script_path); + } + + Err(Error::ProjectFileNotFound { + dir: dir.display().to_string(), + }) + } + + pub fn from_package_json( + package_json_path: Option<&str>, + script_path: Option<&str>, + ) -> Result { + let package_json_path = package_json_path.unwrap_or("package.json"); + let package_dir = Path::new(package_json_path) + .parent() + .unwrap_or(Path::new(".")); + + if !Path::new(package_json_path).exists() { + return Err(Error::PackageJsonNotFound { + path: package_json_path.to_string(), + }); + } + + let content = fs::read_to_string(package_json_path)?; + let package: serde_json::Value = serde_json::from_str(&content) + .map_err(|source| Error::InvalidPackageJson { source })?; + + // An npm scope (`@acme/report`) isn't part of the app's name. + let app_name = package + .get("name") + .and_then(|n| n.as_str()) + .map(|n| { + n.strip_prefix('@') + .and_then(|scoped| scoped.split_once('/')) + .map_or(n, |(_, name)| name) + }) + .filter(|n| !n.is_empty()) + .unwrap_or("my-app"); + + let description = package + .get("description") + .and_then(|d| d.as_str()) + .unwrap_or(""); + + let script = script_path + .map(String::from) + .or_else(|| Self::find_bun_script(package_dir, &package)) + .unwrap_or_else(|| "./index.ts".to_string()); + + let towerfile = toml::toml! { + [app] + name = app_name + script = script + source = [] + description = description + runtime = "bun" + }; + + Ok(toml::to_string(&towerfile)?) + } + + /// The first runnable file named by package.json (`module`, `main`, then the `start` + /// script), then a conventional entry point, then any JS/TS file in `dir`. + fn find_bun_script(dir: &Path, package: &serde_json::Value) -> Option { + let start_script = package + .get("scripts") + .and_then(|scripts| scripts.get("start")) + .and_then(|start| start.as_str()) + .unwrap_or(""); + + let declared = ["module", "main"] + .into_iter() + .filter_map(|field| package.get(field).and_then(|v| v.as_str())) + .chain(start_script.split_whitespace()); + + let conventional = [ + "index.ts", + "main.ts", + "src/index.ts", + "src/main.ts", + "index.js", + "main.js", + ]; + + declared + .chain(conventional) + .map(|candidate| candidate.trim_start_matches("./")) + // `main` often names a build output that doesn't exist yet. + .find(|candidate| is_bun_script(candidate) && dir.join(candidate).is_file()) + .map(|candidate| format!("./{}", candidate)) + .or_else(|| Self::find_any_bun_file(dir)) + } + + fn find_any_bun_file(dir: &Path) -> Option { + let mut files: Vec = fs::read_dir(dir) + .ok()? + .flatten() + .filter(|entry| entry.path().is_file()) + .filter_map(|entry| entry.file_name().to_str().map(String::from)) + // Type declarations and tool configs are never the app. + .filter(|name| { + is_bun_script(name) && !name.ends_with(".d.ts") && !name.contains(".config.") + }) + .collect(); + files.sort(); + files.into_iter().next().map(|name| format!("./{}", name)) + } + pub fn from_pyproject( pyproject_path: Option<&str>, script_path: Option<&str>, @@ -303,4 +422,181 @@ if __name__ == "__main__": let result = TowerfileGenerator::find_main_script(temp_dir.path()); assert_eq!(result, Some("./main.py".to_string())); } + + /// Generates from `package_json` in a fresh dir holding `files`, and parses the result back + /// so it must also pass Towerfile validation. + fn generate_from_package_json( + package_json: &str, + files: &[&str], + script_path: Option<&str>, + ) -> tower_package::Towerfile { + let temp_dir = create_test_env(); + let package_json_path = temp_dir.path().join("package.json"); + fs::write(&package_json_path, package_json).unwrap(); + for file in files { + let path = temp_dir.path().join(file); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "console.log('hello');").unwrap(); + } + + let content = TowerfileGenerator::from_package_json( + Some(package_json_path.to_str().unwrap()), + script_path, + ) + .unwrap(); + tower_package::Towerfile::from_toml(&content).unwrap() + } + + #[test] + fn test_from_package_json_bun_init() { + // What `bun init -y` writes. + let towerfile = generate_from_package_json( + r#"{ + "name": "my-report", + "module": "index.ts", + "type": "module", + "private": true, + "scripts": { + "typecheck": "bun check" + }, + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^7" + } +}"#, + &["index.ts"], + None, + ); + + assert_eq!(towerfile.app.name, "my-report"); + assert_eq!(towerfile.app.script, "./index.ts"); + assert_eq!(towerfile.app.runtime.as_deref(), Some("bun")); + } + + #[test] + fn test_from_package_json_drops_the_npm_scope() { + let towerfile = generate_from_package_json( + r#"{"name": "@acme/report", "description": "Weekly report"}"#, + &["index.ts"], + None, + ); + + assert_eq!(towerfile.app.name, "report"); + assert_eq!(towerfile.app.description.as_deref(), Some("Weekly report")); + } + + #[test] + fn test_from_package_json_skips_a_main_that_does_not_exist() { + let towerfile = generate_from_package_json( + r#"{"name": "app", "main": "dist/index.js"}"#, + &["src/index.ts"], + None, + ); + + assert_eq!(towerfile.app.script, "./src/index.ts"); + } + + #[test] + fn test_from_package_json_reads_the_start_script() { + let towerfile = generate_from_package_json( + r#"{"name": "app", "scripts": {"start": "bun run --watch ./src/job.ts"}}"#, + &["src/job.ts", "index.ts"], + None, + ); + + assert_eq!(towerfile.app.script, "./src/job.ts"); + } + + #[test] + fn test_from_package_json_falls_back_to_any_script_file() { + let towerfile = generate_from_package_json( + r#"{"name": "app"}"#, + &["env.d.ts", "vite.config.ts", "job.ts"], + None, + ); + + assert_eq!(towerfile.app.script, "./job.ts"); + } + + #[test] + fn test_from_package_json_with_custom_script() { + let towerfile = generate_from_package_json( + r#"{"name": "app", "module": "index.ts"}"#, + &["index.ts"], + Some("./run.ts"), + ); + + assert_eq!(towerfile.app.script, "./run.ts"); + } + + #[test] + fn test_from_package_json_errors() { + let missing = + TowerfileGenerator::from_package_json(Some("/nonexistent/path/package.json"), None); + assert!(missing + .unwrap_err() + .to_string() + .contains("package.json not found")); + + let temp_dir = create_test_env(); + let package_json_path = temp_dir.path().join("package.json"); + fs::write(&package_json_path, "{ not json").unwrap(); + let invalid = + TowerfileGenerator::from_package_json(Some(package_json_path.to_str().unwrap()), None); + assert!(invalid + .unwrap_err() + .to_string() + .contains("Failed to parse package.json")); + } + + #[test] + fn test_from_dir_prefers_pyproject() { + let temp_dir = create_test_env(); + fs::write( + temp_dir.path().join("pyproject.toml"), + "[project]\nname = \"python-app\"\n", + ) + .unwrap(); + fs::write(temp_dir.path().join("main.py"), "print('hello')").unwrap(); + fs::write( + temp_dir.path().join("package.json"), + r#"{"name": "js-tools"}"#, + ) + .unwrap(); + + let content = TowerfileGenerator::from_dir(temp_dir.path(), None).unwrap(); + let towerfile = tower_package::Towerfile::from_toml(&content).unwrap(); + + assert_eq!(towerfile.app.name, "python-app"); + assert_eq!(towerfile.app.runtime, None); + } + + #[test] + fn test_from_dir_uses_package_json_without_pyproject() { + let temp_dir = create_test_env(); + fs::write( + temp_dir.path().join("package.json"), + r#"{"name": "js-app", "module": "index.ts"}"#, + ) + .unwrap(); + fs::write(temp_dir.path().join("index.ts"), "console.log('hello');").unwrap(); + + let content = TowerfileGenerator::from_dir(temp_dir.path(), None).unwrap(); + let towerfile = tower_package::Towerfile::from_toml(&content).unwrap(); + + assert_eq!(towerfile.app.name, "js-app"); + assert_eq!(towerfile.app.runtime.as_deref(), Some("bun")); + } + + #[test] + fn test_from_dir_without_a_project_file() { + let temp_dir = create_test_env(); + + let err = TowerfileGenerator::from_dir(temp_dir.path(), None).unwrap_err(); + assert!(err + .to_string() + .contains("No pyproject.toml or package.json found")); + } } diff --git a/crates/tower-duckdb/src/lib.rs b/crates/tower-duckdb/src/lib.rs index 64754fc7..a96242dc 100644 --- a/crates/tower-duckdb/src/lib.rs +++ b/crates/tower-duckdb/src/lib.rs @@ -1188,12 +1188,15 @@ mod tests { false } - /// MinIO pinned to an immutable release tag rather than `latest`. + /// Coollabs builds official MinIO sources; upstream's prebuilt images are unavailable. + /// https://github.com/coollabsio/minio /// - /// A floating tag lets a new upstream image change this test's behaviour, or - /// break it into a skip, without anyone choosing that. A security regression - /// test should only change when someone means it. Bump deliberately and re-run. - const MINIO_IMAGE_TAG: &str = "RELEASE.2025-09-07T16-13-09Z"; + /// Pin the multi-platform image digest so a rebuilt tag cannot change this security test. + /// Bump deliberately and re-run the object-store test on Linux. + const MINIO_IMAGE_TAG: &str = concat!( + "RELEASE.2025-10-15T17-29-55Z@", + "sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9" + ); /// Whether a container runtime that can run this image is reachable. /// @@ -1241,7 +1244,7 @@ mod tests { } let bucket = "warehouse"; - let image = GenericImage::new("quay.io/minio/minio", MINIO_IMAGE_TAG) + let image = GenericImage::new("ghcr.io/coollabsio/minio", MINIO_IMAGE_TAG) .with_wait_for(WaitFor::seconds(1)) .with_exposed_port(9000.tcp()) .with_entrypoint("sh") diff --git a/crates/tower-package/src/core.rs b/crates/tower-package/src/core.rs index 69383c4f..143f1a96 100644 --- a/crates/tower-package/src/core.rs +++ b/crates/tower-package/src/core.rs @@ -74,6 +74,15 @@ pub struct Manifest { pub schedule: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub run_size: Option, + + /// The runtime the Towerfile declared, e.g. "bun". `None` means it didn't declare one and + /// the runtime follows the script; see `resolve_runtime`. A string rather than `Runtime`, + /// so a manifest naming a runtime this version doesn't know still parses. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime: Option, + #[serde(default)] pub import_paths: Vec, @@ -91,6 +100,46 @@ impl Manifest { pub fn from_json(data: &str) -> Result { Ok(serde_json::from_str(data)?) } + + /// The runtime that starts this app: the declared one, or else the one its script implies. + /// `None` when the manifest declares a runtime this version doesn't know. + pub fn resolve_runtime(&self) -> Option { + match &self.runtime { + Some(name) => Runtime::from_name(name), + None => Some(Runtime::infer(&self.invoke)), + } + } +} + +/// How an app's script is started. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Runtime { + /// `uv run`, after installing dependencies from pyproject.toml or requirements.txt. + Python, + /// `bash`, for `.sh` scripts. Only inferred: a Towerfile can't declare it. + Bash, + /// `bun run`, after `bun install` when there's a package.json. + Bun, +} + +impl Runtime { + /// Parses a runtime name as written in a Towerfile or MANIFEST; see `RUNTIMES`. + pub fn from_name(name: &str) -> Option { + match name { + "python" => Some(Runtime::Python), + "bun" => Some(Runtime::Bun), + _ => None, + } + } + + /// The runtime for a script that doesn't declare one: bash for `.sh`, Python otherwise. + pub fn infer(script: &str) -> Self { + if script.ends_with(".sh") { + Runtime::Bash + } else { + Runtime::Python + } + } } #[derive(Debug, Clone)] @@ -108,7 +157,8 @@ pub struct PackageInputs { // module_files have archive_name already rooted under "modules/". pub module_files: Vec, - // towerfile_bytes is the sole source of invoke, parameters, and import_paths. + // towerfile_bytes is the sole source of invoke, parameters, run_size, runtime, and + // import_paths. pub towerfile_bytes: Vec, } @@ -154,6 +204,8 @@ pub fn build_package(inputs: PackageInputs) -> Result { invoke: towerfile.app.script, parameters: towerfile.parameters, schedule: None, + run_size: towerfile.app.run_size, + runtime: towerfile.app.runtime, import_paths, app_dir_name: "app".to_string(), modules_dir_name: "modules".to_string(), @@ -316,4 +368,61 @@ import_paths = ["../shared", "libs/inner", "./weird/"] }); assert!(matches!(result, Err(Error::InvalidTowerfile { .. }))); } + + fn manifest_for(invoke: &str, runtime: Option<&str>) -> Manifest { + Manifest { + version: Some(CURRENT_PACKAGE_VERSION), + invoke: invoke.to_string(), + parameters: vec![], + schedule: None, + run_size: None, + runtime: runtime.map(str::to_string), + import_paths: vec![], + app_dir_name: "app".to_string(), + modules_dir_name: "modules".to_string(), + checksum: String::new(), + } + } + + #[test] + fn test_resolve_runtime_infers_from_the_script_when_undeclared() { + assert_eq!( + manifest_for("./task.py", None).resolve_runtime(), + Some(Runtime::Python) + ); + assert_eq!( + manifest_for("./task.sh", None).resolve_runtime(), + Some(Runtime::Bash) + ); + // Bun is never inferred: a .ts script without a runtime still goes to Python, as before. + assert_eq!( + manifest_for("./task.ts", None).resolve_runtime(), + Some(Runtime::Python) + ); + } + + #[test] + fn test_resolve_runtime_prefers_the_declared_runtime() { + assert_eq!( + manifest_for("./task.ts", Some("bun")).resolve_runtime(), + Some(Runtime::Bun) + ); + assert_eq!( + manifest_for("./task.sh", Some("bash")).resolve_runtime(), + None + ); + assert_eq!( + manifest_for("./task.ts", Some("deno")).resolve_runtime(), + None + ); + } + + #[test] + fn test_manifest_with_unknown_runtime_still_parses() { + let manifest = Manifest::from_json( + r#"{"version":3,"invoke":"main.ts","schedule":null,"runtime":"deno"}"#, + ) + .unwrap(); + assert_eq!(manifest.runtime.as_deref(), Some("deno")); + } } diff --git a/crates/tower-package/src/lib.rs b/crates/tower-package/src/lib.rs index 3f8520ed..49a8eb52 100644 --- a/crates/tower-package/src/lib.rs +++ b/crates/tower-package/src/lib.rs @@ -3,9 +3,9 @@ mod towerfile; pub use core::{ build_package, compute_sha256_bytes, compute_sha256_package, normalize_path, BuiltPackage, - Entry, Manifest, PackageInputs, CURRENT_PACKAGE_VERSION, + Entry, Manifest, PackageInputs, Runtime, CURRENT_PACKAGE_VERSION, }; -pub use towerfile::{App, Parameter, Towerfile}; +pub use towerfile::{is_bun_script, App, Parameter, Towerfile, RUNTIMES, RUN_SIZES}; #[cfg(feature = "native")] mod error; diff --git a/crates/tower-package/src/native.rs b/crates/tower-package/src/native.rs index 516c8c70..8e91ee9f 100644 --- a/crates/tower-package/src/native.rs +++ b/crates/tower-package/src/native.rs @@ -20,10 +20,9 @@ use crate::core::{ use crate::error::Error; use crate::towerfile::Towerfile; -// PackageSpec describes how to build a package. Everything core needs (invoke, parameters, -// import_paths as manifest entries) is derived from the Towerfile on disk, so this struct only -// carries what the file resolver needs: where the Towerfile lives, what's considered the project -// root, which globs match app files, and which import paths to walk. +// PackageSpec describes how to build a package. Manifest fields are derived from the Towerfile +// on disk, so this struct only carries what the file resolver needs: where the Towerfile lives, +// what's considered the project root, which globs match app files, and which import paths to walk. #[derive(Debug)] pub struct PackageSpec { pub towerfile_path: PathBuf, @@ -85,6 +84,8 @@ impl Package { invoke: "".to_string(), parameters: vec![], schedule: None, + run_size: None, + runtime: None, import_paths: vec![], app_dir_name: "app".to_string(), modules_dir_name: "modules".to_string(), @@ -450,6 +451,11 @@ impl FileResolver { return true; } + // Or in installed JavaScript dependencies, which bun installs at run time. + if is_in_dir(p, "node_modules") { + return true; + } + false } @@ -573,6 +579,8 @@ mod test { invoke: invoke.to_string(), parameters: vec![], schedule: None, + run_size: None, + runtime: None, import_paths: vec![], app_dir_name: "app".to_string(), modules_dir_name: "modules".to_string(), diff --git a/crates/tower-package/src/towerfile.rs b/crates/tower-package/src/towerfile.rs index c97250b4..ac0e888f 100644 --- a/crates/tower-package/src/towerfile.rs +++ b/crates/tower-package/src/towerfile.rs @@ -1,7 +1,27 @@ -use crate::core::Error; +use crate::core::{Error, Runtime}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; +/// Compute sizes accepted by `[app] run_size`. Managed runners apply the size; +/// self-hosted runners ignore it and use their own configured resources. When +/// the Towerfile omits it, the server applies its own default, so this list +/// deliberately has no default of its own. +pub const RUN_SIZES: [&str; 4] = ["basic.xsmall", "basic.small", "basic.medium", "basic.large"]; + +/// Runtimes accepted by `[app] runtime`. Without it an app runs with Python, except legacy +/// `.sh` scripts, which still run with bash. +pub const RUNTIMES: [&str; 2] = ["python", "bun"]; + +/// Script extensions the bun runtime accepts. +const BUN_SCRIPT_EXTENSIONS: [&str; 6] = ["ts", "tsx", "js", "jsx", "mjs", "cjs"]; + +/// Whether the bun runtime can run `script`, judged by its extension. +pub fn is_bun_script(script: &str) -> bool { + script + .rsplit_once('.') + .is_some_and(|(_, ext)| BUN_SCRIPT_EXTENSIONS.contains(&ext)) +} + #[derive(Clone, Deserialize, Serialize, Debug)] pub struct Parameter { #[serde(default)] @@ -39,6 +59,18 @@ pub struct App { #[serde(default)] pub import_paths: Vec, + + /// Compute size every run of this app gets, e.g. "basic.medium". `None` + /// means the Towerfile didn't set one (and the key is omitted when + /// serializing), leaving the server to apply its default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub run_size: Option, + + /// What starts the script, e.g. "bun". `None` means the Towerfile didn't + /// set one (and the key is omitted when serializing), so the runtime + /// follows the script's extension. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime: Option, } #[derive(Deserialize, Serialize, Debug)] @@ -66,6 +98,8 @@ impl Towerfile { schedule: String::from("0 0 * * *"), description: None, import_paths: vec![], + run_size: None, + runtime: None, }, } } @@ -74,20 +108,27 @@ impl Towerfile { /// the base_dir field always needs to be set after parsing. pub fn from_toml(toml: &str) -> Result { let towerfile: Towerfile = toml::from_str(toml)?; + towerfile.validate()?; + Ok(towerfile) + } - if towerfile.app.name.is_empty() { + /// validate checks the semantic rules that TOML parsing alone can't enforce. + /// It runs both when a Towerfile is read and before one is written back, so a + /// mutation can't leave an invalid file on disk. + pub fn validate(&self) -> Result<(), Error> { + if self.app.name.is_empty() { return Err(Error::MissingRequiredAppField { field: "name".to_string(), }); } - if towerfile.app.script.is_empty() { + if self.app.script.is_empty() { return Err(Error::MissingRequiredAppField { field: "script".to_string(), }); } - for import_path in &towerfile.app.import_paths { + for import_path in &self.app.import_paths { let as_str = import_path.to_string_lossy(); if as_str.is_empty() { return Err(Error::InvalidTowerfile { @@ -106,7 +147,52 @@ impl Towerfile { } } - Ok(towerfile) + if let Some(run_size) = &self.app.run_size { + // The server rejects unknown sizes at deploy time. Catching it here + // means the user finds out before the upload rather than after. + if !RUN_SIZES.contains(&run_size.as_str()) { + return Err(Error::InvalidTowerfile { + message: format!( + "run_size {:?} is not a valid size (expected one of: {})", + run_size, + RUN_SIZES.join(", ") + ), + }); + } + } + + if let Some(runtime) = &self.app.runtime { + match Runtime::from_name(runtime) { + None => { + return Err(Error::InvalidTowerfile { + message: format!( + "runtime {:?} is not a valid runtime (expected one of: {})", + runtime, + RUNTIMES.join(", ") + ), + }); + } + Some(Runtime::Bun) => self.validate_bun_app()?, + Some(_) => {} + } + } + + Ok(()) + } + + fn validate_bun_app(&self) -> Result<(), Error> { + let script = self.app.script.as_str(); + if !is_bun_script(script) { + return Err(Error::InvalidTowerfile { + message: format!( + "runtime \"bun\" needs a JavaScript or TypeScript script (.{}), but script is {:?}", + BUN_SCRIPT_EXTENSIONS.join(", ."), + script + ), + }); + } + + Ok(()) } /// set_parameter upserts a parameter by lookup name. If a parameter with the given name @@ -161,6 +247,8 @@ impl Towerfile { pub fn save(&self, path: Option<&std::path::Path>) -> Result<(), crate::error::Error> { use crate::error::Error as OuterError; + self.validate()?; + let target_path = path.unwrap_or_else(|| std::path::Path::new("Towerfile")); let serialized = toml::to_string_pretty(self).map_err(|err| OuterError::InvalidTowerfile { @@ -436,6 +524,234 @@ mod test { assert!(serialized.contains(r#"description = "My app""#)); } + #[test] + fn test_run_size_absent_when_not_set() { + let toml = r#" + [app] + name = "test" + script = "./script.py" + "#; + + let towerfile = crate::Towerfile::from_toml(toml).unwrap(); + assert_eq!(towerfile.app.run_size, None); + + // An unset run_size must not be written into the user's Towerfile, so + // the server keeps applying its own default. + let serialized = toml::to_string_pretty(&towerfile).unwrap(); + assert!(!serialized.contains("run_size")); + } + + #[test] + fn test_run_size_accepts_every_known_size() { + for size in crate::RUN_SIZES { + let toml = format!( + r#" + [app] + name = "test" + script = "./script.py" + run_size = "{}" + "#, + size + ); + + let towerfile = crate::Towerfile::from_toml(&toml).unwrap(); + assert_eq!(towerfile.app.run_size.as_deref(), Some(size)); + } + } + + #[test] + fn test_run_size_rejects_unknown_size() { + let toml = r#" + [app] + name = "test" + script = "./script.py" + run_size = "basic.enormous" + "#; + + let err = crate::Towerfile::from_toml(toml).unwrap_err(); + let message = err.to_string(); + assert!(message.contains("basic.enormous"), "was: {}", message); + assert!(message.contains("basic.medium"), "was: {}", message); + } + + #[test] + fn test_run_size_is_case_and_prefix_sensitive() { + // The size names are matched exactly; near misses must not slip through + // to the server. + for bad in ["medium", "basic.Medium", "BASIC.MEDIUM", " basic.medium"] { + let toml = format!( + r#" + [app] + name = "test" + script = "./script.py" + run_size = "{}" + "#, + bad + ); + + assert!( + crate::Towerfile::from_toml(&toml).is_err(), + "expected {:?} to be rejected", + bad + ); + } + } + + #[test] + fn test_run_size_survives_read_modify_write() { + // Regression: every MCP write round-trips the Towerfile through this + // struct, so a field that isn't modelled is silently dropped from the + // user's file. Editing an unrelated field must leave run_size intact. + let toml = r#" + [app] + name = "test" + script = "./script.py" + run_size = "basic.large" + "#; + + let mut towerfile = crate::Towerfile::from_toml(toml).unwrap(); + towerfile.app.description = Some("edited".to_string()); + + let serialized = toml::to_string_pretty(&towerfile).unwrap(); + let reparsed = crate::Towerfile::from_toml(&serialized).unwrap(); + + assert_eq!(reparsed.app.run_size.as_deref(), Some("basic.large")); + assert_eq!(reparsed.app.description.as_deref(), Some("edited")); + } + + #[test] + fn test_save_refuses_to_persist_an_invalid_run_size() { + // MCP edits mutate the struct and write it straight back, so save() has + // to hold the mutation to the same rules as a read. Otherwise a bad + // value lands on disk and the user only finds out on the next read. + let toml = r#" + [app] + name = "test" + script = "./script.py" + run_size = "basic.small" + "#; + + let mut towerfile = crate::Towerfile::from_toml(toml).unwrap(); + towerfile.app.run_size = Some("basic.enormous".to_string()); + + let tempfile = + TestFile::new("Towerfile-save-invalid").expect("Failed to create temporary file"); + let path = PathBuf::from("Towerfile-save-invalid"); + + let err = towerfile.save(Some(&path)).unwrap_err(); + assert!(err.to_string().contains("basic.enormous"), "was: {}", err); + + // The file must be untouched, not half-written. + assert_eq!(std::fs::read_to_string(&path).unwrap(), ""); + + drop(tempfile); + } + + fn towerfile_with_runtime(script: &str, runtime: &str) -> String { + format!( + r#" + [app] + name = "test" + script = "{}" + runtime = "{}" + "#, + script, runtime + ) + } + + #[test] + fn test_runtime_absent_when_not_set() { + let toml = r#" + [app] + name = "test" + script = "./script.py" + "#; + + let towerfile = crate::Towerfile::from_toml(toml).unwrap(); + assert_eq!(towerfile.app.runtime, None); + + // An unset runtime must not be written into the user's Towerfile, so it + // keeps following the script's extension. + let serialized = toml::to_string_pretty(&towerfile).unwrap(); + assert!(!serialized.contains("runtime")); + } + + #[test] + fn test_runtime_accepts_every_known_runtime() { + for (runtime, script) in [("python", "./task.py"), ("bun", "./task.ts")] { + let towerfile = + crate::Towerfile::from_toml(&towerfile_with_runtime(script, runtime)).unwrap(); + assert_eq!(towerfile.app.runtime.as_deref(), Some(runtime)); + } + // Every advertised name is one the runtime can start. + for name in crate::RUNTIMES { + assert!( + crate::Runtime::from_name(name).is_some(), + "{:?} doesn't parse", + name + ); + } + } + + #[test] + fn test_runtime_rejects_unknown_runtime() { + // bash can't be declared: `.sh` scripts keep running with bash only by extension. + for bad in ["bash", "node", "Bun", "uv", ""] { + let err = + crate::Towerfile::from_toml(&towerfile_with_runtime("./task.ts", bad)).unwrap_err(); + let message = err.to_string(); + assert!(message.contains("python, bun"), "was: {}", message); + } + } + + #[test] + fn test_bun_runtime_requires_a_javascript_or_typescript_script() { + for script in [ + "./task.ts", + "./task.tsx", + "./src/task.mjs", + "./task.cjs", + "task.js", + ] { + assert!( + crate::Towerfile::from_toml(&towerfile_with_runtime(script, "bun")).is_ok(), + "expected {:?} to be accepted", + script + ); + } + for script in ["./task.py", "./task.sh", "./task", "./task.ts.bak"] { + let err = + crate::Towerfile::from_toml(&towerfile_with_runtime(script, "bun")).unwrap_err(); + assert!(err.to_string().contains(".ts"), "was: {}", err); + } + } + + #[test] + fn test_bun_runtime_accepts_import_paths() { + let toml = r#" + [app] + name = "test" + script = "./task.ts" + runtime = "bun" + import_paths = ["../shared"] + "#; + + let towerfile = crate::Towerfile::from_toml(toml).unwrap(); + assert_eq!(towerfile.app.import_paths, vec![PathBuf::from("../shared")]); + } + + #[test] + fn test_runtime_survives_read_modify_write() { + let mut towerfile = + crate::Towerfile::from_toml(&towerfile_with_runtime("./task.ts", "bun")).unwrap(); + towerfile.app.description = Some("edited".to_string()); + + let serialized = toml::to_string_pretty(&towerfile).unwrap(); + let reparsed = crate::Towerfile::from_toml(&serialized).unwrap(); + + assert_eq!(reparsed.app.runtime.as_deref(), Some("bun")); + } + #[test] fn test_roundtrip_serialization() { let original_toml = r#"[app] diff --git a/crates/tower-package/tests/package_test.rs b/crates/tower-package/tests/package_test.rs index e39a020f..cf1df09b 100644 --- a/crates/tower-package/tests/package_test.rs +++ b/crates/tower-package/tests/package_test.rs @@ -11,7 +11,7 @@ use tokio::{ use tokio_stream::*; use tokio_tar::Archive; -use tower_package::{Manifest, Package, PackageSpec, Towerfile}; +use tower_package::{Manifest, Package, PackageSpec, Runtime, Towerfile}; use tower_telemetry::debug; const TRIVIAL_TOWERFILE: &str = "[app]\nname = \"test\"\nscript = \"main.py\"\n"; @@ -57,6 +57,13 @@ async fn it_creates_package() { "files {:?} was missing MANIFEST", files ); + + let manifest = &files["MANIFEST"]; + let json: serde_json::Value = serde_json::from_str(manifest).unwrap(); + assert!(json.get("run_size").is_none()); + assert_eq!(Manifest::from_json(manifest).unwrap().run_size, None); + // Without a declared runtime the manifest stays as it was, and the runtime follows the script. + assert!(json.get("runtime").is_none()); } #[tokio::test] @@ -363,6 +370,12 @@ async fn it_excludes_various_content_that_should_not_be_there() { ) .await; create_test_file(tmp_dir.to_path_buf(), ".git/some-file", "").await; + create_test_file( + tmp_dir.to_path_buf(), + "node_modules/ms/index.js", + "module.exports = {}", + ) + .await; let spec = PackageSpec { base_dir: tmp_dir.to_path_buf(), @@ -389,6 +402,12 @@ async fn it_excludes_various_content_that_should_not_be_there() { "files {:?} contained a .pyc", files ); + assert!(files.contains_key("app/main.py"), "files {:?}", files); + assert!( + !files.contains_key("app/node_modules/ms/index.js"), + "files {:?} contained node_modules", + files + ); } #[tokio::test] @@ -605,3 +624,68 @@ where .expect("Failed to read string from stream"); content } + +#[tokio::test] +async fn it_preserves_run_size_in_the_manifest_and_towerfile() { + // The deploy handler reads run_size from MANIFEST when creating the app version. + let tmp_dir = TmpDir::new("example") + .await + .expect("Failed to create temp dir"); + + let towerfile = "[app]\nname = \"test\"\nscript = \"main.py\"\nrun_size = \"basic.large\"\n"; + create_test_file(tmp_dir.to_path_buf(), "Towerfile", towerfile).await; + create_test_file(tmp_dir.to_path_buf(), "main.py", "print('Hello, world!')").await; + + let spec = PackageSpec { + base_dir: tmp_dir.to_path_buf(), + towerfile_path: tmp_dir.to_path_buf().join("Towerfile").to_path_buf(), + file_globs: vec!["*.py".to_string()], + import_paths: vec![], + }; + + let package = Package::build(spec).await.expect("Failed to build package"); + let files = read_package_files(package).await; + + let packaged = files + .get("Towerfile") + .expect("package was missing the Towerfile entry"); + assert_eq!(packaged, towerfile); + + let manifest = files.get("MANIFEST").expect("package was missing MANIFEST"); + let json: serde_json::Value = serde_json::from_str(manifest).unwrap(); + assert_eq!(json["run_size"], "basic.large"); + assert_eq!( + Manifest::from_json(manifest).unwrap().run_size.as_deref(), + Some("basic.large") + ); +} + +#[tokio::test] +async fn it_copies_a_declared_runtime_into_the_manifest() { + // The deploy handler reads runtime from MANIFEST to pick the runner image. + let tmp_dir = TmpDir::new("example") + .await + .expect("Failed to create temp dir"); + + let towerfile = "[app]\nname = \"test\"\nscript = \"main.ts\"\nruntime = \"bun\"\n"; + create_test_file(tmp_dir.to_path_buf(), "Towerfile", towerfile).await; + create_test_file(tmp_dir.to_path_buf(), "main.ts", "console.log('hi')").await; + + let spec = PackageSpec { + base_dir: tmp_dir.to_path_buf(), + towerfile_path: tmp_dir.to_path_buf().join("Towerfile").to_path_buf(), + file_globs: vec!["*.ts".to_string()], + import_paths: vec![], + }; + + let package = Package::build(spec).await.expect("Failed to build package"); + let files = read_package_files(package).await; + + let manifest = files.get("MANIFEST").expect("package was missing MANIFEST"); + let json: serde_json::Value = serde_json::from_str(manifest).unwrap(); + assert_eq!(json["runtime"], "bun"); + assert_eq!( + Manifest::from_json(manifest).unwrap().resolve_runtime(), + Some(Runtime::Bun) + ); +} diff --git a/crates/tower-runtime/src/errors.rs b/crates/tower-runtime/src/errors.rs index 0b4a96fd..45deb146 100644 --- a/crates/tower-runtime/src/errors.rs +++ b/crates/tower-runtime/src/errors.rs @@ -53,6 +53,12 @@ pub enum Error { #[snafu(display("bash could not be found"))] MissingBash, + #[snafu(display("bun could not be found"))] + MissingBun, + + #[snafu(display("unsupported runtime {runtime:?}"))] + UnsupportedRuntime { runtime: String }, + #[snafu(display("failed to create a virtual environment"))] VirtualEnvCreationFailed, diff --git a/crates/tower-runtime/src/local.rs b/crates/tower-runtime/src/local.rs index 51b0cafa..b9e18391 100644 --- a/crates/tower-runtime/src/local.rs +++ b/crates/tower-runtime/src/local.rs @@ -29,7 +29,7 @@ use nix::{ use tokio_util::sync::CancellationToken; -use tower_package::{Manifest, Package}; +use tower_package::{Manifest, Runtime}; use tower_telemetry::debug; use tower_uv::Uv; @@ -132,6 +132,14 @@ async fn find_bash() -> Result { } } +async fn find_bun() -> Result { + if let Some(path) = find_executable_in_path("bun").await { + Ok(path) + } else { + Err(Error::MissingBun) + } +} + /// Run the app to completion, returning the child's exit code on success or an /// `Error` for any platform failure. Cancellation is reported as `Ok(-1)` to /// preserve the historical `Status::Crashed { code: -1 }` semantic. @@ -158,6 +166,11 @@ async fn inner_execute_local_app( debug!(ctx: &ctx, " - working directory: {:?}", &working_dir); let manifest = &package.manifest; + let runtime = manifest + .resolve_runtime() + .ok_or_else(|| Error::UnsupportedRuntime { + runtime: manifest.runtime.clone().unwrap_or_default(), + })?; let secrets = opts.secrets; let params = opts.parameters; let other_env_vars = opts.env_vars; @@ -170,7 +183,7 @@ async fn inner_execute_local_app( .collect(); if !import_paths.is_empty() { - debug!(ctx: &ctx, "adding import paths to PYTHONPATH: {:?}", import_paths); + debug!(ctx: &ctx, "adding import paths to the module search path: {:?}", import_paths); } // We insert these checks for cancellation along the way to see if the process was @@ -182,7 +195,7 @@ async fn inner_execute_local_app( return Err(Error::Cancelled); } - if is_bash_package(&package) { + if runtime == Runtime::Bash { let child = execute_bash_program( &ctx, working_dir, @@ -203,6 +216,38 @@ async fn inner_execute_local_app( return Err(Error::Cancelled); } Ok(code) + } else if runtime == Runtime::Bun { + let script = working_dir.join(&manifest.invoke); + let mut env_vars = make_env_vars( + &ctx, + &working_dir, + &secrets, + ¶ms, + &other_env_vars, + &import_paths, + ); + if !import_paths.is_empty() { + // Shared code lives under modules/, outside the app directory, so it reaches the + // app's packages through NODE_PATH as well. + extend_search_path( + &mut env_vars, + "NODE_PATH", + import_paths + .iter() + .cloned() + .chain([working_dir.join("node_modules")]), + ); + } + execute_bun_app( + &ctx, + &cancel_token, + &opts.output_sender, + working_dir, + script, + opts.cache_dir, + env_vars, + ) + .await } else { // we put Uv in to protected mode when there's no caching configured/enabled. let protected_mode = opts.cache_dir.is_none(); @@ -505,6 +550,90 @@ async fn execute_bash_program( Ok(child) } +/// Runs a bun app: `bun install` when the app has a package.json, then `bun run` on its script. +/// Returns the exit code of whichever step ends the run. +async fn execute_bun_app( + ctx: &tower_telemetry::Context, + cancel_token: &CancellationToken, + output_sender: &OutputSender, + cwd: PathBuf, + script: PathBuf, + cache_dir: Option, + env_vars: HashMap, +) -> Result { + let bun_path = find_bun().await?; + debug!(ctx: &ctx, "using bun at {:?}", bun_path); + + let protected_mode = cache_dir.is_none(); + let mut env_vars = tower_uv::normalize_env_vars(&env_vars); + if let Some(dir) = &cache_dir { + env_vars.insert( + "BUN_INSTALL_CACHE_DIR".to_string(), + dir.join("bun").to_string_lossy().to_string(), + ); + } + + if cwd.join("package.json").exists() { + let mut cmd = bun_command(&bun_path, &cwd, &env_vars, protected_mode); + cmd.arg("install").arg("--no-progress"); + // With a lockfile, install exactly what it pins rather than re-resolving. + if cwd.join("bun.lock").exists() || cwd.join("bun.lockb").exists() { + cmd.arg("--frozen-lockfile"); + } + + let code = run_setup_child(ctx, cancel_token, output_sender, cmd.spawn()?).await; + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + if code != 0 { + // A failed install is a crash with bun's exit code, like a failed `uv sync`. + return Ok(code); + } + } + + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + + debug!(ctx: &ctx, " - bun script {:?}", script); + + let mut cmd = bun_command(&bun_path, &cwd, &env_vars, protected_mode); + let mut child = cmd.arg("run").arg(&script).spawn()?; + forward_output(&mut child, Channel::Program, output_sender); + + let code = wait_for_process(ctx.clone(), cancel_token, child).await; + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + Ok(code) +} + +fn bun_command( + bun_path: &PathBuf, + cwd: &PathBuf, + env_vars: &HashMap, + protected_mode: bool, +) -> Command { + let mut cmd = Command::new(bun_path); + cmd.current_dir(cwd) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + + #[cfg(unix)] + { + cmd.process_group(0); + } + + if protected_mode { + cmd.env_clear(); + } + cmd.envs(env_vars); + + cmd +} + fn make_env_var_key(src: &str) -> String { // TODO: We have this special case defined for dltHub, and I'm not sure that we want to... if src.starts_with("dlt.") { @@ -544,25 +673,37 @@ fn make_env_vars( let added_keys = res.keys().map(|s| &**s).collect::>().join(", "); debug!(ctx: &ctx, "added keys {}", &added_keys); - let mut path_entries: Vec = res - .remove("PYTHONPATH") + extend_search_path( + &mut res, + "PYTHONPATH", + import_paths.iter().cloned().chain([cwd.clone()]), + ); + + res.insert("PYTHONUNBUFFERED".to_string(), "x".to_string()); + + res +} + +/// Appends `extra` to the path list in `env_vars[key]`, after any entries already there. +fn extend_search_path( + env_vars: &mut HashMap, + key: &str, + extra: impl IntoIterator, +) { + let mut entries: Vec = env_vars + .remove(key) .as_deref() .map(env::split_paths) .into_iter() .flatten() .collect(); - path_entries.extend(import_paths.iter().cloned()); - path_entries.push(cwd.clone()); + entries.extend(extra); - let pythonpath = env::join_paths(&path_entries) + let joined = env::join_paths(&entries) .unwrap() .to_string_lossy() .to_string(); - res.insert("PYTHONPATH".to_string(), pythonpath); - - res.insert("PYTHONUNBUFFERED".to_string(), "x".to_string()); - - res + env_vars.insert(key.to_string(), joined); } #[cfg(unix)] @@ -627,6 +768,25 @@ async fn run_setup_child( wait_for_process(ctx.clone(), cancel_token, child).await } +/// Sends the child's stdout and stderr lines to `output` on `channel` until the pipes close. +fn forward_output(child: &mut Child, channel: Channel, output: &OutputSender) { + let stdout = child.stdout.take().expect("no stdout"); + tokio::spawn(drain_output( + FD::Stdout, + channel, + output.clone(), + BufReader::new(stdout), + )); + + let stderr = child.stderr.take().expect("no stderr"); + tokio::spawn(drain_output( + FD::Stderr, + channel, + output.clone(), + BufReader::new(stderr), + )); +} + async fn wait_for_process( ctx: tower_telemetry::Context, cancel_token: &CancellationToken, @@ -675,7 +835,3 @@ async fn drain_output( }); } } - -fn is_bash_package(package: &Package) -> bool { - return package.manifest.invoke.ends_with(".sh"); -} diff --git a/crates/tower-runtime/tests/example-apps/08-bun-hello/Towerfile b/crates/tower-runtime/tests/example-apps/08-bun-hello/Towerfile new file mode 100644 index 00000000..98b5a64d --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/08-bun-hello/Towerfile @@ -0,0 +1,4 @@ +[app] +name = "08-bun-hello" +script = "./main.ts" +runtime = "bun" diff --git a/crates/tower-runtime/tests/example-apps/08-bun-hello/bun.lock b/crates/tower-runtime/tests/example-apps/08-bun-hello/bun.lock new file mode 100644 index 00000000..d6e4faf8 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/08-bun-hello/bun.lock @@ -0,0 +1,15 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "08-bun-hello", + "dependencies": { + "ms": "2.1.3", + }, + }, + }, + "packages": { + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + } +} diff --git a/crates/tower-runtime/tests/example-apps/08-bun-hello/main.ts b/crates/tower-runtime/tests/example-apps/08-bun-hello/main.ts new file mode 100644 index 00000000..4e2e253b --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/08-bun-hello/main.ts @@ -0,0 +1,5 @@ +import ms from "ms"; + +console.log(`One minute is ${ms(60_000)}`); +console.log(`The greeting is: ${process.env.GREETING ?? "unset"}`); +console.log(`The parent variable is: ${process.env.BUN_TEST_PARENT_VARIABLE ?? "unset"}`); diff --git a/crates/tower-runtime/tests/example-apps/08-bun-hello/package.json b/crates/tower-runtime/tests/example-apps/08-bun-hello/package.json new file mode 100644 index 00000000..e47447d8 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/08-bun-hello/package.json @@ -0,0 +1,7 @@ +{ + "name": "08-bun-hello", + "private": true, + "dependencies": { + "ms": "2.1.3" + } +} diff --git a/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/Towerfile b/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/Towerfile new file mode 100644 index 00000000..146c79b4 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/Towerfile @@ -0,0 +1,4 @@ +[app] +name = "09-bun-stale-lockfile" +script = "./main.ts" +runtime = "bun" diff --git a/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/bun.lock b/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/bun.lock new file mode 100644 index 00000000..d6e4faf8 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/bun.lock @@ -0,0 +1,15 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "08-bun-hello", + "dependencies": { + "ms": "2.1.3", + }, + }, + }, + "packages": { + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + } +} diff --git a/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/main.ts b/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/main.ts new file mode 100644 index 00000000..4e2e253b --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/main.ts @@ -0,0 +1,5 @@ +import ms from "ms"; + +console.log(`One minute is ${ms(60_000)}`); +console.log(`The greeting is: ${process.env.GREETING ?? "unset"}`); +console.log(`The parent variable is: ${process.env.BUN_TEST_PARENT_VARIABLE ?? "unset"}`); diff --git a/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/package.json b/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/package.json new file mode 100644 index 00000000..fe2055ef --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/09-bun-stale-lockfile/package.json @@ -0,0 +1,7 @@ +{ + "name": "08-bun-hello", + "private": true, + "dependencies": { + "ms": "2.1.2" + } +} diff --git a/crates/tower-runtime/tests/example-apps/10-bun-import-paths/Towerfile b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/Towerfile new file mode 100644 index 00000000..b034bfb3 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/Towerfile @@ -0,0 +1,7 @@ +[app] +name = "10-bun-import-paths" +script = "./main.ts" +runtime = "bun" +import_paths = [ + "./lib", +] diff --git a/crates/tower-runtime/tests/example-apps/10-bun-import-paths/bun.lock b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/bun.lock new file mode 100644 index 00000000..361fa1d0 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/bun.lock @@ -0,0 +1,15 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "10-bun-import-paths", + "dependencies": { + "ms": "2.1.2", + }, + }, + }, + "packages": { + "ms": ["ms@2.1.2", "", {}, "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="], + } +} diff --git a/crates/tower-runtime/tests/example-apps/10-bun-import-paths/lib/greet.ts b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/lib/greet.ts new file mode 100644 index 00000000..26eefb13 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/lib/greet.ts @@ -0,0 +1,6 @@ +import ms from "ms"; +import { version } from "ms/package.json"; + +export function greet() { + console.log(`hello from lib, one minute is ${ms(60_000)} (ms ${version})`); +} diff --git a/crates/tower-runtime/tests/example-apps/10-bun-import-paths/main.ts b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/main.ts new file mode 100644 index 00000000..9c752910 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/main.ts @@ -0,0 +1,3 @@ +import { greet } from "greet"; + +greet(); diff --git a/crates/tower-runtime/tests/example-apps/10-bun-import-paths/package.json b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/package.json new file mode 100644 index 00000000..10074e10 --- /dev/null +++ b/crates/tower-runtime/tests/example-apps/10-bun-import-paths/package.json @@ -0,0 +1,7 @@ +{ + "name": "10-bun-import-paths", + "private": true, + "dependencies": { + "ms": "2.1.2" + } +} diff --git a/crates/tower-runtime/tests/local_test.rs b/crates/tower-runtime/tests/local_test.rs index fdd634cf..558817a6 100644 --- a/crates/tower-runtime/tests/local_test.rs +++ b/crates/tower-runtime/tests/local_test.rs @@ -431,3 +431,220 @@ async fn test_running_app_with_multiple_import_paths() { status ); } + +/// The bun runtime runs bun from PATH. CI installs it; elsewhere the bun tests skip without it. +#[cfg(unix)] +fn bun_available() -> bool { + let found = std::env::var_os("PATH") + .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join("bun").is_file())) + .unwrap_or(false); + if !found { + assert!( + std::env::var_os("CI").is_none(), + "bun must be on PATH in CI" + ); + eprintln!("skipping: bun is not on PATH"); + } + found +} + +#[cfg(unix)] +#[tokio::test] +async fn test_running_bun_app() { + if !bun_available() { + return; + } + debug!("Running 08-bun-hello"); + + // Protected mode must keep this out of the app's environment. + std::env::set_var("BUN_TEST_PARENT_VARIABLE", "should not reach the app"); + + let dir = get_example_app_dir("08-bun-hello"); + let package = build_package_from_dir(&dir).await; + let (sender, mut receiver) = unbounded_channel(); + + let mut parameters = HashMap::new(); + parameters.insert("GREETING".to_string(), "hello from a parameter".to_string()); + + let opts = StartOptions { + ctx: tower_telemetry::Context::new("runner-id".to_string()), + package, + output_sender: sender, + cwd: None, + environment: "local".to_string(), + secrets: HashMap::new(), + parameters, + env_vars: HashMap::new(), + // No cache dir means protected mode. + cache_dir: None, + }; + + let app = LocalApp::start(opts).await.expect("Failed to start app"); + + let mut setup = Vec::new(); + let mut program = Vec::new(); + while let Some(output) = receiver.recv().await { + match output.channel { + tower_runtime::Channel::Setup => setup.push(output.line), + tower_runtime::Channel::Program => program.push(output.line), + } + } + + assert!( + !setup.is_empty(), + "bun install should report on the setup channel" + ); + // The dependency resolved, so `bun install` ran before the script. + assert!( + program.contains(&"One minute is 1m".to_string()), + "expected the app's output; got {:?}", + program + ); + assert!(program.contains(&"The greeting is: hello from a parameter".to_string())); + assert!( + program.contains(&"The parent variable is: unset".to_string()), + "protected mode should not pass the host environment through" + ); + + let status = app.status().await.expect("Failed to get app status"); + assert!( + status == Status::Exited, + "App should have exited cleanly, got {:?}", + status + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn test_running_bun_app_with_import_paths() { + if !bun_available() { + return; + } + debug!("Running 10-bun-import-paths"); + + let dir = get_example_app_dir("10-bun-import-paths"); + let package = build_package_from_dir(&dir).await; + let (sender, mut receiver) = unbounded_channel(); + + let opts = StartOptions { + ctx: tower_telemetry::Context::new("runner-id".to_string()), + package, + output_sender: sender, + cwd: None, + environment: "local".to_string(), + secrets: HashMap::new(), + parameters: HashMap::new(), + env_vars: HashMap::new(), + cache_dir: None, + }; + + let app = LocalApp::start(opts).await.expect("Failed to start app"); + + let mut program = Vec::new(); + while let Some(output) = receiver.recv().await { + if let tower_runtime::Channel::Program = output.channel { + program.push(output.line); + } + } + + // main.ts imports the shared module by name, and the shared module gets the app's pinned + // `ms`. Without the app's node_modules on NODE_PATH, bun would auto-install the latest. + assert!( + program.contains(&"hello from lib, one minute is 1m (ms 2.1.2)".to_string()), + "expected output from the shared module; got {:?}", + program + ); + + let status = app.status().await.expect("Failed to get app status"); + assert!( + status == Status::Exited, + "App should have exited cleanly, got {:?}", + status + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn test_running_bun_app_installs_into_the_cache_dir() { + if !bun_available() { + return; + } + + let dir = get_example_app_dir("08-bun-hello"); + let package = build_package_from_dir(&dir).await; + let (sender, mut receiver) = unbounded_channel(); + let cache_dir = tmpdir::TmpDir::new("tower-bun-cache") + .await + .expect("Failed to create cache dir"); + + let opts = StartOptions { + ctx: tower_telemetry::Context::new("runner-id".to_string()), + package, + output_sender: sender, + cwd: None, + environment: "local".to_string(), + secrets: HashMap::new(), + parameters: HashMap::new(), + env_vars: HashMap::new(), + cache_dir: Some(cache_dir.to_path_buf()), + }; + + let app = LocalApp::start(opts).await.expect("Failed to start app"); + while receiver.recv().await.is_some() {} + + let status = app.status().await.expect("Failed to get app status"); + assert!( + status == Status::Exited, + "App should have exited cleanly, got {:?}", + status + ); + assert!( + cache_dir.to_path_buf().join("bun").is_dir(), + "bun install should use the runtime's cache dir" + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn test_abort_on_bun_install_failure() { + if !bun_available() { + return; + } + // package.json no longer matches bun.lock, so a frozen install fails. + let dir = get_example_app_dir("09-bun-stale-lockfile"); + let package = build_package_from_dir(&dir).await; + let (sender, mut receiver) = unbounded_channel(); + + let opts = StartOptions { + ctx: tower_telemetry::Context::new("runner-id".to_string()), + package, + output_sender: sender, + cwd: None, + environment: "local".to_string(), + secrets: HashMap::new(), + parameters: HashMap::new(), + env_vars: HashMap::new(), + cache_dir: Some(config::default_cache_dir()), + }; + + let app = LocalApp::start(opts).await.expect("Failed to start app"); + + let mut program = Vec::new(); + while let Some(output) = receiver.recv().await { + if matches!(output.channel, tower_runtime::Channel::Program) { + program.push(output.line); + } + } + + assert!( + program.is_empty(), + "the script should not run: {:?}", + program + ); + let status = app.status().await.expect("Failed to get app status"); + assert!( + matches!(status, Status::Crashed { code } if code != 0), + "a failed install should crash the app, got {:?}", + status + ); +} diff --git a/crates/tower-runtime/tests/missing_bun_test.rs b/crates/tower-runtime/tests/missing_bun_test.rs new file mode 100644 index 00000000..6009d041 --- /dev/null +++ b/crates/tower-runtime/tests/missing_bun_test.rs @@ -0,0 +1,45 @@ +// This test clears PATH for the whole process, so it lives in its own test binary. +#![cfg(unix)] + +use std::collections::HashMap; +use std::path::PathBuf; + +use tokio::sync::mpsc::unbounded_channel; + +use config::Towerfile; +use tower_package::{Package, PackageSpec}; +use tower_runtime::{errors::Error, local::LocalApp, App, AppFailure, StartOptions, Status}; + +#[tokio::test] +async fn test_bun_app_fails_without_bun() { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/example-apps/08-bun-hello"); + let towerfile = Towerfile::from_path(dir.join("Towerfile")).expect("Failed to load Towerfile"); + let mut package = Package::build(PackageSpec::from_towerfile(&towerfile)) + .await + .expect("Failed to build package"); + package.unpack().await.expect("Failed to unpack package"); + + std::env::set_var("PATH", ""); + + let (sender, mut receiver) = unbounded_channel(); + let opts = StartOptions { + ctx: tower_telemetry::Context::new("runner-id".to_string()), + package, + output_sender: sender, + cwd: None, + environment: "local".to_string(), + secrets: HashMap::new(), + parameters: HashMap::new(), + env_vars: HashMap::new(), + cache_dir: None, + }; + + let app = LocalApp::start(opts).await.expect("Failed to start app"); + while receiver.recv().await.is_some() {} + + let status = app.status().await.expect("Failed to get app status"); + assert_eq!( + status, + Status::Failed(AppFailure::Runtime(Error::MissingBun)) + ); +} diff --git a/crates/tower-uv/src/lib.rs b/crates/tower-uv/src/lib.rs index 7af6006c..20ff084e 100644 --- a/crates/tower-uv/src/lib.rs +++ b/crates/tower-uv/src/lib.rs @@ -58,7 +58,9 @@ impl From for Error { } } -fn normalize_env_vars(env_vars: &HashMap) -> HashMap { +/// Adds the host variables a child process needs (PATH, HOME, temp dirs, TZ, LANG, ...) to +/// `env_vars`. Use it for any app process, since protected mode clears everything else. +pub fn normalize_env_vars(env_vars: &HashMap) -> HashMap { // we copy this locally so we can mutate the results. let mut env_vars = env_vars.clone(); diff --git a/plugin/skills/tower/SKILL.md b/plugin/skills/tower/SKILL.md index a6fe0ad7..281e6c3a 100644 --- a/plugin/skills/tower/SKILL.md +++ b/plugin/skills/tower/SKILL.md @@ -82,6 +82,8 @@ uv init Creates `pyproject.toml`, `main.py`, `README.md`. Keep `pyproject.toml` minimal — `[project]` metadata and dependencies only. No `[build-system]`, `[tool.hatchling]`, or similar. Skip if a `pyproject.toml` already exists. +For a JavaScript or TypeScript app, run `bun init -y` instead, which creates `package.json` and `index.ts`. Skip it if a `package.json` already exists. Without a `pyproject.toml`, `tower_file_generate` reads `package.json` and sets `runtime = "bun"`. + ### 1. Towerfile ``` @@ -90,6 +92,10 @@ tower_file_generate → tower_file_update → tower_file_add/edit/remove_paramet Always use `tower_file_update` or `tower_file_add/edit/remove_parameter` to modify. Never edit the TOML directly. +To give an app more compute, set `run_size` via `tower_file_update`: one of `basic.xsmall`, `basic.small`, `basic.medium` (the default), `basic.large`. It is versioned with the code, and self-hosted runners ignore it. + +To run JavaScript or TypeScript, set `runtime` to `bun` via `tower_file_update` and point `script` at a `.ts`, `.tsx`, `.js`, `.jsx`, `.mjs` or `.cjs` file. Dependencies come from `package.json` (and `bun.lock`). Leave `runtime` unset for Python apps. + ### 2. Local development (preferred) ``` diff --git a/pyproject.toml b/pyproject.toml index afb00418..d1265b9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "tower" -version = "0.3.75" +version = "0.3.76" description = "Tower CLI and runtime environment for Tower." authors = [{ name = "Tower Computing GmbH", email = "brad@tower.dev" }] readme = "README.md" diff --git a/tests/integration/features/mcp_app_management.feature b/tests/integration/features/mcp_app_management.feature index ccae5790..2d528be1 100644 --- a/tests/integration/features/mcp_app_management.feature +++ b/tests/integration/features/mcp_app_management.feature @@ -34,6 +34,24 @@ Feature: MCP App Management When I call tower_file_read via MCP Then I should receive the parsed Towerfile configuration + Scenario: Set the app's compute size through the Towerfile + Given I have a valid Towerfile in the current directory + When I call tower_file_update with run_size "basic.large" + Then I should receive a success response + And the Towerfile on disk should contain run_size "basic.large" + + Scenario: Reject an invalid compute size + Given I have a valid Towerfile in the current directory + When I call tower_file_update with run_size "basic.enormous" + Then I should receive an error response + And the Towerfile on disk should contain run_size "basic.small" + + Scenario: Editing an unrelated field preserves the compute size + Given I have a valid Towerfile in the current directory + When I call tower_file_update with description "an edited description" + Then I should receive a success response + And the Towerfile on disk should contain run_size "basic.small" + Scenario: Run simple application successfully locally Given I have a simple hello world application When I call tower_run_local via MCP @@ -58,6 +76,12 @@ Feature: MCP App Management Then I should receive a valid TOML Towerfile And the Towerfile should contain the project name and description + Scenario: Generate Towerfile from package.json + Given I have a package.json file with project metadata + When I call tower_file_generate via MCP + Then I should receive a valid TOML Towerfile + And the Towerfile should run index.ts with bun + Scenario: List schedules when none exist When I call tower_schedules_list via MCP Then I should receive a response with empty schedules data diff --git a/tests/integration/features/steps/mcp_steps.py b/tests/integration/features/steps/mcp_steps.py index b9afa60e..7696c282 100644 --- a/tests/integration/features/steps/mcp_steps.py +++ b/tests/integration/features/steps/mcp_steps.py @@ -235,6 +235,21 @@ def step_create_pyproject_toml(context): f.write('print("Hello from test project")\n') +@given("I have a package.json file with project metadata") +def step_create_package_json(context): + package_json_content = """{ + "name": "@acme/test-bun-project", + "description": "A test bun project for Towerfile generation", + "module": "index.ts", + "type": "module" +} +""" + with open("package.json", "w") as f: + f.write(package_json_content) + with open("index.ts", "w") as f: + f.write('console.log("Hello from test bun project");\n') + + @step("I call {tool_name} via MCP") @async_run_until_complete async def step_call_mcp_tool(context, tool_name): @@ -249,6 +264,26 @@ async def step_call_mcp_tool_with_unique_app_name(context, tool_name, app_name): ) +@step('I call tower_file_update with run_size "{run_size}"') +@async_run_until_complete +async def step_update_run_size(context, run_size): + await call_mcp_tool(context, "tower_file_update", {"run_size": run_size}) + + +@step('I call tower_file_update with description "{description}"') +@async_run_until_complete +async def step_update_description(context, description): + await call_mcp_tool(context, "tower_file_update", {"description": description}) + + +@then('the Towerfile on disk should contain run_size "{run_size}"') +def step_check_towerfile_run_size(context, run_size): + contents = Path("Towerfile").read_text() + assert ( + f'run_size = "{run_size}"' in contents + ), f"Towerfile should keep run_size {run_size!r}, got: {contents}" + + @step("I should receive a response") def step_check_response_exists(context): assert hasattr(context, "mcp_response") and context.mcp_response is not None @@ -374,6 +409,22 @@ def step_check_towerfile_metadata(context): ), f"Towerfile should contain project name and description, got: {context.mcp_response.get('content', [])}" +@step("the Towerfile should run index.ts with bun") +def step_check_bun_towerfile(context): + """Verify the Towerfile generated from package.json runs its module with bun.""" + assert hasattr(context, "mcp_response"), "No MCP response was recorded" + + found_bun_towerfile = has_text_content( + context.mcp_response, + lambda text: 'name = "test-bun-project"' in text + and 'script = "./index.ts"' in text + and 'runtime = "bun"' in text, + ) + assert ( + found_bun_towerfile + ), f"Towerfile should run index.ts with bun, got: {context.mcp_response.get('content', [])}" + + @step("the MCP server should remain responsive") @async_run_until_complete async def step_check_server_responsive(context): diff --git a/tests/integration/templates/Towerfile.j2 b/tests/integration/templates/Towerfile.j2 index a6e926ef..11a521fb 100644 --- a/tests/integration/templates/Towerfile.j2 +++ b/tests/integration/templates/Towerfile.j2 @@ -3,6 +3,7 @@ name = "{{ app_name }}" script = "./{{ script_name }}" description = "{{ description }}" source = ["./{{ script_name }}"] +run_size = "basic.small" [build] python = "3.11" \ No newline at end of file diff --git a/uv.lock b/uv.lock index c20f1de8..a10f890f 100644 --- a/uv.lock +++ b/uv.lock @@ -2264,7 +2264,7 @@ wheels = [ [[package]] name = "tower" -version = "0.3.75" +version = "0.3.76" source = { editable = "." } dependencies = [ { name = "attrs" },