From 8d62c570b6716efc6c3ffacf98588a85655899f4 Mon Sep 17 00:00:00 2001 From: lvbaiqing Date: Thu, 19 Nov 2026 23:19:14 +0800 Subject: [PATCH 01/13] Remove legacy Ask UI and isolate the empty Ask window --- .../components/AndroidPermissionsPanel.tsx | 3 +- .../app/android/frontend/lib/androidTypes.ts | 2 +- .../kotlin/OpenLessAndroidPreferences.kt | 8 +- .../app/android/kotlin/OpenLessNative.kt | 1 - .../android/kotlin/OpenLessOverlayService.kt | 18 - .../crates/openless-core/src/android_types.rs | 4 +- .../crates/openless-core/src/shared_types.rs | 4 +- .../openless-core/src/shortcut_types.rs | 13 - openless-all/app/e2e/ask-anything.spec.ts | 102 +- .../app/e2e/aux-window-geometry.spec.ts | 65 - openless-all/app/e2e/auxiliary-smoke.spec.ts | 3 +- .../app/e2e/capsule-reference-motion.spec.ts | 70 - openless-all/app/e2e/capsule-theme.spec.ts | 55 - .../app/e2e/fixtures/capsule-motion.tsx | 54 +- openless-all/app/e2e/fixtures/qa-footer.html | 11 - openless-all/app/e2e/fixtures/qa-footer.tsx | 88 -- .../app/e2e/interaction-polish.spec.ts | 96 -- openless-all/app/e2e/notifications.spec.ts | 2 +- openless-all/app/e2e/openless-default.spec.ts | 15 - openless-all/app/e2e/qa-footer.spec.ts | 84 -- openless-all/app/e2e/refinements.spec.ts | 3 +- openless-all/app/e2e/startup-gate.spec.ts | 39 +- openless-all/app/e2e/switch-motion.spec.ts | 8 - .../app/e2e/visual-refinements.spec.ts | 4 +- openless-all/app/harmony/ipc-audit.json | 95 +- openless-all/app/public/ask-window/index.html | 10 + .../deepseek-read-aloud-runtime.test.mjs | 199 --- .../app/scripts/egui-icons-contract.test.mjs | 4 +- .../app/scripts/qa-panel-runtime.test.mjs | 944 ------------ .../selection-polish-panel-contract.test.mjs | 145 -- .../shared-backend-wire-contract.test.mjs | 12 - ...windows-hang-diagnostics-contract.test.mjs | 2 +- .../app/scripts/windows-ui-config.test.mjs | 2 +- .../backend-tests/tests/mouse_dictation.rs | 2 - .../app/src-tauri/capabilities/default.json | 2 +- .../app/src-tauri/capabilities/mobile.json | 2 +- .../src-tauri/src/android/native_bridge.rs | 21 - .../app/src-tauri/src/ask_window/mod.rs | 24 + .../src-tauri/src/commands/deepseek_tts.rs | 152 -- .../app/src-tauri/src/commands/hotkeys.rs | 2 +- .../app/src-tauri/src/commands/mod.rs | 7 - openless-all/app/src-tauri/src/commands/qa.rs | 179 +-- .../src/commands/selection_polish_preview.rs | 93 -- .../src-tauri/src/commands/selection_voice.rs | 3 + openless-all/app/src-tauri/src/coordinator.rs | 221 +-- .../src/coordinator/dictation_core.rs | 13 - .../src-tauri/src/coordinator/hotkey_loops.rs | 166 +-- .../app/src-tauri/src/coordinator/qa.rs | 50 - .../src/coordinator/restore_runtime.rs | 16 - .../coordinator/selection_voice_session.rs | 3 + .../app/src-tauri/src/core_adapters.rs | 17 +- openless-all/app/src-tauri/src/hotkey.rs | 1 - openless-all/app/src-tauri/src/lib.rs | 476 +----- .../src-tauri/src/mobile_stubs/qa_hotkey.rs | 33 - .../app/src-tauri/src/mouse_dictation.rs | 1 - openless-all/app/src-tauri/src/qa_hotkey.rs | 264 ---- .../app/src-tauri/src/recording_companion.rs | 218 --- .../app/src-tauri/src/side_aware_combo.rs | 18 - openless-all/app/src/api/ask-window.ts | 5 + openless-all/app/src/api/qa.ts | 60 - openless-all/app/src/api/selection.ts | 9 - openless-all/app/src/app/MainWindow.tsx | 19 +- openless-all/app/src/app/androidPanelState.ts | 2 +- openless-all/app/src/app/destinations.ts | 3 - .../app/src/app/shell/FloatingShell.tsx | 12 +- openless-all/app/src/app/shell/navigation.ts | 3 - openless-all/app/src/app/shell/pages.tsx | 6 - openless-all/app/src/app/useAndroidPanels.ts | 27 +- openless-all/app/src/app/windowRoutes.tsx | 10 +- openless-all/app/src/components/QaSurface.tsx | 1 - .../app/src/features/capsule/Capsule.tsx | 48 +- .../input/SelectionWorkspaceSection.tsx | 35 +- .../src/features/input/ShortcutsSection.tsx | 22 - .../app/src/features/qa/DeepSeekReadAloud.tsx | 116 -- .../app/src/features/qa/QaAnswerFooter.tsx | 85 -- openless-all/app/src/features/qa/QaPanel.tsx | 1301 ----------------- .../app/src/features/qa/QaSurface.tsx | 64 - .../app/src/features/qa/chat/chat.css | 1 - .../app/src/features/qa/chat/lifecycle.ts | 44 - openless-all/app/src/features/qa/qa-panel.css | 1165 --------------- .../app/src/features/qa/qaSources.test.ts | 19 - openless-all/app/src/features/qa/qaSources.ts | 27 - .../app/src/features/qa/useQaPipeline.ts | 45 - .../app/src/features/qa/useQaWindowLayout.ts | 77 - .../features/selection-guide/SelectionAsk.css | 16 - .../features/selection-guide/SelectionAsk.tsx | 68 - .../SelectionVoiceIntentPicker.tsx | 12 +- .../settings/sections/AskHistorySection.tsx | 34 + .../src/features/settings/shell/navigation.ts | 6 + .../app/src/features/settings/shell/tabs.tsx | 2 + openless-all/app/src/i18n/de.ts | 2 +- openless-all/app/src/i18n/en.ts | 2 +- openless-all/app/src/i18n/es.ts | 2 +- openless-all/app/src/i18n/fr.ts | 2 +- openless-all/app/src/i18n/ja.ts | 2 +- openless-all/app/src/i18n/ko.ts | 2 +- openless-all/app/src/i18n/zh-CN.ts | 2 +- openless-all/app/src/i18n/zh-TW.ts | 2 +- openless-all/app/src/lib/ipc/hotkeys.ts | 2 +- openless-all/app/src/lib/ipc/index.ts | 20 - openless-all/app/src/lib/ipc/mock-data.ts | 2 +- openless-all/app/src/lib/ipc/qa.ts | 73 - .../src/lib/ipc/selection-polish-preview.ts | 21 - 103 files changed, 179 insertions(+), 7448 deletions(-) delete mode 100644 openless-all/app/e2e/fixtures/qa-footer.html delete mode 100644 openless-all/app/e2e/fixtures/qa-footer.tsx delete mode 100644 openless-all/app/e2e/qa-footer.spec.ts create mode 100644 openless-all/app/public/ask-window/index.html delete mode 100644 openless-all/app/scripts/deepseek-read-aloud-runtime.test.mjs delete mode 100644 openless-all/app/scripts/qa-panel-runtime.test.mjs delete mode 100644 openless-all/app/scripts/selection-polish-panel-contract.test.mjs create mode 100644 openless-all/app/src-tauri/src/ask_window/mod.rs delete mode 100644 openless-all/app/src-tauri/src/commands/deepseek_tts.rs delete mode 100644 openless-all/app/src-tauri/src/commands/selection_polish_preview.rs delete mode 100644 openless-all/app/src-tauri/src/coordinator/qa.rs delete mode 100644 openless-all/app/src-tauri/src/mobile_stubs/qa_hotkey.rs delete mode 100644 openless-all/app/src-tauri/src/qa_hotkey.rs delete mode 100644 openless-all/app/src-tauri/src/recording_companion.rs create mode 100644 openless-all/app/src/api/ask-window.ts delete mode 100644 openless-all/app/src/api/qa.ts delete mode 100644 openless-all/app/src/components/QaSurface.tsx delete mode 100644 openless-all/app/src/features/qa/DeepSeekReadAloud.tsx delete mode 100644 openless-all/app/src/features/qa/QaAnswerFooter.tsx delete mode 100644 openless-all/app/src/features/qa/QaPanel.tsx delete mode 100644 openless-all/app/src/features/qa/QaSurface.tsx delete mode 100644 openless-all/app/src/features/qa/chat/lifecycle.ts delete mode 100644 openless-all/app/src/features/qa/qa-panel.css delete mode 100644 openless-all/app/src/features/qa/qaSources.test.ts delete mode 100644 openless-all/app/src/features/qa/qaSources.ts delete mode 100644 openless-all/app/src/features/qa/useQaPipeline.ts delete mode 100644 openless-all/app/src/features/qa/useQaWindowLayout.ts delete mode 100644 openless-all/app/src/features/selection-guide/SelectionAsk.css delete mode 100644 openless-all/app/src/features/selection-guide/SelectionAsk.tsx create mode 100644 openless-all/app/src/features/settings/sections/AskHistorySection.tsx delete mode 100644 openless-all/app/src/lib/ipc/qa.ts delete mode 100644 openless-all/app/src/lib/ipc/selection-polish-preview.ts diff --git a/openless-all/app/android/frontend/components/AndroidPermissionsPanel.tsx b/openless-all/app/android/frontend/components/AndroidPermissionsPanel.tsx index d728c2d4a..95dee60d8 100644 --- a/openless-all/app/android/frontend/components/AndroidPermissionsPanel.tsx +++ b/openless-all/app/android/frontend/components/AndroidPermissionsPanel.tsx @@ -621,7 +621,7 @@ export function AndroidPermissionsPanel({ mode = 'all' }: AndroidPermissionsPane up: 'cancel', down: 'none', left: 'translation', - right: 'qa', + right: 'none', }), [direction]: action, })); @@ -635,7 +635,6 @@ export function AndroidPermissionsPanel({ mode = 'all' }: AndroidPermissionsPane 'translation', 'style_pack', 'cancel', - 'qa', 'voice_edit', ] as const ).map((action) => ( diff --git a/openless-all/app/android/frontend/lib/androidTypes.ts b/openless-all/app/android/frontend/lib/androidTypes.ts index 14303fc22..b995fc90e 100644 --- a/openless-all/app/android/frontend/lib/androidTypes.ts +++ b/openless-all/app/android/frontend/lib/androidTypes.ts @@ -6,7 +6,7 @@ export type AndroidOverlayActivationMode = 'tap' | 'long_press'; export type AndroidOverlayLeftSwipeAction = 'translation' | 'style_pack'; export type AndroidOverlayCancelSwipeDirection = 'up' | 'down'; export type AndroidOverlayGestureAction = - 'none' | 'quick_note' | 'translation' | 'style_pack' | 'cancel' | 'qa' | 'voice_edit'; + 'none' | 'quick_note' | 'translation' | 'style_pack' | 'cancel' | 'voice_edit'; export interface AndroidOverlayGestureActions { up: AndroidOverlayGestureAction; diff --git a/openless-all/app/android/kotlin/OpenLessAndroidPreferences.kt b/openless-all/app/android/kotlin/OpenLessAndroidPreferences.kt index 4b15c0314..79f0a070b 100644 --- a/openless-all/app/android/kotlin/OpenLessAndroidPreferences.kt +++ b/openless-all/app/android/kotlin/OpenLessAndroidPreferences.kt @@ -25,7 +25,7 @@ object OpenLessAndroidPreferences { private val VALID_OVERLAY_LEFT_SWIPE_ACTIONS = setOf("translation", "style_pack") private val VALID_OVERLAY_CANCEL_SWIPE_DIRECTIONS = setOf("up", "down") private val VALID_OVERLAY_GESTURE_ACTIONS = - setOf("none", "quick_note", "translation", "style_pack", "cancel", "qa", "voice_edit") + setOf("none", "quick_note", "translation", "style_pack", "cancel", "voice_edit") fun overlayTriggerMode(context: Context): String? { val value = readPreferenceString(context, KEY_OVERLAY_TRIGGER) ?: return null @@ -70,7 +70,9 @@ object OpenLessAndroidPreferences { if (!file.isFile) continue try { val actions = JSONObject(file.readText()).optJSONObject(KEY_OVERLAY_GESTURE_ACTIONS) - val value = actions?.optString(direction, "")?.takeIf { + val stored = actions?.optString(direction, "") + if (stored == "qa") return "none" + val value = stored?.takeIf { it in VALID_OVERLAY_GESTURE_ACTIONS } if (value != null) return value @@ -84,7 +86,7 @@ object OpenLessAndroidPreferences { "up" -> if (legacyCancel == "up") "cancel" else "none" "down" -> if (legacyCancel == "down") "cancel" else "none" "left" -> legacyLeft - "right" -> "qa" + "right" -> "none" else -> "none" } } diff --git a/openless-all/app/android/kotlin/OpenLessNative.kt b/openless-all/app/android/kotlin/OpenLessNative.kt index 467904784..17ba7870f 100644 --- a/openless-all/app/android/kotlin/OpenLessNative.kt +++ b/openless-all/app/android/kotlin/OpenLessNative.kt @@ -118,7 +118,6 @@ object OpenLessNative { @JvmStatic external fun nativeSwitchStylePack() - @JvmStatic external fun nativeFinalizeQaFromOverlay() @JvmStatic external fun nativeFinalizeVoiceEditFromOverlay() diff --git a/openless-all/app/android/kotlin/OpenLessOverlayService.kt b/openless-all/app/android/kotlin/OpenLessOverlayService.kt index 473dcb049..2b9c184d0 100644 --- a/openless-all/app/android/kotlin/OpenLessOverlayService.kt +++ b/openless-all/app/android/kotlin/OpenLessOverlayService.kt @@ -658,7 +658,6 @@ class OpenLessOverlayService : Service(), OpenLessOverlayBridge.OverlayStateList "quick_note", "translation", "style_pack" -> applyVisualState(OverlayVisualState.Processing) - "qa", "voice_edit" -> applyVisualState(OverlayVisualState.Processing) "cancel" -> applyVisualState(OverlayVisualState.Error) else -> Unit @@ -674,7 +673,6 @@ class OpenLessOverlayService : Service(), OpenLessOverlayBridge.OverlayStateList switchStylePackFromOverlay() if (recording) stopRecordingFromOverlay() } - "qa" -> finalizeQaFromOverlay() "voice_edit" -> finalizeVoiceEditFromOverlay() "cancel" -> cancelRecordingFromOverlay(direction) else -> Unit @@ -711,22 +709,6 @@ class OpenLessOverlayService : Service(), OpenLessOverlayBridge.OverlayStateList } } - private fun finalizeQaFromOverlay() { - try { - Log.i(TAG, "finalize QA from overlay") - OpenLessNative.nativeFinalizeQaFromOverlay() - recording = false - processing = true - setArmed(false) - applyVisualState(OverlayVisualState.Processing) - } catch (error: Throwable) { - Log.w(TAG, "finalize QA bridge unavailable", error) - processing = false - applyVisualState(OverlayVisualState.Error) - showToast("问答服务未就绪,请打开 OpenLess 后重试") - } - } - private fun finalizeVoiceEditFromOverlay() { if (!recording) return try { diff --git a/openless-all/app/crates/openless-core/src/android_types.rs b/openless-all/app/crates/openless-core/src/android_types.rs index 27f6f274c..523946b72 100644 --- a/openless-all/app/crates/openless-core/src/android_types.rs +++ b/openless-all/app/crates/openless-core/src/android_types.rs @@ -54,12 +54,12 @@ pub enum AndroidOverlayCancelSwipeDirection { #[serde(rename_all = "snake_case")] pub enum AndroidOverlayGestureAction { #[default] + #[serde(alias = "qa")] None, QuickNote, Translation, StylePack, Cancel, - Qa, VoiceEdit, } @@ -78,7 +78,7 @@ impl Default for AndroidOverlayGestureActions { up: AndroidOverlayGestureAction::Cancel, down: AndroidOverlayGestureAction::None, left: AndroidOverlayGestureAction::Translation, - right: AndroidOverlayGestureAction::Qa, + right: AndroidOverlayGestureAction::None, } } } diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index 4e2d34083..6474b56d1 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -170,7 +170,9 @@ pub use crate::types::HistoryInsertStatus as InsertStatus; #[serde(rename_all = "camelCase")] pub enum SelectionPolishOutputMode { #[default] + #[serde(alias = "previewConfirm")] DirectReplace, + #[serde(skip_deserializing)] PreviewConfirm, } @@ -1560,7 +1562,7 @@ impl<'de> Deserialize<'de> for UserPreferences { AndroidOverlayGestureAction::StylePack } }, - right: AndroidOverlayGestureAction::Qa, + right: AndroidOverlayGestureAction::None, }); Ok(Self { diff --git a/openless-all/app/crates/openless-core/src/shortcut_types.rs b/openless-all/app/crates/openless-core/src/shortcut_types.rs index 2fd5e7d4f..6a3248207 100644 --- a/openless-all/app/crates/openless-core/src/shortcut_types.rs +++ b/openless-all/app/crates/openless-core/src/shortcut_types.rs @@ -776,19 +776,6 @@ pub fn reconcile_hotkey_collisions( } pub fn reject_hotkey_collisions(preferences: &UserPreferences) -> Result<(), String> { - let companion = preferences.qa_companion_key.trim(); - if !companion.is_empty() { - let binding = ShortcutBinding { primary: companion.into(), modifiers: Vec::new() }; - validate_shortcut_binding(&binding).map_err(|error| error.to_string())?; - if legacy_modifier_trigger(&binding).is_some() - || companion.eq_ignore_ascii_case("Shift") - || companion.eq_ignore_ascii_case("Escape") - || companion.starts_with("Mouse") - || companion.eq_ignore_ascii_case(&preferences.dictation_hotkey.primary) - { - return Err("提问附加键需选择一个与录音主键不同的普通键,例如 Space".into()); - } - } reject_non_dictation_side_specific_shortcuts(preferences)?; let switch_style = preferences.switch_style_hotkey.as_ref(); let open_app = preferences.open_app_hotkey.as_ref(); diff --git a/openless-all/app/e2e/ask-anything.spec.ts b/openless-all/app/e2e/ask-anything.spec.ts index d9bcd6d7f..d7128b363 100644 --- a/openless-all/app/e2e/ask-anything.spec.ts +++ b/openless-all/app/e2e/ask-anything.spec.ts @@ -69,100 +69,12 @@ test('translation popup closes when the recording owner changes', async ({ page expect(await page.evaluate(() => (window as any).fixture.targetChanges)).toEqual([]); }); -test('Ask opens a conversation on record, streams activity, and keeps a composer', async ({ - page, -}, info) => { - await page.goto(fixture + '?qa'); - await page.locator('.ol-default-capsule').waitFor(); - await expect(page.locator('input')).toHaveCount(0); - await emit(page, 'qa:state', { - kind: 'recording', - sessionId: 'ask-a', - messages: [], - selectionPreview: 'selected source', - }); - await expect(page.locator('.qa-answer-panel')).toBeVisible(); - await expect(page.locator('.qa-recording-selection')).toContainText('selected source'); - await expect(page.locator('#native-host')).toHaveCSS('width', '720px'); - await emit(page, 'qa:state', { kind: 'thinking', sessionId: 'ask-a', messages: [] }); - await emit(page, 'qa:state', { - kind: 'activity', - sessionId: 'ask-a', - activity: { kind: 'searching_web', toolName: 'web_search' }, - }); - await expect(page.locator('.qa-thought')).toContainText('正在搜索网页'); - await emit(page, 'qa:state', { kind: 'answer_delta', sessionId: 'ask-a', chunk: 'partial' }); - await expect(page.locator('.qa-assistant-message')).toContainText('partial'); - await emit(page, 'qa:state', { - kind: 'activity', - sessionId: 'old-ask', - activity: { kind: 'using_tool', toolName: 'stale' }, - }); - await expect(page.locator('.qa-thought')).not.toContainText('stale'); - await emit(page, 'qa:state', { - kind: 'answer', - sessionId: 'ask-a', - messages: [ - { role: 'user', content: '问题', selectionText: 'selected source' }, - { role: 'assistant', content: '最终回答' }, - ], - }); - await expect(page.locator('.qa-user-turn')).toContainText('问题'); - await expect(page.locator('.qa-user-turn')).toContainText('selected source'); - await expect(page.locator('.qa-assistant-message')).toContainText('最终回答'); - await expect(page.locator('.qa-thought')).toContainText('已思考'); - await expect(page.locator('.qa-follow-up')).toHaveCount(0); - await expect(page.locator('.qa-prompt input')).toBeVisible(); - await expect(page.locator('.qa-prompt .qa-close')).toHaveCount(0); - await page.screenshot({ path: info.outputPath('ask-result.png') }); -}); - -test('Stop cancels the current turn and retains the open conversation for a follow-up', async ({ - page, -}) => { - await page.goto(fixture + '?qa'); - await page.locator('.ol-default-capsule').waitFor(); - const firstTurn = [ - { role: 'user', content: 'first question' }, - { role: 'assistant', content: 'first answer' }, - ]; - await emit(page, 'qa:state', { kind: 'answer', sessionId: 'ask-a', messages: firstTurn }); - await expect(page.locator('.qa-assistant-message')).toContainText('first answer'); - await emit(page, 'qa:state', { - kind: 'thinking', - sessionId: 'ask-b', - messages: [...firstTurn, { role: 'user', content: 'follow-up' }], - }); - await emit(page, 'qa:state', { kind: 'answer_delta', sessionId: 'ask-b', chunk: 'draft answer' }); - const stop = page.locator('.qa-send.is-stop'); - await expect(stop).toBeVisible(); - await stop.click(); - await expect - .poll(() => - page.evaluate(() => - (window as any).fixture.qaActions.map((action: { command: string }) => action.command), - ), - ) - .toContain('qa_cancel_turn'); - await emit(page, 'qa:state', { kind: 'cancelled', sessionId: 'ask-b', messages: firstTurn }); - await expect(page.locator('.qa-answer-panel')).toBeVisible(); - await expect(page.locator('.qa-assistant-message')).toContainText('first answer'); - await expect(page.getByText('draft answer')).toHaveCount(0); - await expect(page.locator('.qa-prompt input')).toBeEnabled(); - await expect(stop).toHaveCount(0); -}); - -test('overview is restored and the Ask page remains without the added three-row guide', async ({ - page, -}, info) => { +test('Ask history lives in General instead of the tools menu', async ({ page }) => { await page.goto('/'); - await expect(page.locator('[data-ol-page="overview"]')).toBeVisible(); - await expect(page.locator('.ol-recording-key-guide')).toHaveCount(0); - await page.screenshot({ path: info.outputPath('overview-restored.png') }); - const nav = page.locator('.ol-sidebar-surface nav'); - await nav.getByRole('button', { name: '工具', exact: true }).click(); - await nav.getByRole('button', { name: '随便问', exact: true }).click(); - await expect(page.getByText('提问附加键', { exact: true })).toBeVisible(); - await expect(page.locator('.ol-recording-key-guide')).toHaveCount(0); - await expect(page.getByText('保存历史', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: '设置', exact: true }).click(); + const history = page.locator('[data-ol-general-entry="askHistory"]'); + await expect(page.locator('[data-ol-general-entry="recording"] + [data-ol-general-entry="askHistory"]')).toBeVisible(); + await history.click(); + await expect(page.getByRole('switch', { name: '保存历史' })).toBeVisible(); + await expect(page.getByText('开启后在此设备上保存「随便问」的历史记录,默认关闭。', { exact: true }).first()).toBeVisible(); }); diff --git a/openless-all/app/e2e/aux-window-geometry.spec.ts b/openless-all/app/e2e/aux-window-geometry.spec.ts index 11573df70..452abd388 100644 --- a/openless-all/app/e2e/aux-window-geometry.spec.ts +++ b/openless-all/app/e2e/aux-window-geometry.spec.ts @@ -6,71 +6,6 @@ const fixture = `${fixtureOrigin}/e2e/fixtures/capsule-motion.html`; test.use({ reducedMotion: 'reduce' }); -test('QA waits for observed geometry and reopens without the closed transcript', async ({ - page, -}) => { - await page.goto(fixture + '?qa'); - await expect(page.locator('.ol-default-capsule')).toBeVisible(); - const host = page.locator('#native-host'); - await page.evaluate(() => { - const bridge = (window as any).fixture; - bridge.holdQaGeometry = true; - bridge.applyQaGeometry(false, 320); - bridge.emit('qa:state', { kind: 'recording', sessionId: 'old', messages: [] }); - bridge.emit('qa:state', { - kind: 'answer', - sessionId: 'old', - messages: [{ role: 'assistant', content: 'Old result' }], - } satisfies QaStatePayload); - }); - await expect.poll(() => page.evaluate(() => (window as any).fixture.sizes.at(-1))).toBe(true); - await expect(host).toHaveCSS('height', '80px'); - await expect(page.locator('.qa-answer-panel')).toBeHidden(); - - await page.evaluate(() => (window as any).fixture.applyQaGeometry(true, 320, 220)); - const answer = page.locator('.qa-answer-panel'); - await expect(answer).toBeVisible(); - await expect(answer.getByText('Old result', { exact: true })).toBeVisible(); - await expect(host).toHaveCSS('width', '320px'); - await expect(host).toHaveCSS('height', '220px'); - - await page.evaluate(() => { - const bridge = (window as any).fixture; - bridge.emit('chat-panel:closing', {}); - bridge.emit('chat-panel:shown', {}); - bridge.emit('qa:state', { kind: 'idle' } satisfies QaStatePayload); - }); - await expect.poll(() => page.evaluate(() => (window as any).fixture.sizes.at(-1))).toBe(false); - await expect(page.locator('.ol-default-capsule')).toBeHidden(); - await expect(answer).toHaveCount(0); - await page.evaluate(() => (window as any).fixture.applyQaGeometry(false, 320)); - await expect(page.locator('.ol-default-capsule')).toBeVisible(); -}); - -test('QA keeps a reachable close action when the host rejects expansion', async ({ page }) => { - await page.goto(fixture + '?qa'); - await expect(page.locator('.ol-default-capsule')).toBeVisible(); - await page.evaluate(() => { - const bridge = (window as any).fixture; - bridge.rejectQaExpansion = true; - bridge.emit('qa:state', { kind: 'recording', sessionId: 'failed', messages: [] }); - bridge.emit('qa:state', { - kind: 'answer', - sessionId: 'failed', - messages: [{ role: 'assistant', content: 'Answer' }], - } satisfies QaStatePayload); - }); - const alert = page.locator('.qa-layout-failure'); - await expect(alert).toBeVisible(); - await expect(page.locator('#native-host')).toHaveCSS('height', '80px'); - const close = alert.getByRole('button'); - await expect(close).toBeInViewport(); - await close.click(); - const closing = page.locator('.qa-capsule-shell.is-closing'); - await expect(closing).toHaveAttribute('inert', ''); - await expect(closing).toHaveCSS('opacity', '0'); -}); - test('language menu waits for drawable room and releases its own reservation on Escape', async ({ page, }) => { diff --git a/openless-all/app/e2e/auxiliary-smoke.spec.ts b/openless-all/app/e2e/auxiliary-smoke.spec.ts index aaa4ce291..de9eb89b7 100644 --- a/openless-all/app/e2e/auxiliary-smoke.spec.ts +++ b/openless-all/app/e2e/auxiliary-smoke.spec.ts @@ -6,7 +6,6 @@ test('auxiliary window entries load without blank-page exceptions or startup vid const errors: string[] = []; page.on('pageerror', (error) => errors.push(error.message)); for (const kind of [ - 'qa', 'selection-voice-intent', 'voice-edit', 'less-computer', @@ -19,7 +18,7 @@ test('auxiliary window entries load without blank-page exceptions or startup vid await expect(page.locator('.ol-first-frame')).toHaveCount(0); await expect(page.locator('video')).toHaveCount(0); // Active business surfaces render content; retired agent URLs remain inert. - if (['qa', 'voice-edit'].includes(kind)) { + if (['voice-edit'].includes(kind)) { expect(await page.locator('#root').textContent()).not.toBe(''); } if (['less-computer', 'less-computer-glow'].includes(kind)) { diff --git a/openless-all/app/e2e/capsule-reference-motion.spec.ts b/openless-all/app/e2e/capsule-reference-motion.spec.ts index 20fabb604..fcbea813d 100644 --- a/openless-all/app/e2e/capsule-reference-motion.spec.ts +++ b/openless-all/app/e2e/capsule-reference-motion.spec.ts @@ -135,76 +135,6 @@ test('translation target popup stays interactive during a recording session', as .toBe(false); }); -test('QA opens the conversation while recording and starts fresh after close', async ({ - page, -}, info) => { - const errors: string[] = []; - page.on('pageerror', (error) => errors.push(error.message)); - await page.goto(fixture + '?qa'); - await expect(page.locator('.ol-default-capsule')).toBeVisible(); - await expect(page.locator('.qa-prompt')).toHaveCount(0); - await emit(page, 'qa:state', { kind: 'recording', sessionId: 'a', messages: [] }); - await expect(page.locator('.qa-answer-panel')).toBeVisible(); - await expect(page.locator('.qa-prompt')).toBeVisible(); - const host = page.locator('#native-host'); - await expect(host).toHaveCSS('width', '720px'); - await expect(host).toHaveCSS('height', '380px'); - await emit(page, 'qa:state', { kind: 'thinking', sessionId: 'a', messages: [messages[0]] }); - await expect(page.locator('.qa-thought')).toBeVisible(); - await expect(page.locator('.ol-default-capsule')).toHaveCount(0); - await emit(page, 'qa:state', { kind: 'answer', sessionId: 'a', messages }); - await expect(page.locator('.qa-assistant-message')).toContainText('我会尽力为你解答'); - await expect(host).toHaveCSS('width', '720px'); - await expect(page.locator('.qa-capsule-shell')).toHaveCSS('opacity', '1'); - await page.screenshot({ path: info.outputPath('qa-result.png') }); - await emit(page, 'chat-panel:closing', {}); - await expect(page.locator('.qa-assistant-message')).toContainText('我会尽力为你解答'); - await expect - .poll( - async () => { - const shell = page.locator('.qa-capsule-shell'); - if ((await shell.count()) === 0) return 0; - return shell.evaluate((el) => Number(getComputedStyle(el).opacity)).catch(() => 0); - }, - { intervals: [10, 25, 50] }, - ) - .toBeLessThan(1); - await emit(page, 'chat-panel:shown', {}); - await emit(page, 'qa:state', { kind: 'recording', sessionId: 'b', messages: [] }); - await expect(page.locator('.qa-capsule-shell')).not.toHaveClass(/is-closing/); - await expect(page.locator('.qa-answer-panel')).toBeVisible(); - await expect(page.locator('.qa-assistant-message')).toHaveCount(0); - await expect(host).toHaveCSS('height', '380px'); - expect(errors).toEqual([]); -}); - -test('immediate geometry acknowledgements keep each QA result visible', async ({ page }) => { - await page.goto(fixture + '?qa'); - const compact = page.locator('.ol-default-capsule'); - await expect(compact).toBeVisible(); - for (let index = 0; index < 12; index++) { - const sessionId = `instant-layout-${index}`; - await emit(page, 'qa:state', { kind: 'recording', sessionId, messages: [] }); - await expect(page.locator('.qa-prompt')).toBeVisible(); - await expect(page.locator('.qa-assistant-message')).toHaveCount(0); - const content = `Answer for turn ${index}`; - await emit(page, 'qa:state', { - kind: 'answer', - sessionId, - messages: [messages[0], { role: 'assistant', content }], - }); - const result = page.locator('.qa-assistant-message'); - await expect(result).toContainText(content); - await expect(result).toBeInViewport(); - await page.evaluate(() => { - const bridge = (window as any).fixture; - bridge.emit('chat-panel:closing', {}); - bridge.emit('chat-panel:shown', {}); - }); - await expect(compact).toBeVisible(); - } -}); - test('reduced motion disables sweeps and geometry interpolation', async ({ page }) => { await page.emulateMedia({ reducedMotion: 'reduce' }); await page.goto(fixture); diff --git a/openless-all/app/e2e/capsule-theme.spec.ts b/openless-all/app/e2e/capsule-theme.spec.ts index 2eb52ba9e..1bd972fb3 100644 --- a/openless-all/app/e2e/capsule-theme.spec.ts +++ b/openless-all/app/e2e/capsule-theme.spec.ts @@ -50,61 +50,6 @@ test('a hidden capsule receives theme changes before the next recording', async await expect(page.locator('.ol-default-capsule')).toHaveCSS('color-scheme', 'dark'); }); -test('the ask-anything badge remains readable in both app themes', async ({ page }) => { - await page.goto(fixture + '&full=1'); - await page.waitForFunction(() => (window as any).fixture.hasListener('capsule:state')); - await page.evaluate(() => - (window as any).fixture.emit('capsule:state', { - state: 'recording', - qa: true, - level: 0.2, - capsuleStyle: 'classic', - }), - ); - const badge = page.locator('.ol-qa-capsule-badge'); - await expect(badge).toHaveText('随便问'); - for (const theme of ['light', 'dark'] as const) { - await preference(page, theme); - await expect(badge).toHaveCSS( - 'color', - theme === 'light' ? 'rgb(24, 24, 27)' : 'rgb(244, 247, 251)', - ); - await expect(badge).toHaveCSS( - 'background-color', - theme === 'light' ? 'rgb(250, 250, 250)' : 'rgb(27, 34, 44)', - ); - } -}); - -test('capsule buttons address the active recording mode', async ({ page }) => { - await page.goto(fixture + '&full=1'); - await page.waitForFunction(() => (window as any).fixture.hasListener('capsule:state')); - const pill = page.locator('.ol-default-capsule'); - const sendState = (qa: boolean, sessionId: string) => - page.evaluate((payload) => (window as any).fixture.emit('capsule:state', payload), { - state: 'recording', - qa, - sessionId, - capsuleStyle: 'classic', - }); - await sendState(true, 'question'); - await expect(page.locator('.ol-qa-capsule-badge')).toHaveAttribute('data-visible', 'true'); - await pill.getByRole('button', { name: '胶囊确认插入', exact: true }).click(); - await pill.getByRole('button', { name: '取消', exact: true }).click(); - const actions = () => page.evaluate(() => (window as any).fixture.dictationActions); - await expect.poll(actions).toEqual(['qa_stop_recording', 'qa_cancel_turn']); - await sendState(false, 'dictation'); - await expect(page.locator('.ol-qa-capsule-badge')).toHaveCount(0); - await pill.getByRole('button', { name: '胶囊确认插入', exact: true }).click(); - await pill.getByRole('button', { name: '取消', exact: true }).click(); - await expect.poll(actions).toEqual([ - 'qa_stop_recording', - 'qa_cancel_turn', - 'stop_dictation', - 'cancel_dictation', - ]); -}); - test('late startup reads cannot overwrite a newer capsule theme', async ({ page }) => { await page.goto(fixture + '&holdThemeRead=1'); await page.waitForFunction(() => (window as any).fixture.themeReads >= 1); diff --git a/openless-all/app/e2e/fixtures/capsule-motion.tsx b/openless-all/app/e2e/fixtures/capsule-motion.tsx index 534348717..05f52d947 100644 --- a/openless-all/app/e2e/fixtures/capsule-motion.tsx +++ b/openless-all/app/e2e/fixtures/capsule-motion.tsx @@ -2,7 +2,6 @@ import React, { useState } from 'react'; import { createRoot } from 'react-dom/client'; import type { CapsuleState, ThemeMode } from '../../src/lib/types'; -import type { QaStatePayload } from '../../src/api/contracts'; import '../../src/styles/tokens.css'; import '../../src/styles/global.css'; import { AppViewport } from '../../src/ui/layout/AppViewport'; @@ -11,9 +10,8 @@ const callbacks = new Map void>(); const listeners = new Map>(); const listenerTargets = new Map(); let callbackId = 0; -const qa = new URLSearchParams(location.search).has('qa'); const params = new URLSearchParams(location.search); -const windowLabel = params.get('label') ?? (qa ? 'qa' : 'capsule'); +const windowLabel = params.get('label') ?? 'capsule'; const themeSync = params.has('themeSync'); const holdThemeRead = params.has('holdThemeRead'); let releaseThemeRead: (() => void) | undefined; @@ -21,16 +19,8 @@ let themeReadStarted!: () => void; const firstThemeRead = new Promise((resolve) => { themeReadStarted = resolve; }); -let qaSnapshot: QaStatePayload = { kind: 'idle', messages: [] }; const bridge = { emit(event: string, payload: unknown) { - if (event === 'chat-panel:closing') qaSnapshot = { kind: 'idle', messages: [] }; - if (event === 'qa:state') { - const state = payload as QaStatePayload; - // Stream chunks, activity and the host's lightweight idle are not snapshots. - if (state.kind !== 'answer_delta' && state.kind !== 'activity' && state.messages) - qaSnapshot = state; - } for (const id of listeners.get(event) ?? []) callbacks.get(id)?.({ event, id, payload }); }, emitTo(label: string, event: string, payload: unknown) { @@ -44,7 +34,6 @@ const bridge = { inputRegions: [] as any[], regionWrites: [] as any[], dictationActions: [] as string[], - qaActions: [] as { command: string; args: unknown }[], pipelineMode: params.get('pipelineMode') === 'deepseek' ? 'deepseek' : 'traditional', hasListener: (event: string) => Boolean(listeners.get(event)?.size), setCapsule: (_: Record) => {}, @@ -56,19 +45,8 @@ const bridge = { bridge.theme = themeMode; bridge.emit('prefs:changed', { themeMode }); }, - sizes: [] as boolean[], - holdQaGeometry: false, fallbackReports: [] as { presentationId: number; height: number }[], copiedFallback: [] as string[], - rejectQaExpansion: false, - applyQaGeometry(expanded: boolean, width?: number, height?: number) { - const host = document.querySelector('#native-host'); - if (!host) return; - // Match the native window's nominal dimensions; tests can deliver a work-area clamp. - host.style.width = `${width ?? (expanded ? 720 : 480)}px`; - host.style.height = `${height ?? (expanded ? 380 : 80)}px`; - host.style.bottom = expanded ? '48px' : '8px'; - }, target: { sessionId: 'recording-a', targetLanguage: 'English', @@ -96,16 +74,13 @@ window.__TAURI_INTERNALS__ = { callbacks.delete(id); }, async invoke(command: string, args: any) { - if (['qa_cancel_turn', 'qa_stop_recording', 'qa_submit_text'].includes(command)) { - bridge.qaActions.push({ command, args }); - } if (command === 'set_capsule_interactive_regions') { bridge.inputRegions = args.rects; bridge.regionWrites.push(args.rects); return; } if ( - ['stop_dictation', 'cancel_dictation', 'qa_stop_recording', 'qa_cancel_turn'].includes( + ['stop_dictation', 'cancel_dictation'].includes( command, ) ) { @@ -185,14 +160,6 @@ window.__TAURI_INTERNALS__ = { bridge.copiedFallback.push(args.text); return; } - if (command === 'qa_get_snapshot') return structuredClone(qaSnapshot); - if (command === 'qa_window_set_expanded') { - bridge.sizes.push(args.expanded); - if (args.expanded && bridge.rejectQaExpansion) throw new Error('native resize failed'); - if (!bridge.holdQaGeometry) bridge.applyQaGeometry(args.expanded); - return; - } - if (command === 'qa_window_dismiss') bridge.emit('chat-panel:closing', {}); return null; }, }; @@ -203,7 +170,6 @@ const { InsertFallbackCard } = await import('../../src/features/capsule/InsertFa const { OpenLessPill } = await import('../../src/features/capsule/OpenLessCapsule'); const { CapsuleTranslationBadge, CapsuleTranscriptOverlay } = await import('../../src/features/capsule/LiveTranscriptPill'); -const { QaPanel } = await import('../../src/features/qa/QaPanel'); const { Capsule } = await import('../../src/features/capsule/Capsule'); if (themeSync) { localStorage.setItem('ol.theme', 'light'); @@ -273,22 +239,6 @@ createRoot(document.getElementById('root')!).render( > - ) : qa ? ( -
- - - -
) : ( diff --git a/openless-all/app/e2e/fixtures/qa-footer.html b/openless-all/app/e2e/fixtures/qa-footer.html deleted file mode 100644 index d980c9632..000000000 --- a/openless-all/app/e2e/fixtures/qa-footer.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - QA footer - - -
- - - diff --git a/openless-all/app/e2e/fixtures/qa-footer.tsx b/openless-all/app/e2e/fixtures/qa-footer.tsx deleted file mode 100644 index c473a583a..000000000 --- a/openless-all/app/e2e/fixtures/qa-footer.tsx +++ /dev/null @@ -1,88 +0,0 @@ -import React, { useState } from 'react'; -import { createRoot } from 'react-dom/client'; -import { createInstance } from 'i18next'; -import { I18nextProvider } from 'react-i18next'; -import '../../src/styles/tokens.css'; -import '../../src/styles/global.css'; -import '../../src/features/qa/qa-panel.css'; -const tauriConfig = await import('../../src-tauri/tauri.conf.json'); -const policy = document.createElement('meta'); -policy.httpEquiv = 'Content-Security-Policy'; -policy.content = `img-src ${tauriConfig.default.app.security.csp['img-src']}`; -document.head.appendChild(policy); -const i18n = createInstance(); -await i18n.init({ - lng: 'zh-CN', - resources: { 'zh-CN': { translation: { common: { copy: '复制' }, qa: { sources: '来源' } } } }, -}); -const playback: Array<{ command: string; requestId: string }> = []; -let resumeAudio: (() => void) | undefined; -let finishAudio: (() => void) | undefined; -let holdResume = false; -Object.assign(window, { - qaFooterFixture: { - playback, - holdResume: () => { - holdResume = true; - }, - releaseResume: () => resumeAudio?.(), - finish: () => finishAudio?.(), - }, - AudioContext: class { - currentTime = 0; - resume() { - if (!holdResume) return Promise.resolve(); - return new Promise((resolve) => { - resumeAudio = resolve; - }); - } - close() { - return Promise.resolve(); - } - }, - __TAURI_INTERNALS__: { - transformCallback: () => 1, - unregisterCallback: () => undefined, - async invoke(command: string, args?: { requestId: string }) { - if (command === 'get_startup_snapshot') - return { contractVersion: '2.0.0', backend: { running: true } }; - if (command === 'deepseek_tts_start' || command === 'deepseek_tts_stop') { - playback.push({ command, requestId: args!.requestId }); - if (command === 'deepseek_tts_start') - await new Promise((resolve) => { - finishAudio = resolve; - }); - } - }, - }, -}); -const { QaAnswerFooter } = await import('../../src/features/qa/QaAnswerFooter'); -function Fixture() { - const [mode, setMode] = useState('traditional'); - return ( - -
-
- {['traditional', 'multimodal', 'deepseek'].map((value) => ( - - ))} -
-
-

这是回答的第一段。

-

这是回答的最后一段。

- -
-
-
- ); -} -createRoot(document.getElementById('root')!).render(); diff --git a/openless-all/app/e2e/interaction-polish.spec.ts b/openless-all/app/e2e/interaction-polish.spec.ts index f36bc7cd4..33c8f30b3 100644 --- a/openless-all/app/e2e/interaction-polish.spec.ts +++ b/openless-all/app/e2e/interaction-polish.spec.ts @@ -85,99 +85,3 @@ test('message follow yields to the reader and resumes at the bottom', async ({ p }); await expect.poll(distance).toBeLessThanOrEqual(1); }); - -test('QA jump control appears only more than 120 pixels from the latest message', async ({ - page, -}) => { - await page.goto(`${fixtureOrigin}/e2e/fixtures/capsule-motion.html?qa`); - await page.locator('.ol-default-capsule').waitFor(); - await page.evaluate(() => { - const fixture = (window as any).fixture; - fixture.emit('qa:state', { kind: 'recording', sessionId: 'follow-test', messages: [] }); - fixture.emit('qa:state', { - kind: 'answer', - sessionId: 'follow-test', - messages: Array.from({ length: 12 }, (_, index) => ({ - role: index % 2 ? 'assistant' : 'user', - content: `Message ${index}: ${'content '.repeat(30)}`, - })), - }); - }); - const viewport = page.locator('[data-slot="message-scroller-viewport"]'); - await expect(viewport).toBeVisible(); - const jump = page.getByRole('button', { name: '跳至最新' }); - await expect(jump).toHaveText('跳至最新'); - await expect - .poll(() => viewport.evaluate((node) => node.scrollHeight - node.clientHeight)) - .toBeGreaterThan(150); - await viewport.evaluate((node) => { - node.scrollTop = node.scrollHeight; - node.dispatchEvent(new Event('scroll')); - }); - await expect(jump).toHaveAttribute('data-active', 'false'); - await viewport.evaluate((node) => { - node.dispatchEvent(new Event('touchmove')); - node.scrollTop = node.scrollHeight - node.clientHeight - 80; - node.dispatchEvent(new Event('scroll')); - }); - await expect(jump).toHaveAttribute('data-active', 'false'); - await viewport.evaluate((node) => { - node.scrollTop = node.scrollHeight - node.clientHeight - 140; - node.dispatchEvent(new Event('scroll')); - }); - await expect(jump).toHaveAttribute('data-active', 'true'); - await jump.click(); - await expect - .poll(() => viewport.evaluate((node) => node.scrollHeight - node.clientHeight - node.scrollTop)) - .toBeLessThanOrEqual(1); -}); - -for (const platform of ['mac', 'win'] as const) { - test(`QA header and scrolling follow ${platform} conventions`, async ({ page }) => { - await page.addInitScript((platform) => { - const value = platform === 'mac' ? 'MacIntel' : 'Win32'; - Object.defineProperty(navigator, 'platform', { configurable: true, value }); - Object.defineProperty(navigator, 'userAgent', { configurable: true, value }); - Object.defineProperty(navigator, 'userAgentData', { configurable: true, value: undefined }); - }, platform); - await page.goto(`${fixtureOrigin}/e2e/fixtures/capsule-motion.html?qa`); - await page.locator('.ol-default-capsule').waitFor(); - await page.evaluate(() => { - const fixture = (window as any).fixture; - fixture.emit('qa:state', { kind: 'recording', sessionId: 'chrome', messages: [] }); - fixture.emit('qa:state', { - kind: 'answer', - sessionId: 'chrome', - messages: Array.from({ length: 12 }, (_, index) => ({ - role: index % 2 ? 'assistant' : 'user', - content: `Message ${index}: ${'content '.repeat(30)}`, - })), - }); - }); - const panel = page.locator('.qa-answer-panel'); - const header = panel.locator('.qa-answer-header'); - await expect(panel).toHaveAttribute('data-platform', platform); - await expect(header.locator('h1')).toContainText('Message 0:'); - await expect(header).toHaveCSS('background-color', 'rgb(255, 255, 255)'); - await expect(header.locator('.qa-symbol')).toHaveCount(0); - const headerBox = (await header.boundingBox())!; - const closeBox = (await header.locator('.qa-close').boundingBox())!; - expect( - platform === 'mac' - ? closeBox.x - headerBox.x - : headerBox.x + headerBox.width - closeBox.x - closeBox.width, - ).toBeLessThan(25); - const viewport = panel.locator('[data-slot="message-scroller-viewport"]'); - await expect(viewport).toHaveCSS('scrollbar-width', 'none'); - await expect - .poll(() => viewport.evaluate((el) => el.scrollHeight - el.clientHeight)) - .toBeGreaterThan(150); - await viewport.evaluate((el) => { - el.scrollTop = 0; - }); - expect(await viewport.evaluate((el) => el.scrollTop)).toBe(0); - expect(await header.evaluate((el) => getComputedStyle(el, '::after').backdropFilter)).toContain( - 'blur', - ); - }); -} diff --git a/openless-all/app/e2e/notifications.spec.ts b/openless-all/app/e2e/notifications.spec.ts index 72a715f94..f97951efb 100644 --- a/openless-all/app/e2e/notifications.spec.ts +++ b/openless-all/app/e2e/notifications.spec.ts @@ -118,7 +118,7 @@ test('save bridge and independent in-flight saves keep their own lifecycle', asy await expect(page.getByText('Second saving')).toBeVisible(); }); -for (const kind of ['qa', 'voice-edit', 'selection-voice-intent']) { +for (const kind of ['voice-edit', 'selection-voice-intent']) { test(`auxiliary ${kind} window has its own notification host`, async ({ page }) => { await page.goto(`/?window=${kind}&os=mac`); await expect(page.locator('.ol-notification-center')).toHaveCount(1); diff --git a/openless-all/app/e2e/openless-default.spec.ts b/openless-all/app/e2e/openless-default.spec.ts index bb8399458..85f43f363 100644 --- a/openless-all/app/e2e/openless-default.spec.ts +++ b/openless-all/app/e2e/openless-default.spec.ts @@ -200,18 +200,3 @@ test('rapid page choices settle immediately and reduced motion leaves no animati })), ).toEqual({ inert: false, animations: 0 }); }); - -test('quick follow-up uses white surfaces and blue accents even in dark mode', async ({ - page, -}, info) => { - await page.goto('/?window=qa'); - const shell = page.locator('.qa-capsule-shell'); - await expect(shell).toBeVisible(); - expect( - await shell.evaluate((el) => ({ - paper: getComputedStyle(el).getPropertyValue('--qa-paper').trim(), - accent: getComputedStyle(el).getPropertyValue('--qa-accent').trim(), - })), - ).toEqual({ paper: '#fff', accent: '#2563eb' }); - await page.screenshot({ path: info.outputPath('quick-followup.png'), animations: 'disabled' }); -}); diff --git a/openless-all/app/e2e/qa-footer.spec.ts b/openless-all/app/e2e/qa-footer.spec.ts deleted file mode 100644 index eaa7c9dc1..000000000 --- a/openless-all/app/e2e/qa-footer.spec.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { fixtureOrigin } from './support/server'; -test.use({ baseURL: fixtureOrigin }); -test('sources sit left, actions sit below the article on the right, and only suite mode offers read aloud', async ({ - page, -}) => { - await page.route('https://icons.duckduckgo.com/**', (route) => - route.fulfill({ - status: 200, - contentType: 'image/svg+xml', - body: '', - }), - ); - await page.goto('/e2e/fixtures/qa-footer.html'); - const read = page.getByRole('button', { name: 'DeepSeek 朗读', exact: true }); - await expect(read).toHaveCount(0); - await page.getByRole('button', { name: 'multimodal', exact: true }).click(); - await expect(read).toHaveCount(0); - await page.getByRole('button', { name: 'deepseek', exact: true }).click(); - await expect(read).toBeVisible(); - const footer = page.locator('.qa-answer-footer'); - await expect(footer.locator('img')).toHaveCount(2); - await expect - .poll(() => - footer - .locator('img') - .evaluateAll((images) => - images.every((image) => (image as HTMLImageElement).naturalWidth > 0), - ), - ) - .toBe(true); - const boxes = await page.evaluate(() => { - const rect = (selector: string) => - document.querySelector(selector)!.getBoundingClientRect().toJSON(); - return { - paragraph: rect('article p:last-of-type'), - footer: rect('.qa-answer-footer'), - sources: rect('.qa-answer-sources'), - copy: rect('.qa-answer-actions button:first-child'), - read: rect('.qa-answer-actions button:last-child'), - }; - }); - expect(boxes.footer.top).toBeGreaterThanOrEqual(boxes.paragraph.bottom); - expect(boxes.sources.right).toBeLessThanOrEqual(boxes.copy.left); - expect(boxes.copy.left).toBeLessThan(boxes.read.left); - await expect(footer).not.toHaveCSS('animation-name', 'none'); - await page.emulateMedia({ reducedMotion: 'reduce' }); - await expect(footer).toHaveCSS('animation-name', 'none'); - await page.screenshot({ path: test.info().outputPath('qa-footer.png') }); -}); - -test('stop and pipeline changes cancel the native speech request by ID', async ({ page }) => { - await page.goto('/e2e/fixtures/qa-footer.html'); - await page.getByRole('button', { name: 'deepseek', exact: true }).click(); - const read = page.getByRole('button', { name: 'DeepSeek 朗读', exact: true }); - const stop = page.getByRole('button', { name: '停止朗读', exact: true }); - const actions = () => page.evaluate(() => (window as any).qaFooterFixture.playback); - await read.click(); - await expect(stop).toBeVisible(); - await expect - .poll(async () => (await actions()).map(({ command }) => command)) - .toEqual(['deepseek_tts_start']); - await page.getByRole('button', { name: 'traditional', exact: true }).click(); - await expect(stop).toHaveCount(0); - await expect - .poll(async () => (await actions()).map(({ command }) => command)) - .toEqual(['deepseek_tts_start', 'deepseek_tts_stop']); - const first = await actions(); - expect(first[1].requestId).toBe(first[0].requestId); - await page.evaluate(() => { - (window as any).qaFooterFixture.finish(); - (window as any).qaFooterFixture.holdResume(); - }); - await page.getByRole('button', { name: 'deepseek', exact: true }).click(); - await read.click(); - await stop.click(); - await page.evaluate(() => (window as any).qaFooterFixture.releaseResume()); - await expect(read).toBeVisible(); - await expect - .poll(async () => (await actions()).map(({ command }) => command)) - .toEqual(['deepseek_tts_start', 'deepseek_tts_stop', 'deepseek_tts_stop']); - const all = await actions(); - expect(all[2].requestId).not.toBe(all[0].requestId); -}); diff --git a/openless-all/app/e2e/refinements.spec.ts b/openless-all/app/e2e/refinements.spec.ts index 13e17021f..365c4778d 100644 --- a/openless-all/app/e2e/refinements.spec.ts +++ b/openless-all/app/e2e/refinements.spec.ts @@ -215,7 +215,6 @@ const styleDestinations = [ ] as const; const toolDestinations = [ ['纠正规则', 'corrections'], - ['随便问', 'selectionAsk'], ['翻译', 'translation'], ['速记', 'quickNote'], ] as const; @@ -265,7 +264,7 @@ for (const layout of ['desktop', 'narrow'] as const) { const tools = nav.getByRole('button', { name: '工具', exact: true }); await tools.click(); await expect(tools).toHaveAttribute('aria-expanded', 'true'); - await expect(nav.getByRole('button', { name: /^(纠正规则|随便问|翻译|速记)$/ })).toHaveText( + await expect(nav.getByRole('button', { name: /^(纠正规则|翻译|速记)$/ })).toHaveText( toolDestinations.map(([name]) => name), ); for (const [name, id] of toolDestinations) await visit(nav, name, id); diff --git a/openless-all/app/e2e/startup-gate.spec.ts b/openless-all/app/e2e/startup-gate.spec.ts index a05d1962c..b45447099 100644 --- a/openless-all/app/e2e/startup-gate.spec.ts +++ b/openless-all/app/e2e/startup-gate.spec.ts @@ -73,45 +73,18 @@ test('the optional-probe fallback never bypasses a failed backend contract hands await expect(page.getByRole('heading')).toHaveCount(0); }); -test('Android keeps one panel through failed and superseded dismissals, then Back closes it', async ({ - page, -}) => { +test('Android Back closes Voice Edit and ignores legacy Ask events', async ({ page }) => { await page.goto(`${fixture}?os=android`); const app = page.getByRole('heading', { name: 'Startup reached the app' }); const voice = page.locator('.ol-voice-edit-window'); await expect(app).toBeVisible(); - await expect - .poll(() => page.evaluate(() => (window as any).startupFixture.listening('qa:state'))) - .toBe(true); - await page.evaluate(() => (window as any).startupFixture.showVoice('old-draft')); + await expect.poll(() => page.evaluate(() => (window as any).startupFixture.listening('voice-edit:show'))).toBe(true); + await page.evaluate(() => (window as any).startupFixture.showVoice('draft')); await expect(voice).toBeVisible(); - await page.evaluate(() => (window as any).startupFixture.emit('qa:state', { kind: 'idle' })); - await expect - .poll(() => page.evaluate(() => (window as any).startupFixture.pendingCancels())) - .toEqual(['old-draft']); - await page.evaluate(() => (window as any).startupFixture.showVoice('new-draft')); - await expect(voice.getByText('new-draft').first()).toBeVisible(); - await page.evaluate(() => (window as any).startupFixture.finishCancel(true)); - await expect(voice.getByText('new-draft').first()).toBeVisible(); - - for (const succeed of [false, true]) { - await page.evaluate(() => (window as any).startupFixture.emit('qa:state', { kind: 'idle' })); - await expect - .poll(() => page.evaluate(() => (window as any).startupFixture.pendingCancels())) - .toEqual(['new-draft']); - await expect(voice).toBeVisible(); - await page.evaluate((result) => (window as any).startupFixture.finishCancel(result), succeed); - if (!succeed) { - await expect(voice.getByRole('alert')).toHaveText('Microphone is still busy'); - } - } - await expect(voice).toHaveCount(0); - await expect(app).toHaveCount(0); - await expect(page.locator('.ol-embedded-panel')).toBeVisible(); await page.evaluate(() => history.back()); + await expect.poll(() => page.evaluate(() => (window as any).startupFixture.pendingCancels())).toEqual(['draft']); + await page.evaluate(() => (window as any).startupFixture.finishCancel(true)); await expect(app).toBeVisible(); - await page.evaluate(() => - (window as any).startupFixture.emit('qa:state', { kind: 'answer', sessionId: 'closed' }), - ); + await page.evaluate(() => (window as any).startupFixture.emit('qa:state', { kind: 'idle' })); await expect(app).toBeVisible(); }); diff --git a/openless-all/app/e2e/switch-motion.spec.ts b/openless-all/app/e2e/switch-motion.spec.ts index cd37e22c0..a7c494a3a 100644 --- a/openless-all/app/e2e/switch-motion.spec.ts +++ b/openless-all/app/e2e/switch-motion.spec.ts @@ -299,14 +299,6 @@ test('channel sorting waits for a slow knob transition to finish', async ({ page await expect(control).toHaveAttribute('data-test-slide', 'transitionend'); }); -test('the Selection Ask history switch slides', async ({ page }) => { - await page.getByRole('button', { name: '工具', exact: true }).click(); - await page.getByRole('button', { name: '随便问', exact: true }).click(); - const main = page.locator('main'); - await expect(main.getByRole('switch').first()).toBeVisible(); - expect(await exerciseVisibleSwitches(main, 'selectionAsk')).toBeGreaterThan(0); -}); - test('toggling a switch never moves its neighbours', async ({ page }) => { const dialog = await openSettings(page); await openMicrophoneSettings(dialog); diff --git a/openless-all/app/e2e/visual-refinements.spec.ts b/openless-all/app/e2e/visual-refinements.spec.ts index d761803ed..0ee58eec5 100644 --- a/openless-all/app/e2e/visual-refinements.spec.ts +++ b/openless-all/app/e2e/visual-refinements.spec.ts @@ -43,8 +43,8 @@ for (const theme of ['dark', 'light'] as const) { if (section === '快捷键') { const rows = dialog.locator('.ol-shortcut-row'); await expect(rows.first()).toBeVisible(); - expect(await rows.count()).toBeGreaterThanOrEqual(9); - await expect(dialog.getByText('随便问', { exact: true })).toBeVisible(); + expect(await rows.count()).toBeGreaterThanOrEqual(8); + await expect(dialog.getByText('随便问', { exact: true })).toHaveCount(0); const bounds = await rows.evaluateAll((elements) => elements.map((element) => { const rect = element.getBoundingClientRect(); diff --git a/openless-all/app/harmony/ipc-audit.json b/openless-all/app/harmony/ipc-audit.json index 1a01e88e2..a4b59379f 100644 --- a/openless-all/app/harmony/ipc-audit.json +++ b/openless-all/app/harmony/ipc-audit.json @@ -1,8 +1,8 @@ { "scope": "Static shared IPC and runtime adapter command coverage; excludes event subscriptions and dynamic command names.", - "total": 197, + "total": 184, "wired": 108, - "missing": 89, + "missing": 76, "commands": [ { "name": "accept_pending_correction", @@ -67,13 +67,6 @@ ], "implementation": "arkts" }, - { - "name": "cancel_selection_polish_preview", - "calledFrom": [ - "src/lib/ipc/selection-polish-preview.ts" - ], - "implementation": "unimplemented" - }, { "name": "cancel_selection_voice_intent_prompt", "calledFrom": [ @@ -298,13 +291,6 @@ ], "implementation": "unimplemented" }, - { - "name": "confirm_selection_polish_preview", - "calledFrom": [ - "src/lib/ipc/selection-polish-preview.ts" - ], - "implementation": "unimplemented" - }, { "name": "confirm_selection_voice_intent_prompt", "calledFrom": [ @@ -508,13 +494,6 @@ ], "implementation": "rust" }, - { - "name": "get_qa_hotkey_label", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, { "name": "get_remote_input_status", "calledFrom": [ @@ -522,13 +501,6 @@ ], "implementation": "unimplemented" }, - { - "name": "get_selection_polish_preview", - "calledFrom": [ - "src/lib/ipc/selection-polish-preview.ts" - ], - "implementation": "unimplemented" - }, { "name": "get_selection_voice_intent_prompt", "calledFrom": [ @@ -837,62 +809,6 @@ ], "implementation": "rust" }, - { - "name": "qa_cancel_turn", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, - { - "name": "qa_get_snapshot", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, - { - "name": "qa_set_edit_instruction_mode", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, - { - "name": "qa_stop_recording", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, - { - "name": "qa_submit_text", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, - { - "name": "qa_toggle_recording", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, - { - "name": "qa_window_dismiss", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, - { - "name": "qa_window_set_expanded", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, { "name": "read_audio_recording", "calledFrom": [ @@ -1152,13 +1068,6 @@ ], "implementation": "unimplemented" }, - { - "name": "set_qa_hotkey", - "calledFrom": [ - "src/lib/ipc/qa.ts" - ], - "implementation": "unimplemented" - }, { "name": "set_quick_note_hotkey", "calledFrom": [ diff --git a/openless-all/app/public/ask-window/index.html b/openless-all/app/public/ask-window/index.html new file mode 100644 index 000000000..477f4f469 --- /dev/null +++ b/openless-all/app/public/ask-window/index.html @@ -0,0 +1,10 @@ + + + + + + + OpenLess — Ask Anything + + + diff --git a/openless-all/app/scripts/deepseek-read-aloud-runtime.test.mjs b/openless-all/app/scripts/deepseek-read-aloud-runtime.test.mjs deleted file mode 100644 index e17b87637..000000000 --- a/openless-all/app/scripts/deepseek-read-aloud-runtime.test.mjs +++ /dev/null @@ -1,199 +0,0 @@ -import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; -import { createRequire } from 'node:module'; -const require = createRequire(import.meta.url); -const ts = require('typescript'); -const source = ts.createSourceFile( - 'DeepSeekReadAloud.tsx', - readFileSync(new URL('../src/features/qa/DeepSeekReadAloud.tsx', import.meta.url), 'utf8'), - ts.ScriptTarget.ES2020, - true, - ts.ScriptKind.TSX, -); -const body = source.statements - .filter((n) => !ts.isImportDeclaration(n)) - .map((n) => n.getFullText(source)) - .join('\n'); -const compiled = ts.transpileModule(body, { - compilerOptions: { - target: ts.ScriptTarget.ES2020, - module: ts.ModuleKind.CommonJS, - jsx: ts.JsxEmit.ReactJSX, - }, -}).outputText; -const factory = new Function( - 'useEffect', - 'useRef', - 'useState', - 'useTranslation', - 'startQaReadAloud', - 'stopQaReadAloud', - 'isTauriHost', - 'AudioContext', - 'crypto', - 'setTimeout', - 'clearTimeout', - 'exports', - 'require', - `${compiled}; return {DeepSeekReadAloud, stopDeepSeekReading};`, -); -function deferred() { - let resolve; - const promise = new Promise((r) => { - resolve = r; - }); - return { promise, resolve }; -} -const tick = () => new Promise((resolve) => setImmediate(resolve)); -function harness({ failAudio = false } = {}) { - const slots = [], - effects = [], - contexts = [], - starts = [], - stops = [], - timers = new Map(); - let cursor = 0, - serial = 0, - nextTimer = 0; - const state = (initial) => { - const index = cursor++; - slots[index] ??= { value: initial }; - return [ - slots[index].value, - (value) => { - slots[index].value = value; - }, - ]; - }; - const effect = (fn, deps) => { - const index = cursor++, - old = slots[index]; - if (old && deps.every((d, i) => Object.is(d, old.deps[i]))) return; - slots[index] = { deps }; - effects.push(() => { - old?.cleanup?.(); - slots[index].cleanup = fn(); - }); - }; - class Audio { - activation = deferred(); - currentTime = 0; - destination = {}; - closed = false; - constructor() { - if (failAudio) throw new Error('audio unavailable'); - contexts.push(this); - } - resume() { - return this.activation.promise; - } - close() { - this.closed = true; - return Promise.resolve(); - } - } - const api = factory( - effect, - (value) => state({ current: value })[0], - state, - () => ({ i18n: { language: 'en' } }), - (text, requestId, onAudio) => { - const completion = deferred(); - starts.push({ text, requestId, onAudio, completion }); - return completion.promise; - }, - (requestId) => { - stops.push(requestId); - return Promise.resolve(); - }, - () => true, - Audio, - { randomUUID: () => `request-${++serial}` }, - (callback) => { - const id = ++nextTimer; - timers.set(id, callback); - return id; - }, - (id) => timers.delete(id), - {}, - () => ({ jsx: (type, props) => ({ type, props }), jsxs: (type, props) => ({ type, props }) }), - ); - const render = (text = 'hello') => { - cursor = 0; - const tree = api.DeepSeekReadAloud({ text }); - effects.splice(0).forEach((fn) => fn()); - return tree; - }; - const button = (text) => render(text).props.children.find((n) => n?.type === 'button'); - const unmount = () => { - slots.forEach((s) => s.cleanup?.()); - }; - return { - api, - render, - click: (text) => button(text).props.onClick(), - unmount, - contexts, - starts, - stops, - timers, - }; -} - -{ - const h = harness(); - const pending = h.click(); - await h.click(); - h.contexts[0].activation.resolve(); - await tick(); - assert.equal(h.starts.length, 0, 'stopped device activation must not start a billed TTS call'); - assert(h.contexts[0].closed); - await pending; - h.unmount(); -} -{ - const h = harness(); - const old = h.click(); - await h.click(); - const next = h.click(); - h.contexts[1].activation.resolve(); - await tick(); - assert.deepEqual( - h.starts.map((s) => s.requestId), - ['request-2'], - ); - h.contexts[0].activation.resolve(); - await old; - assert.deepEqual( - h.starts.map((s) => s.requestId), - ['request-2'], - 'late old activation cannot replace newer speech', - ); - assert(!h.contexts[1].closed); - h.starts[0].completion.resolve(); - await next; - await h.click(); - assert.equal(h.timers.size, 0, 'Stop releases the pending playback completion timer'); - assert(h.stops.includes('request-2')); - h.unmount(); -} -for (const operation of ['unmount', 'replace']) { - const h = harness(); - const pending = h.click(); - if (operation === 'unmount') h.unmount(); - else h.render('replacement answer'); - h.contexts[0].activation.resolve(); - await pending; - assert.equal(h.starts.length, 0, `${operation} cancels pending audio activation`); - h.unmount(); -} -{ - const h = harness({ failAudio: true }); - await h.click(); - assert(h.render().props.children.some((n) => n?.props?.role === 'alert')); - assert.equal(h.starts.length, 0, 'audio initialization failure must not send a request'); - h.unmount(); -} -console.log( - 'DeepSeek read aloud: Stop, replacement, disposal, timer cleanup and device failure passed', -); diff --git a/openless-all/app/scripts/egui-icons-contract.test.mjs b/openless-all/app/scripts/egui-icons-contract.test.mjs index 1be9854a1..69d1313e1 100644 --- a/openless-all/app/scripts/egui-icons-contract.test.mjs +++ b/openless-all/app/scripts/egui-icons-contract.test.mjs @@ -141,12 +141,12 @@ const desktopEntries = propertyValue(desktopTools, 'children'); assert.ok(desktopEntries && ts.isArrayLiteralExpression(desktopEntries), 'Tauri desktop tools'); assert.deepEqual( desktopEntries.elements.map((item) => stringProperty(item, 'id')), - ['corrections', 'selectionAsk', 'translation', 'quickNote'], + ['corrections', 'translation', 'quickNote'], 'Tauri sidebar tools order', ); assert.deepEqual( navArray('MORE_TABS').map((item) => stringProperty(item, 'id')), - ['vocab', 'translation', 'selectionAsk', 'quickNote', 'corrections'], + ['vocab', 'translation', 'quickNote', 'corrections'], 'Tauri compact More sheet order', ); console.log('egui SVG outlines match Tauri; sidebar and compact navigation orders verified'); diff --git a/openless-all/app/scripts/qa-panel-runtime.test.mjs b/openless-all/app/scripts/qa-panel-runtime.test.mjs deleted file mode 100644 index 6bdde5078..000000000 --- a/openless-all/app/scripts/qa-panel-runtime.test.mjs +++ /dev/null @@ -1,944 +0,0 @@ -import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; -import { createRequire } from 'node:module'; -import { resolve } from 'node:path'; -import { pathToFileURL, fileURLToPath } from 'node:url'; -const app = fileURLToPath(new URL('../', import.meta.url)), - require = createRequire(resolve(app, 'package.json')), - ts = require('typescript'); -const helpers = await import(pathToFileURL(resolve(app, 'src/lib/qaMessage.ts'))); -const source = readFileSync(resolve(app, 'src/features/qa/QaPanel.tsx'), 'utf8'); -const parsed = ts.createSourceFile( - 'QaPanel.tsx', - source, - ts.ScriptTarget.ES2020, - true, - ts.ScriptKind.TSX, -); -const names = parsed.statements - .filter(ts.isImportDeclaration) - .flatMap((n) => - n.importClause?.namedBindings && ts.isNamedImports(n.importClause.namedBindings) - ? n.importClause.namedBindings.elements.map((n) => n.name.text) - : [], - ); -const body = parsed.statements - .filter((n) => !ts.isImportDeclaration(n)) - .map((n) => n.getFullText(parsed)) - .join('\n'); -const compiled = ts.transpileModule(body, { - compilerOptions: { - target: ts.ScriptTarget.ES2020, - module: ts.ModuleKind.CommonJS, - jsx: ts.JsxEmit.ReactJSX, - }, -}).outputText; -const factory = new Function( - ...names, - 'exports', - 'require', - `${compiled};return {QaPanel,Composer,MessageRow,ErrorContent};`, -); -function compileModule(file) { - const ast = ts.createSourceFile( - file, - readFileSync(resolve(app, 'src', file), 'utf8'), - ts.ScriptTarget.ES2020, - true, - ); - return ts.transpileModule( - ast.statements - .filter( - (statement) => !ts.isImportDeclaration(statement) && !ts.isExportDeclaration(statement), - ) - .map((statement) => statement.getFullText(ast)) - .join('\n'), - { compilerOptions: { target: ts.ScriptTarget.ES2020, module: ts.ModuleKind.CommonJS } }, - ).outputText; -} -const makeLayout = new Function( - 'useEffect', - 'useRef', - 'useState', - 'useAppViewport', - 'qaWindowSetExpanded', - 'waitForViewportGeometry', - 'exports', - `${compileModule('features/qa/useQaWindowLayout.ts')};return useQaWindowLayout;`, -); -const makeLifecycle = new Function( - 'useEffect', - 'useRef', - 'useState', - 'isTauriHost', - 'listenQaWindowShown', - 'listenQaWindowClosing', - 'exports', - `${compileModule('features/qa/chat/lifecycle.ts')};return useChatPanelLifecycle;`, -); -const makeQaApi = new Function( - 'listenTauriEvent', - 'exports', - `${compileModule('api/qa.ts')};return exports;`, -); -const makeSelectionApi = new Function( - 'listenTauriEvent', - 'exports', - `${compileModule('api/selection.ts')};return exports;`, -); -const tick = () => new Promise((resolve) => setImmediate(resolve)); -const deferred = () => { - let resolve, reject; - const promise = new Promise((a, b) => { - resolve = a; - reject = b; - }); - return { promise, resolve, reject }; -}; -const jsx = (type, props) => ({ type, props: props ?? {} }); -function nodes(tree) { - if (Array.isArray(tree)) return tree.flatMap(nodes); - if (!tree || typeof tree !== 'object' || !tree.props) return []; - return [tree, ...nodes(tree.props.children)]; -} -function find(tree, predicate) { - const node = nodes(tree).find(predicate); - assert(node, 'expected control'); - return node; -} -let active; -class Hooks { - slots = []; - cursor = 0; - pending = []; - state(initial) { - const i = this.cursor++; - this.slots[i] ??= { value: typeof initial === 'function' ? initial() : initial }; - return [ - this.slots[i].value, - (value) => { - this.slots[i].value = typeof value === 'function' ? value(this.slots[i].value) : value; - }, - ]; - } - ref(initial) { - return this.state({ current: initial })[0]; - } - effect(fn, deps) { - const i = this.cursor++, - prev = this.slots[i]; - if ( - prev && - deps && - prev.deps?.length === deps.length && - deps.every((v, i) => Object.is(v, prev.deps[i])) - ) - return; - const slot = { deps }; - this.slots[i] = slot; - this.pending.push(() => { - prev?.cleanup?.(); - slot.cleanup = fn(); - }); - } - render(fn) { - this.cursor = 0; - active = this; - const tree = fn(); - this.pending.splice(0).forEach((fn) => fn()); - return tree; - } - unmount() { - this.slots.forEach((s) => s.cleanup?.()); - } -} -function context(native = true, embedded = false) { - const events = new Map(), - keys = new Set(), - calls = { - resize: [], - submit: [], - submitArgs: [], - snapshot: 0, - mic: 0, - stopMic: 0, - cancelTurn: 0, - stopReading: 0, - mode: [], - preview: [], - polishPreview: 0, - polishConfirm: [], - confirm: [], - revert: [], - dismiss: 0, - close: 0, - }; - const pending = {}; - globalThis.window = { - location: { search: '?window=qa&demo=1' }, - addEventListener: (name, fn) => { - if (name === 'keydown') keys.add(fn); - }, - removeEventListener: (name, fn) => { - if (name === 'keydown') keys.delete(fn); - }, - }; - const imports = Object.fromEntries(names.map((n) => [n, n])); - const boundary = {}; - const listen = async (name, fn) => { - const listeners = events.get(name) ?? new Set(); - listeners.add(fn); - events.set(name, listeners); - const ready = pending.listen?.(name); - if (ready) await ready; - return () => listeners.delete(fn); - }; - const resize = async (next) => { - calls.resize.push(next); - return pending.resize?.(next); - }; - const state = (value) => active.state(value); - const ref = (value) => active.ref(value); - const effect = (fn, deps) => active.effect(fn, deps); - const qaApi = makeQaApi(listen, {}); - Object.assign(imports, qaApi, makeSelectionApi(listen, {}), { - useState: state, - useRef: ref, - useEffect: effect, - useTranslation: () => ({ t: (k) => k }), - useGithubLogin: () => '', - useChatPanelLifecycle: makeLifecycle( - effect, - ref, - state, - () => native, - qaApi.listenQaWindowShown, - qaApi.listenQaWindowClosing, - {}, - ), - useModalActive: () => false, - useQaPipeline: () => 'deepseek', - getRuntimeOS: () => 'mac', - stopDeepSeekReading: () => { - calls.stopReading++; - }, - qaSources: (body) => ({ body, sources: [] }), - qaCancelTurn: async () => { - calls.cancelTurn++; - return pending.cancelTurn?.promise; - }, - qaStopRecording: async () => { - calls.stopMic++; - return pending.stopMic?.promise; - }, - isTauriHost: () => native, - ...helpers, - useQaWindowLayout: makeLayout( - effect, - ref, - state, - () => ({ boundary }), - resize, - async (_boundary, _fits, signal) => { - if (signal.aborted) throw new DOMException('Layout superseded', 'AbortError'); - return {}; - }, - {}, - ), - qaSubmitText: async (text, session) => { - calls.submit.push(text); - calls.submitArgs.push([text, session]); - return pending.submit?.promise; - }, - qaGetSnapshot: async () => { - calls.snapshot++; - return pending.snapshot?.() ?? { kind: 'idle', messages: [] }; - }, - getSelectionPolishPreview: async () => { - calls.polishPreview++; - return pending.polishPreview?.() ?? null; - }, - qaToggleRecording: async () => { - calls.mic++; - return pending.mic?.promise; - }, - qaSetEditInstructionMode: async (mode) => { - calls.mode.push(mode); - return pending.mode?.promise; - }, - qaWindowDismiss: async () => { - calls.dismiss++; - return pending.dismiss?.promise; - }, - confirmSelectionPolishPreview: async (text) => { - calls.polishConfirm.push(text); - return pending.polishConfirm?.promise; - }, - cancelSelectionPolishPreview: async () => {}, - getSelectionVoicePreview: async (session) => { - calls.preview.push(session); - return pending.preview?.promise ?? { text: ' fixture edit ' }; - }, - confirmSelectionVoicePreview: async (...args) => { - calls.confirm.push(args); - return pending.confirm?.promise; - }, - revertSelectionVoicePreview: async (session) => { - calls.revert.push(session); - return pending.revert?.promise; - }, - chatPanelFocusKeyboard: async () => {}, - }); - const api = factory(...names.map((n) => imports[n]), {}, () => ({ - jsx, - jsxs: jsx, - Fragment: 'fragment', - })); - const hooks = new Hooks(), - render = () => - hooks.render(() => api.QaPanel({ embedded, onRequestClose: () => calls.close++ })); - const emit = (name, payload) => { - for (const fn of events.get(name) ?? []) fn(payload); - }; - const composer = (tree) => find(tree, (n) => n.type === api.Composer).props; - const closePanel = () => - find( - render(), - (n) => n.type === 'button' && n.props['aria-label'] === 'common.close', - ).props.onClick(); - const beginConversation = (sessionId = 'open') => { - emit('qa:state', { - kind: 'answer', - sessionId, - messages: [ - { role: 'user', content: 'First question' }, - { role: 'assistant', content: 'First answer' }, - ], - }); - render(); - }; - return { - api, - hooks, - render, - emit, - calls, - pending, - events, - keys, - composer, - closePanel, - beginConversation, - }; -} -let passed = 0, - failed = 0; -async function test(name, fn) { - try { - await fn(); - console.log('PASS ' + name); - passed++; - } catch (error) { - console.log('FAIL ' + name + '\n' + error.stack); - failed++; - } -} -await test('browser compact has no demo conversations and cannot execute native actions', async () => { - const compact = context(false); - const tree = compact.render(); - assert(!tree.props.className.includes('is-expanded')); - assert( - !nodes(tree).some((n) => n.type === compact.api.Composer || n.type === compact.api.MessageRow), - ); - compact.hooks.unmount(); - const c = context(false, true); - let p = c.composer(c.render()); - p.onChange('fixture'); - p = c.composer(c.render()); - await p.onSubmit(); - await p.onToggleRecording(); - await p.onEditInstructionModeChange(true); - await tick(); - assert.deepEqual(c.calls.resize, []); - assert.deepEqual(c.calls.submit, []); - assert.equal(c.calls.mic, 0); - assert.deepEqual(c.calls.mode, []); - const h = new Hooks(), - ui = h.render(() => c.api.Composer(p)); - for (const button of nodes(ui).filter((n) => n.type === 'button')) - assert.equal(button.props.disabled, true); - c.hooks.unmount(); -}); -await test('polish preview expands native window after payload arrives', async () => { - const c = context(); - c.render(); - await tick(); - assert.deepEqual(c.calls.resize, [false]); - c.pending.polishPreview = () => ({ text: 'polished result', sourceText: 'source text' }); - c.emit('selection-polish-preview:shown', {}); - await tick(); - const tree = c.render(); - await tick(); - assert.deepEqual(c.calls.resize, [false, true]); - assert(nodes(tree).some((n) => n.props.className === 'qa-polish-text')); - assert( - nodes(tree).some( - (n) => - n.type === 'button' && - Array.isArray(n.props.children) && - n.props.children.includes('selectionPolishPreview.confirmReplace'), - ), - ); - c.hooks.unmount(); -}); -await test('polish confirmation submits edited text and retains it after a failed insert', async () => { - const c = context(); - c.pending.polishPreview = () => ({ text: 'model result', sourceText: 'source text' }); - c.render(); - await tick(); - const editor = () => - find(c.render(), (n) => n.type === 'textarea' && n.props.className === 'qa-polish-text').props; - const confirm = () => - find( - c.render(), - (n) => - n.type === 'button' && - Array.isArray(n.props.children) && - n.props.children.includes('selectionPolishPreview.confirmReplace'), - ).props; - assert.equal(editor().value, 'model result'); - assert.equal(editor().readOnly, false); - editor().onChange({ currentTarget: { value: ' \n ' } }); - assert.equal(confirm().disabled, true); - confirm().onClick(); - assert.deepEqual(c.calls.polishConfirm, []); - const edited = ' 用户修改后的文字\n第二行🙂 '; - editor().onChange({ currentTarget: { value: edited } }); - assert.equal(editor().value, edited); - assert.equal(confirm().disabled, false); - const first = deferred(); - c.pending.polishConfirm = first; - confirm().onClick(); - assert.deepEqual(c.calls.polishConfirm, [edited]); - assert.equal(editor().readOnly, true); - assert.equal(confirm().disabled, true); - first.reject(new Error('insert failed')); - await tick(); - assert.equal(editor().value, edited); - assert.equal(editor().readOnly, false); - assert.equal(c.calls.dismiss, 0); - c.pending.polishConfirm = undefined; - confirm().onClick(); - await tick(); - assert.deepEqual(c.calls.polishConfirm, [edited, edited]); - assert.equal( - nodes(c.render()).some((n) => n.props.className === 'qa-polish-text'), - false, - ); - assert.equal(c.calls.dismiss, 1); - c.hooks.unmount(); -}); -await test('an open conversation supports follow-up text, excludes repeats and preserves a new draft', async () => { - const c = context(); - c.render(); - await tick(); - c.beginConversation('answer'); - await tick(); - c.composer(c.render()).onChange('question'); - const p = c.composer(c.render()), - send = deferred(); - c.pending.submit = send; - const first = p.onSubmit(); - void p.onSubmit(); - c.render(); - await tick(); - assert.deepEqual(c.calls.resize, [false, true]); - assert.deepEqual(c.calls.submit, ['question']); - c.composer(c.render()).onChange('next draft'); - send.resolve(); - await first; - assert.equal(c.composer(c.render()).value, 'next draft'); - c.emit('qa:state', { - kind: 'answer', - sessionId: 'answer', - messages: [ - { role: 'user', content: 'question' }, - { role: 'assistant', content: 'answer' }, - ], - }); - c.render(); - await tick(); - assert.deepEqual(c.calls.resize, [false, true]); - c.hooks.unmount(); -}); -await test('send failure preserves draft and exposes error without a fake user message', async () => { - const c = context(true, true); - c.render(); - await tick(); - c.composer(c.render()).onChange('keep this'); - const d = deferred(); - c.pending.submit = d; - const result = c.composer(c.render()).onSubmit(); - await tick(); - d.reject(new Error('fixture')); - await result; - const tree = c.render(); - assert.equal(c.composer(tree).value, 'keep this'); - assert.equal(c.composer(tree).status, 'error'); - assert(!nodes(tree).some((n) => n.type === c.api.MessageRow)); - c.hooks.unmount(); -}); -await test('cancel during resize prevents queued question and late layout failures do not overwrite reopened compact state', async () => { - const c = context(); - const d = deferred(); - c.render(); - await tick(); - c.pending.resize = (expanded) => (expanded ? d.promise : Promise.resolve()); - c.beginConversation(); - c.composer(c.render()).onChange('do not send'); - const submission = c.composer(c.render()).onSubmit(); - c.render(); - await tick(); - await c.closePanel(); - c.emit('chat-panel:closing', {}); - c.render(); - c.emit('chat-panel:shown', {}); - c.render(); - assert.deepEqual(c.calls.resize, [false, true]); - c.pending.resize = undefined; - d.reject(new Error('old geometry')); - await submission; - await tick(); - c.render(); - await tick(); - assert.deepEqual(c.calls.submit, []); - assert.deepEqual(c.calls.resize, [false, true, false]); - assert(!c.render().props.className.includes('is-expanded')); - assert(!nodes(c.render()).some((n) => n.props.className === 'qa-layout-failure')); - c.hooks.unmount(); -}); -await test('recording level requires active session; no fake recording before backend phase', async () => { - const c = context(true, true); - c.render(); - await tick(); - const d = deferred(); - c.pending.mic = d; - let p = c.composer(c.render()); - const first = p.onToggleRecording(); - void p.onToggleRecording(); - assert.equal(c.calls.mic, 1); - p = c.composer(c.render()); - assert.equal(p.status, 'idle'); - assert.equal(p.micBusy, true); - assert.equal(p.level, 0); - c.emit('qa:state', { - kind: 'recording', - sessionId: 'a', - selectionPreview: 'selected', - messages: [], - }); - c.emit('qa:level', { sessionId: 'other', level: 0.9 }); - assert.equal(c.composer(c.render()).level, 0); - c.emit('qa:level', { sessionId: 'a', level: 0.6 }); - p = c.composer(c.render()); - assert.equal(p.level, 0.6); - assert.equal(p.status, 'recording'); - assert.equal(p.voiceActive, true); - assert.equal(p.selectionPreview, 'selected'); - d.resolve(); - await first; - c.emit('qa:state', { - kind: 'thinking', - sessionId: 'a', - messages: [{ role: 'user', content: 'actual question' }], - }); - c.emit('qa:level', { sessionId: 'a', level: 0.9 }); - p = c.composer(c.render()); - assert.equal(p.level, 0); - assert.equal(p.voiceActive, false); - c.hooks.unmount(); -}); -await test('stale answer is ignored and final answer replaces streaming buffer exactly once', async () => { - const c = context(true, true); - c.render(); - await tick(); - c.emit('qa:state', { - kind: 'thinking', - sessionId: 'new', - messages: [{ role: 'user', content: 'question' }], - }); - c.emit('qa:state', { kind: 'answer_delta', sessionId: 'old', chunk: 'stale' }); - c.emit('qa:state', { kind: 'answer_delta', sessionId: 'new', chunk: 'real ' }); - c.emit('qa:state', { kind: 'answer_delta', sessionId: 'new', chunk: 'answer' }); - let tree = c.render(); - assert.equal(find(tree, (n) => n.type === 'AssistantMarkdown').props.markdown, 'real answer'); - c.emit('qa:state', { - kind: 'answer', - sessionId: 'new', - messages: [ - { role: 'user', content: 'question' }, - { role: 'assistant', content: 'real answer' }, - ], - }); - tree = c.render(); - assert(!nodes(tree).some((n) => n.type === 'AssistantMarkdown')); - assert.equal(nodes(tree).filter((n) => n.type === c.api.MessageRow).length, 2); - c.hooks.unmount(); -}); -await test('Stop belongs to the presented turn, and repeated clicks cannot cancel a later turn', async () => { - const c = context(true, true); - c.render(); - await tick(); - c.beginConversation('first'); - const firstTurn = [ - { role: 'user', content: 'First question' }, - { role: 'assistant', content: 'First answer' }, - ]; - c.emit('qa:state', { kind: 'thinking', sessionId: 'second', messages: firstTurn }); - const stop = c.composer(c.render()).onCancelTurn; - const request = deferred(); - c.pending.cancelTurn = request; - const stopped = c.calls.stopReading; - stop(); - stop(); - assert.equal(c.calls.cancelTurn, 1); - assert.equal(c.calls.stopReading, stopped + 1); - assert.equal(c.composer(c.render()).cancelBusy, true); - c.emit('qa:state', { kind: 'cancelled', sessionId: 'second', messages: firstTurn }); - c.emit('qa:state', { kind: 'thinking', sessionId: 'third', messages: firstTurn }); - stop(); - assert.equal(c.calls.cancelTurn, 1); - request.reject(new Error('a superseded cancel failed')); - await tick(); - const next = c.composer(c.render()); - assert.equal(next.status, 'thinking'); - assert.equal(next.cancelBusy, false); - assert.equal(nodes(c.render()).filter((n) => n.type === c.api.MessageRow).length, 2); - c.hooks.unmount(); -}); -await test('recording stop commits the recorded question without issuing a turn cancellation', async () => { - const c = context(true, true); - c.render(); - await tick(); - c.emit('qa:state', { kind: 'recording', sessionId: 'recording', messages: [] }); - await c.composer(c.render()).onToggleRecording(); - assert.equal(c.calls.stopMic, 1); - assert.equal(c.calls.mic, 0); - assert.equal(c.calls.cancelTurn, 0); - c.hooks.unmount(); -}); -await test('edit preview captured in old session cannot be applied after a new recording starts', async () => { - const c = context(); - c.render(); - await tick(); - c.emit('qa:state', { - kind: 'answer', - sessionId: 'a', - messages: [{ role: 'assistant', content: 'preview' }], - editApplyAvailable: true, - editRevertAvailable: true, - }); - let tree = c.render(); - const d = deferred(); - c.pending.preview = d; - find(tree, (n) => n.type === 'button' && n.props.className === 'qa-apply').props.onClick(); - assert.deepEqual(c.calls.preview, ['a']); - c.emit('qa:state', { kind: 'recording', sessionId: 'b', messages: [] }); - d.resolve({ text: 'must not apply' }); - await tick(); - assert.deepEqual(c.calls.confirm, []); - assert.equal(c.composer(c.render()).status, 'recording'); - c.hooks.unmount(); -}); -await test('an old rendered edit button cannot approve a newer answer before React presents it', async () => { - const c = context(); - c.render(); - await tick(); - c.emit('qa:state', { - kind: 'answer', - sessionId: 'a', - messages: [{ role: 'assistant', content: 'old preview' }], - editApplyAvailable: true, - }); - const oldTree = c.render(); - c.emit('qa:state', { kind: 'thinking', sessionId: 'b', messages: [] }); - c.emit('qa:state', { - kind: 'answer', - sessionId: 'b', - messages: [{ role: 'assistant', content: 'new unreviewed preview' }], - editApplyAvailable: true, - }); - find(oldTree, (n) => n.type === 'button' && n.props.className === 'qa-apply').props.onClick(); - await tick(); - assert.deepEqual(c.calls.preview, []); - assert.deepEqual(c.calls.confirm, []); - c.hooks.unmount(); -}); -await test('edit apply/revert are single-flight and send exact text plus captured session', async () => { - const c = context(); - c.render(); - await tick(); - c.emit('qa:state', { - kind: 'answer', - sessionId: 'a', - messages: [{ role: 'assistant', content: 'preview' }], - editApplyAvailable: true, - editRevertAvailable: true, - }); - let tree = c.render(); - const d = deferred(); - c.pending.confirm = d; - const button = find(tree, (n) => n.type === 'button' && n.props.className === 'qa-apply'); - button.props.onClick(); - button.props.onClick(); - await tick(); - assert.deepEqual(c.calls.confirm, [['fixture edit', 'a']]); - d.resolve(); - await tick(); - assert(!nodes(c.render()).some((n) => n.props.className === 'qa-edit-actions')); - c.emit('qa:state', { - kind: 'answer', - sessionId: 'a', - messages: [{ role: 'assistant', content: 'preview' }], - editApplyAvailable: true, - editRevertAvailable: true, - }); - tree = c.render(); - const r = deferred(); - c.pending.revert = r; - const back = find( - tree, - (n) => n.type === 'button' && n.props.children === 'qa.editRevertPrevious', - ); - back.props.onClick(); - back.props.onClick(); - assert.deepEqual(c.calls.revert, ['a']); - r.resolve(); - await tick(); - tree = c.render(); - assert(nodes(tree).some((n) => n.props.className === 'qa-apply')); - assert(!nodes(tree).some((n) => n.props.children === 'qa.editRevertPrevious')); - c.hooks.unmount(); -}); -await test('mode update waits for backend and repeated click cannot flip twice', async () => { - const c = context(true, true); - c.render(); - await tick(); - const d = deferred(); - c.pending.mode = d; - let p = c.composer(c.render()); - const first = p.onEditInstructionModeChange(true); - void p.onEditInstructionModeChange(false); - p = c.composer(c.render()); - assert.equal(p.editInstructionMode, false); - assert.deepEqual(c.calls.mode, [true]); - d.reject(new Error('fixture')); - await first; - p = c.composer(c.render()); - assert.equal(p.editInstructionMode, false); - assert.equal(p.status, 'error'); - c.hooks.unmount(); -}); -await test('embedded skip native resize, preserve existing parent close path and event cleanup', async () => { - const c = context(true, true); - c.render(); - await tick(); - c.composer(c.render()).onChange('embedded'); - await c.composer(c.render()).onSubmit(); - c.render(); - await tick(); - assert.deepEqual(c.calls.resize, []); - c.emit('qa:dismiss', {}); - assert.equal(c.calls.close, 1); - c.hooks.unmount(); - assert.equal( - [...c.events.values()].reduce((n, set) => n + set.size, 0), - 0, - ); - assert.equal(c.keys.size, 0); -}); -await test('unmount during dismiss registration releases the late handle and stops setup', async () => { - const c = context(); - const dismiss = deferred(); - c.pending.listen = (name) => (name === 'qa:dismiss' ? dismiss.promise : undefined); - c.render(); - await tick(); - assert.equal(c.events.get('qa:dismiss')?.size, 1); - c.hooks.unmount(); - dismiss.resolve(); - await tick(); - assert.equal(c.events.has('selection-polish-preview:shown'), false); - assert([...c.events.values()].every((listeners) => listeners.size === 0)); - assert.equal(c.calls.snapshot, 0); -}); -await test('composer IME enter, WebKit 229 and form submission guards; genuine enter sends', async () => { - const c = context(); - const h = new Hooks(); - let sent = 0; - const p = { - value: '中文', - status: 'idle', - level: 0, - voiceActive: false, - selectionPreview: '', - busy: false, - micBusy: false, - embedded: false, - editInstructionMode: false, - onEditInstructionModeChange() {}, - onChange() {}, - onSubmit() { - sent++; - }, - onToggleRecording() {}, - onClose() {}, - t: (k) => k, - }; - const tree = h.render(() => c.api.Composer(p)); - const input = find(tree, (n) => n.type === 'input'); - const key = (extra = {}) => ({ - key: 'Enter', - keyCode: 13, - nativeEvent: { isComposing: false }, - preventDefault() {}, - ...extra, - }); - input.props.onCompositionStart(); - input.props.onKeyDown(key()); - find(tree, (n) => n.type === 'form').props.onSubmit({ preventDefault() {} }); - input.props.onCompositionEnd(); - input.props.onKeyDown(key({ nativeEvent: { isComposing: true } })); - input.props.onKeyDown(key({ keyCode: 229 })); - assert.equal(sent, 0); - input.props.onKeyDown(key()); - assert.equal(sent, 1); -}); - -await test('session drift during delayed native resize cancels old draft', async () => { - const c = context(); - const d = deferred(); - c.pending.resize = () => d.promise; - c.render(); - await tick(); - c.beginConversation('a'); - c.composer(c.render()).onChange('belongs to a'); - - const result = c.composer(c.render()).onSubmit(); - await tick(); - c.emit('qa:state', { kind: 'idle', sessionId: 'b', messages: [] }); - d.resolve(); - await result; - await tick(); - assert.deepEqual(c.calls.submit, []); - c.hooks.unmount(); -}); -await test('text always carries the exact session captured before resize', async () => { - const c = context(); - c.render(); - await tick(); - c.beginConversation('a'); - c.composer(c.render()).onChange('question a'); - await c.composer(c.render()).onSubmit(); - assert.deepEqual(c.calls.submitArgs, [['question a', 'a']]); - c.hooks.unmount(); -}); -for (const embedded of [false, true]) - await test(`cold ${embedded ? 'embedded' : 'desktop'} snapshot recovers initial recording and accepts its levels`, async () => { - const c = context(true, embedded); - c.pending.snapshot = () => ({ - kind: 'recording', - sessionId: 'cold', - messages: [], - selectionPreview: 'captured text', - }); - c.render(); - await tick(); - c.emit('qa:level', { sessionId: 'cold', level: 0.4 }); - const p = c.composer(c.render()); - assert.equal(p.status, 'recording'); - assert.equal(p.level, 0.4); - assert.equal(p.selectionPreview, 'captured text'); - assert.equal(c.calls.snapshot, 1); - c.hooks.unmount(); - }); -await test('snapshot reply overtaken by a live session event is discarded and reread', async () => { - const c = context(); - const first = deferred(); - c.pending.snapshot = () => - c.calls.snapshot === 1 - ? first.promise - : { kind: 'recording', sessionId: 'new', messages: [], selectionPreview: 'new' }; - c.render(); - await tick(); - c.emit('qa:state', { - kind: 'recording', - sessionId: 'new', - messages: [], - selectionPreview: 'new', - }); - first.resolve({ kind: 'recording', sessionId: 'old', messages: [], selectionPreview: 'old' }); - await tick(); - c.emit('qa:level', { sessionId: 'new', level: 0.7 }); - const p = c.composer(c.render()); - assert.equal(p.selectionPreview, 'new'); - assert.equal(p.level, 0.7); - assert.equal(c.calls.snapshot, 2); - c.hooks.unmount(); -}); -await test('snapshot completion after dismiss cannot reopen old recording state', async () => { - const c = context(); - const first = deferred(); - c.pending.snapshot = () => first.promise; - c.render(); - await tick(); - c.emit('chat-panel:closing', {}); - c.emit('chat-panel:shown', {}); - c.emit('qa:state', { - kind: 'recording', - sessionId: 'new', - messages: [], - selectionPreview: 'new', - }); - first.resolve({ kind: 'recording', sessionId: 'old', messages: [], selectionPreview: 'old' }); - await tick(); - assert.equal(c.composer(c.render()).selectionPreview, 'new'); - c.hooks.unmount(); -}); -await test('a snapshot read failure does not silently remove all established live subscriptions', async () => { - const c = context(); - c.pending.snapshot = () => Promise.reject(new Error('transient snapshot unavailable')); - c.render(); - await tick(); - c.emit('qa:state', { kind: 'recording', sessionId: 'live', messages: [] }); - c.emit('qa:level', { sessionId: 'live', level: 0.4 }); - const p = c.composer(c.render()); - assert.equal(p.status, 'recording'); - assert.equal(p.level, 0.4); - c.hooks.unmount(); -}); -await test('old microphone RPC failure cannot overwrite a newer native recording session', async () => { - const c = context(true, true); - c.render(); - await tick(); - c.emit('qa:state', { kind: 'idle', sessionId: 'before', messages: [] }); - const old = deferred(); - c.pending.mic = old; - const request = c.composer(c.render()).onToggleRecording(); - c.emit('qa:state', { kind: 'recording', sessionId: 'a', messages: [] }); - c.emit('qa:state', { - kind: 'error', - sessionId: 'a', - messages: [], - error: 'old recording failed', - }); - c.emit('qa:state', { kind: 'recording', sessionId: 'b', messages: [] }); - old.reject(new Error('old RPC completed late')); - await request; - const p = c.composer(c.render()); - assert.equal(p.status, 'recording'); - c.emit('qa:level', { sessionId: 'b', level: 0.8 }); - assert.equal(c.composer(c.render()).level, 0.8); - c.hooks.unmount(); -}); -console.log(JSON.stringify({ passed, failed })); -process.exitCode = failed ? 1 : 0; diff --git a/openless-all/app/scripts/selection-polish-panel-contract.test.mjs b/openless-all/app/scripts/selection-polish-panel-contract.test.mjs deleted file mode 100644 index acf5568b8..000000000 --- a/openless-all/app/scripts/selection-polish-panel-contract.test.mjs +++ /dev/null @@ -1,145 +0,0 @@ -// 「划词润色结果」不再有独立窗口:它复用选区助手(qa)面板显示第二套 UI -// (可编辑结果 + 「确认并替换」)。本契约锁住这次合并,防止有人把独立窗口加回来 -// 或把 /qa 面板拆回去。 -// -// 为什么需要它: -// - 独立窗口 `selection-polish-preview` 曾是第 6 个桌面浮窗(多一个 WebKit 进程 -// + 一套只服务一个弹窗的前端页面);合并后入口只剩「预览确认」这一个输出模式。 -// - 合并依赖三处配合:后端把窗口指向 qa、前端 QaPanel 认事件、IPC 用 pending -// 标志区分「真的在润色模式」和「残留快照」。任一处回退都会静默失灵(窗口不再 -// 出现),所以逐条断言。 - -import { readFile, stat } from 'node:fs/promises'; -import { fileURLToPath } from 'node:url'; - -const appRoot = fileURLToPath(new URL('..', import.meta.url)); -const read = (relativePath) => readFile(`${appRoot}/${relativePath}`, 'utf8'); - -const violations = []; - -const lib = await read('src-tauri/src/lib.rs'); -const capabilities = await read('src-tauri/capabilities/default.json'); -const commands = await read('src-tauri/src/commands/selection_polish_preview.rs'); -const qaPanel = await read('src/features/qa/QaPanel.tsx'); -const selectionApi = await read('src/api/selection.ts'); -const appTsx = await read('src/app/windowRoutes.tsx'); -const mainTsx = await read('src/app/bootstrap.tsx'); - -// 1. 独立窗口彻底下线:能力表、前端路由、页面文件都不再有它。 -const windowCapabilities = JSON.parse(capabilities).windows; -if (windowCapabilities.includes('selection-polish-preview')) { - violations.push('capabilities/default.json still grants the selection-polish-preview window'); -} -if (!windowCapabilities.includes('qa')) { - violations.push('capabilities/default.json dropped the qa window (the merged panel needs it)'); -} -for (const [name, source] of [ - ['src/app/windowRoutes.tsx', appTsx], - ['src/app/bootstrap.tsx', mainTsx], -]) { - if (source.includes('selection-polish-preview')) { - violations.push(`${name}: still routes the deleted selection-polish-preview window`); - } - if (source.includes('SelectionPolishPreview')) { - violations.push(`${name}: still imports the deleted SelectionPolishPreview page`); - } -} -for (const source of [ - 'src/pages/SelectionPolishPreview.tsx', - 'src/features/qa/SelectionPolishPreview.tsx', -]) { - const previewPageExists = await stat(`${appRoot}/${source}`).then( - () => true, - () => false, - ); - if (previewPageExists) { - violations.push(`${source} still exists`); - } -} - -// 2. 后端把「润色结果」交给 qa 面板,而不是新窗口。 -if ( - !/fn show_selection_polish_preview[\s\S]{0,400}?show_qa_window\(app, "polish-preview"\)/.test(lib) -) { - violations.push('show_selection_polish_preview no longer opens the qa panel'); -} -if (!/emit_to\("qa", SELECTION_POLISH_PREVIEW_SHOWN/.test(lib)) { - violations.push('the polish-result "shown" event is not emitted to the qa window'); -} -if (!/emit_to\("qa", SELECTION_POLISH_PREVIEW_HIDE/.test(lib)) { - violations.push('the polish-result "hide" event is not emitted to the qa window'); -} -if (/WebviewWindowBuilder::new\(\s*app,\s*"selection-polish-preview"/.test(lib)) { - violations.push('lib.rs builds a selection-polish-preview window again'); -} - -// 3. 是否退出润色模式由前端决定(面板可能正在提问对话中)。 -if ( - !/fn hide_selection_polish_preview[\s\S]{0,300}?clear_selection_polish_preview_pending\(\)/.test( - lib, - ) -) { - violations.push('hide_selection_polish_preview no longer clears the pending flag'); -} - -// 4. IPC:预览负载只在「真的处于润色模式」时给出。 -if ( - !/fn get_selection_polish_preview[\s\S]{0,400}?if !crate::selection_polish_preview_pending\(\)/.test( - commands, - ) -) { - violations.push('get_selection_polish_preview lost its pending gate'); -} -if ( - !/fn confirm_selection_polish_preview[\s\S]{0,200}?clear_selection_polish_preview_pending\(\)/.test( - commands, - ) -) { - violations.push('confirm_selection_polish_preview no longer clears the pending flag'); -} -if ( - !/fn cancel_selection_polish_preview[\s\S]{0,200}?clear_selection_polish_preview_pending\(\)/.test( - commands, - ) -) { - violations.push('cancel_selection_polish_preview no longer clears the pending flag'); -} - -// 5. 面板侧:认事件、编辑结果、提供「确认并替换」。 -for (const [eventName, listen] of [ - ['selection-polish-preview:shown', 'listenSelectionPolishPreviewShown'], - ['selection-polish-preview:hide', 'listenSelectionPolishPreviewHidden'], -]) { - if (!selectionApi.includes(`'${eventName}'`) || !qaPanel.includes(`${listen}(`)) { - violations.push(`QaPanel.tsx does not subscribe to ${eventName} through the selection API`); - } -} -if (!qaPanel.includes('selectionPolishPreview.confirmReplace')) { - violations.push('QaPanel.tsx dropped the 「确认并替换」 button of the polish-result mode'); -} -if ( - !/ readFile(new URL(`../${path}`, import.meta.url), 'utf8'); const [ types, - qaPanel, - qaApi, inputApi, remoteIpc, remoteStatusController, @@ -48,8 +46,6 @@ const [ mobileSelection, ] = await Promise.all([ read('src/lib/types.ts'), - read('src/features/qa/QaPanel.tsx'), - read('src/api/qa.ts'), read('src/api/input.ts'), read('src/lib/ipc/remote-server.ts'), read('src/features/remote-input/useRemoteInputStatus.ts'), @@ -97,7 +93,6 @@ const [ for (const kind of ['awaiting_approval', 'cancelled', 'error']) { assert.match(types, new RegExp(`\\| '${kind}'`), `QaStateKind must retain ${kind}`); - assert.match(qaPanel, new RegExp(`case '${kind}':`), `QaPanel must handle ${kind}`); } for (const field of [ 'sessionId', @@ -112,13 +107,6 @@ for (const field of [ ]) { assert.match(types, new RegExp(`\\b${field}\\?`), `QaStatePayload is missing ${field}`); } -assert.match( - qaApi, - /listenTauriEvent\('qa:state'/, - 'QA API must type the state event', -); -assert.match(qaPanel, /listenQaStateChanges\(/, 'QA must subscribe through its domain API'); - const remoteStatus = remoteIpc.match(/export interface RemoteInputStatus \{([\s\S]*?)\n\}/)?.[1]; assert.ok(remoteStatus, 'RemoteInputStatus interface must exist'); const remoteFields = [...remoteStatus.matchAll(/^\s+(\w+):/gm)].map((match) => match[1]); diff --git a/openless-all/app/scripts/windows-hang-diagnostics-contract.test.mjs b/openless-all/app/scripts/windows-hang-diagnostics-contract.test.mjs index 1dbc59148..82a1b7d1d 100644 --- a/openless-all/app/scripts/windows-hang-diagnostics-contract.test.mjs +++ b/openless-all/app/scripts/windows-hang-diagnostics-contract.test.mjs @@ -49,7 +49,7 @@ assert.deepEqual( ); assert.match(coordinator, /issue1266_debug::lock_host\(&self\.inner\.settings_host_gate/); assert.doesNotMatch(coordinator + loops, /settings_host_gate\.lock\(\)/); -for (const kind of ['MainHotkey', 'QaHotkey', 'ComboHotkey', 'ActionHotkey', 'StyleHotkey']) { +for (const kind of ['MainHotkey', 'ComboHotkey', 'ActionHotkey', 'StyleHotkey']) { assert.ok(loops.includes(`lock_host(&inner.settings_host_gate, DebugKind::${kind})`)); } assert.doesNotMatch(loops, /fn coding_agent_hotkey_supervisor_loop/); diff --git a/openless-all/app/scripts/windows-ui-config.test.mjs b/openless-all/app/scripts/windows-ui-config.test.mjs index a1e05eb4f..0e1cc5c70 100644 --- a/openless-all/app/scripts/windows-ui-config.test.mjs +++ b/openless-all/app/scripts/windows-ui-config.test.mjs @@ -301,7 +301,7 @@ assertMatch( ); if ( - !/const hostMetrics = getCapsuleHostMetrics\(os,\s*translation \|\| qa,\s*capsuleStyle\);/.test( + !/const hostMetrics = getCapsuleHostMetrics\(os,\s*translation,\s*capsuleStyle\);/.test( capsuleTsx, ) ) { diff --git a/openless-all/app/src-tauri/backend-tests/tests/mouse_dictation.rs b/openless-all/app/src-tauri/backend-tests/tests/mouse_dictation.rs index 338143c57..df6664fee 100644 --- a/openless-all/app/src-tauri/backend-tests/tests/mouse_dictation.rs +++ b/openless-all/app/src-tauri/backend-tests/tests/mouse_dictation.rs @@ -13,8 +13,6 @@ mod shortcut_binding { mod combo_hotkey; #[allow(dead_code)] -#[path = "../../src/recording_companion.rs"] -mod recording_companion; #[allow(dead_code)] #[path = "../../src/mouse_dictation.rs"] diff --git a/openless-all/app/src-tauri/capabilities/default.json b/openless-all/app/src-tauri/capabilities/default.json index d1bbc0d85..40dda4cb3 100644 --- a/openless-all/app/src-tauri/capabilities/default.json +++ b/openless-all/app/src-tauri/capabilities/default.json @@ -3,7 +3,7 @@ "identifier": "default", "description": "Default capabilities for OpenLess windows", "platforms": ["macOS", "windows", "linux"], - "windows": ["main", "capsule", "capsule-mirror-*", "qa", "less-computer", "less-computer-glow"], + "windows": ["main", "capsule", "capsule-mirror-*", "less-computer", "less-computer-glow"], "permissions": [ "core:default", "core:window:default", diff --git a/openless-all/app/src-tauri/capabilities/mobile.json b/openless-all/app/src-tauri/capabilities/mobile.json index 911ad21d0..07db242b7 100644 --- a/openless-all/app/src-tauri/capabilities/mobile.json +++ b/openless-all/app/src-tauri/capabilities/mobile.json @@ -3,7 +3,7 @@ "identifier": "mobile", "description": "Capabilities for OpenLess Android main window", "platforms": ["android"], - "windows": ["main", "qa"], + "windows": ["main"], "permissions": [ "core:default", "core:window:default", diff --git a/openless-all/app/src-tauri/src/android/native_bridge.rs b/openless-all/app/src-tauri/src/android/native_bridge.rs index 3754ca609..a87c4cfd3 100644 --- a/openless-all/app/src-tauri/src/android/native_bridge.rs +++ b/openless-all/app/src-tauri/src/android/native_bridge.rs @@ -1129,19 +1129,6 @@ fn spawn_switch_style_pack() { coordinator.switch_to_previous_style_pack(); } -fn spawn_finalize_qa_from_overlay() { - let Some(coordinator) = COORDINATOR.get().cloned() else { - log::warn!("[android-native] coordinator unavailable"); - return; - }; - log::info!("[android-native] finalize_qa_from_overlay requested"); - tauri::async_runtime::spawn(async move { - if let Err(error) = coordinator.finalize_qa_from_overlay().await { - log::warn!("[android-native] finalize_qa_from_overlay failed: {error}"); - } - }); -} - fn spawn_finalize_voice_edit_from_overlay() { let Some(coordinator) = COORDINATOR.get().cloned() else { return; @@ -1587,14 +1574,6 @@ mod jni_exports { spawn_switch_style_pack(); } - #[no_mangle] - pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeFinalizeQaFromOverlay( - _env: *mut JNIEnv, - _class: JClass, - ) { - spawn_finalize_qa_from_overlay(); - } - #[no_mangle] pub unsafe extern "system" fn Java_com_openless_app_OpenLessNative_nativeFinalizeVoiceEditFromOverlay( _env: jni::JNIEnv, diff --git a/openless-all/app/src-tauri/src/ask_window/mod.rs b/openless-all/app/src-tauri/src/ask_window/mod.rs new file mode 100644 index 000000000..79fd6ea0d --- /dev/null +++ b/openless-all/app/src-tauri/src/ask_window/mod.rs @@ -0,0 +1,24 @@ +use tauri::{AppHandle, Manager, WebviewUrl, WebviewWindowBuilder}; + +const LABEL: &str = "ask-anything"; +const PAGE: &str = "ask-window/index.html"; +static OPEN: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +/// Opens the standalone Ask surface without creating a conversation or starting a recording. +#[tauri::command] +pub async fn ask_window_open(app: AppHandle) -> Result<(), String> { + // Native window creation runs from the async command, not a Windows event handler. + let _opening = OPEN.lock().await; + let window = match app.get_webview_window(LABEL) { + Some(window) => window, + None => WebviewWindowBuilder::new(&app, LABEL, WebviewUrl::App(PAGE.into())) + .title("OpenLess — Ask Anything") + .inner_size(920.0, 800.0) + .center() + .visible(false) + .build() + .map_err(|error| error.to_string())?, + }; + window.show().map_err(|error| error.to_string())?; + window.set_focus().map_err(|error| error.to_string()) +} diff --git a/openless-all/app/src-tauri/src/commands/deepseek_tts.rs b/openless-all/app/src-tauri/src/commands/deepseek_tts.rs deleted file mode 100644 index 327d86c67..000000000 --- a/openless-all/app/src-tauri/src/commands/deepseek_tts.rs +++ /dev/null @@ -1,152 +0,0 @@ -use super::*; -use std::collections::{HashMap, VecDeque}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{LazyLock, Mutex as StdMutex}; - -#[derive(Default)] -struct PlaybackState { - active: Option<(String, Arc)>, - stopped: VecDeque, -} -impl PlaybackState { - fn begin(&mut self, request_id: String) -> Option> { - if self.stopped.contains(&request_id) { - return None; - } - let token = Arc::new(AtomicBool::new(false)); - if let Some((_, old)) = self.active.replace((request_id, Arc::clone(&token))) { - old.store(true, Ordering::Release); - } - Some(token) - } - fn stop(&mut self, request_id: String) { - // A synchronous Stop command can arrive before the async Start task runs. - // Retain bounded cancellation markers so that task cannot revive playback. - if !self.stopped.contains(&request_id) { - self.stopped.push_back(request_id.clone()); - if self.stopped.len() > 64 { - self.stopped.pop_front(); - } - } - if self - .active - .as_ref() - .is_some_and(|(id, _)| id == &request_id) - { - if let Some((_, token)) = self.active.take() { - token.store(true, Ordering::Release); - } - } - } -} -static PLAYBACK: LazyLock>> = - LazyLock::new(|| StdMutex::new(HashMap::new())); -struct PlaybackGuard { - owner: String, - token: Arc, -} -impl Drop for PlaybackGuard { - fn drop(&mut self) { - let mut map = PLAYBACK.lock().unwrap(); - if let Some(state) = map.get_mut(&self.owner) { - if state - .active - .as_ref() - .is_some_and(|(_, token)| Arc::ptr_eq(token, &self.token)) - { - state.active = None; - } - } - } -} -#[tauri::command] -pub fn deepseek_tts_stop(window: Window, request_id: String) { - if !matches!(window.label(), "qa" | "main") || request_id.len() > 128 { - return; - } - PLAYBACK - .lock() - .unwrap() - .entry(window.label().into()) - .or_default() - .stop(request_id); -} -#[tauri::command] -pub async fn deepseek_tts_start( - window: Window, - core: CoreState<'_>, - text: String, - request_id: String, - audio: tauri::ipc::Channel>, -) -> Result<(), String> { - if !matches!(window.label(), "qa" | "main") { - return Err("qa_window_required".into()); - } - if core.get_preferences().pipeline_mode != openless_core::shared_types::PipelineMode::Deepseek { - return Err("Read aloud is available only in DeepSeek suite mode".into()); - } - if request_id.is_empty() || request_id.len() > 128 { - return Err("Invalid speech request id".into()); - } - let Some(token) = PLAYBACK - .lock() - .unwrap() - .entry(window.label().into()) - .or_default() - .begin(request_id) - else { - return Ok(()); - }; - let _guard = PlaybackGuard { - owner: window.label().into(), - token: token.clone(), - }; - // Resolve the channel identity and key from one vault snapshot so a channel - // repurposed for another vendor cannot send that vendor's secret to DeepSeek. - let (secret, voice) = - tokio::task::spawn_blocking(CredentialsVault::get_deepseek_tts_credentials) - .await - .map_err(|_| "Could not read speech configuration")? - .map_err(|_| "Configure an official DeepSeek channel in Model services")?; - let cancelled = async { - while !token.load(Ordering::Acquire) - && core.get_preferences().pipeline_mode - == openless_core::shared_types::PipelineMode::Deepseek - { - tokio::time::sleep(std::time::Duration::from_millis(25)).await; - } - }; - tokio::select! { - biased; - _ = cancelled => Ok(()), - result = tokio::time::timeout(std::time::Duration::from_secs(1800), openless_core::deepseek_tts::stream(secret.expose_secret(), &text, &voice, |bytes| audio.send(bytes).map_err(|_| "Playback window closed".into()))) => result.map_err(|_| "Speech playback timed out".to_string())?, - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn stop_before_start_prevents_late_network_work() { - let mut state = PlaybackState::default(); - state.stop("cancelled".into()); - assert!(state.begin("cancelled".into()).is_none()); - assert!(state.active.is_none()); - } - #[test] - fn late_stop_cannot_cancel_a_new_request() { - let mut state = PlaybackState::default(); - let first = state.begin("first".into()).unwrap(); - let second = state.begin("second".into()).unwrap(); - assert!(first.load(Ordering::Acquire)); - state.stop("first".into()); - assert!(!second.load(Ordering::Acquire)); - state.stop("second".into()); - assert!(second.load(Ordering::Acquire)); - assert!(state.begin("second".into()).is_none()); - for i in 0..100 { - state.stop(format!("request-{i}")); - } - assert_eq!(state.stopped.len(), 64); - } -} diff --git a/openless-all/app/src-tauri/src/commands/hotkeys.rs b/openless-all/app/src-tauri/src/commands/hotkeys.rs index 3bd555ac8..b48d0a647 100644 --- a/openless-all/app/src-tauri/src/commands/hotkeys.rs +++ b/openless-all/app/src-tauri/src/commands/hotkeys.rs @@ -46,7 +46,7 @@ pub fn set_translation_hotkey( } /// Sets the "switch style" global hotkey. `binding == None` (frontend passes null) = disable: -/// clears the binding and unregisters the global key. Mirrors `set_qa_hotkey`'s `Option=None` disable mode (issue #576). +/// clears the binding and unregisters the global key. #[tauri::command] pub fn set_switch_style_hotkey( coord: CoordinatorState<'_>, diff --git a/openless-all/app/src-tauri/src/commands/mod.rs b/openless-all/app/src-tauri/src/commands/mod.rs index f41c74630..01e4c651c 100644 --- a/openless-all/app/src-tauri/src/commands/mod.rs +++ b/openless-all/app/src-tauri/src/commands/mod.rs @@ -73,8 +73,6 @@ mod qa; mod remote_input; #[cfg(all(not(mobile), debug_assertions))] mod selection_polish; -#[cfg(not(mobile))] -mod selection_polish_preview; #[cfg(all(not(mobile), target_os = "windows"))] mod selection_voice; mod settings; @@ -108,8 +106,6 @@ pub use qa::*; pub use remote_input::*; #[cfg(all(not(mobile), debug_assertions))] pub use selection_polish::*; -#[cfg(not(mobile))] -pub use selection_polish_preview::*; #[cfg(all(not(mobile), target_os = "windows"))] pub use selection_voice::*; pub(crate) use settings::sync_low_latency_capture_preferences; @@ -977,6 +973,3 @@ mod tests { assert!(!is_valid_local_pack_id(&"a".repeat(129))); } } - -mod deepseek_tts; -pub use deepseek_tts::*; diff --git a/openless-all/app/src-tauri/src/commands/qa.rs b/openless-all/app/src-tauri/src/commands/qa.rs index 0e843209e..2216181b3 100644 --- a/openless-all/app/src-tauri/src/commands/qa.rs +++ b/openless-all/app/src-tauri/src/commands/qa.rs @@ -1,161 +1,16 @@ use super::*; -#[tauri::command] -pub fn get_qa_hotkey_label(core: CoreState<'_>) -> String { - let prefs = core.get_preferences(); - if prefs.qa_companion_key.is_empty() { - return String::new(); - } - format!( - "{} + {}", - prefs.dictation_hotkey.display_label(), - prefs.qa_companion_key - ) -} - -/// Sets the auxiliary key for recording-key + Ask. Null disables this chord; -/// the retired independent follow-up shortcut is never registered. -#[tauri::command] -pub fn set_qa_hotkey( - coord: CoordinatorState<'_>, - binding: Option, -) -> Result<(), String> { - if binding - .as_ref() - .is_some_and(|binding| !binding.modifiers.is_empty()) - { - return Err("只需设置提问附加键,录音主键会自动跟随当前选择".into()); - } - let mut prefs = coord.backend().get_preferences(); - prefs.qa_hotkey = None; - prefs.qa_companion_key = binding.map(|binding| binding.primary).unwrap_or_default(); - reject_hotkey_collisions(&prefs)?; - super::settings::persist_strict_settings(&coord, prefs) -} - -/// User clicks ✕ or presses Esc to close the QA panel. -#[tauri::command] -pub async fn qa_window_dismiss(core: CoreState<'_>) -> Result<(), String> { - core.services() - .qa - .dismiss() - .await - .map_err(|error| error.message) -} - -/// Mobile QA panel record button: Idle -> begin_qa_session, Recording -> -/// end_qa_session. -#[tauri::command] -pub async fn qa_toggle_recording(core: CoreState<'_>) -> Result<(), String> { - core.services() - .qa - .toggle_recording() - .await - .map_err(|error| error.message) -} - -/// QA panel keyboard input: reuses the voice-QA LLM pipeline, replacing only -/// the question source. -#[tauri::command] -pub async fn qa_submit_text( - core: CoreState<'_>, - text: String, - expected_session_id: Option, - enforce_context: Option, -) -> Result<(), String> { - if enforce_context.unwrap_or(false) { - core.services() - .qa - .submit_text_in_context(text, expected_session_id) - .await - .map_err(|error| error.message) - } else { - core.services() - .qa - .submit_text(text) - .await - .map_err(|error| error.message) - } -} - -#[tauri::command] -pub async fn qa_get_snapshot( - window: Window, - core: CoreState<'_>, -) -> Result { - if !matches!(window.label(), "qa" | "main") { - return Err("qa_window_required".into()); - } - let snapshot = core - .services() - .qa - .snapshot() - .await - .map_err(|_| "qa_snapshot_unavailable")?; - let mut event = openless_core::events::QaStateEvent::from_snapshot(&snapshot); - event.edit_instruction_mode = Some(snapshot.edit_instruction_mode); - event.edit_apply_available = Some(snapshot.edit_apply_available); - event.edit_revert_available = Some(snapshot.edit_revert_available); - Ok(event) -} - -#[tauri::command] -pub async fn qa_window_set_expanded(window: Window, expanded: bool) -> Result<(), String> { - use tauri::Manager; - if window.label() != "qa" { - return Err("qa_window_required".into()); - } - let app = window.app_handle().clone(); - let (sender, receiver) = tokio::sync::oneshot::channel(); - window - .app_handle() - .run_on_main_thread(move || { - let _ = sender.send(crate::set_qa_window_expanded(&app, expanded)); - }) - .map_err(|_| "qa_main_thread_unavailable".to_string())?; - receiver - .await - .map_err(|_| "qa_resize_cancelled".to_string())? -} - -/// Selection QA panel "edit instruction" checkbox. -#[tauri::command] -pub async fn qa_set_edit_instruction_mode( - core: CoreState<'_>, - enabled: bool, -) -> Result<(), String> { - core.services() - .qa - .set_edit_instruction_mode(enabled) - .await - .map_err(|error| error.message) -} - /// User clicks ✕ or presses Esc to close the Less Computer panel. #[tauri::command] pub async fn less_computer_window_dismiss(coord: CoordinatorState<'_>) -> Result<(), String> { coord.dismiss_less_computer().await } -/// Chat panels (qa / less-computer) request keyboard focus. -/// -/// Both panels display without stealing the foreground (macOS -/// orderFrontRegardless, never makeKey), so keystrokes never reach the webview -/// while the window is not the key window — the root cause of "clicked the -/// input box but cannot type". -/// -/// macOS: the window is already a non-activating NSPanel -/// (make_chat_window_panel_macos), so makeKeyAndOrderFront gives the panel -/// keyboard focus without activating the app (same as Spotlight) — the earlier -/// window.set_focus() activated the whole app, dragged the main window -/// (settings) to the front, and made OpenLess frontmost so AX could not read -/// the original app's selection. Other platforms still use set_focus. Only the -/// two chat panel windows may call this (same tightening as -/// less_computer_approve). +/// Gives the non-activating Less Computer panel keyboard focus. #[tauri::command] pub fn chat_panel_focus_keyboard(window: Window) -> Result<(), String> { let label = window.label(); - if label != "qa" && label != "less-computer" { + if label != "less-computer" { return Err("chat_panel_focus_keyboard is only available to chat panels".to_string()); } #[cfg(target_os = "macos")] @@ -343,33 +198,3 @@ pub async fn less_computer_approve( .await .map_err(|error| error.to_string()) } - -#[tauri::command] -pub async fn qa_cancel_turn(core: CoreState<'_>) -> Result<(), String> { - core.services() - .qa - .cancel(None) - .await - .map_err(|error| error.message) -} - -#[tauri::command] -pub async fn qa_stop_recording(core: CoreState<'_>) -> Result<(), String> { - let snapshot = core - .services() - .qa - .snapshot() - .await - .map_err(|error| error.message)?; - if snapshot.phase != openless_core::QaPhase::Recording { - return Ok(()); - } - if let Some(id) = snapshot.session_id { - core.services() - .qa - .stop_recording(id) - .await - .map_err(|error| error.message)?; - } - Ok(()) -} diff --git a/openless-all/app/src-tauri/src/commands/selection_polish_preview.rs b/openless-all/app/src-tauri/src/commands/selection_polish_preview.rs deleted file mode 100644 index e8717aba7..000000000 --- a/openless-all/app/src-tauri/src/commands/selection_polish_preview.rs +++ /dev/null @@ -1,93 +0,0 @@ -use super::*; - -#[derive(Debug, Clone, serde::Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SelectionPolishPreviewPayload { - pub text: String, - pub source_text: String, -} - -fn map_selection_preview( - snapshot: openless_core::SelectionSnapshot, -) -> Option { - match (snapshot.preview_text, snapshot.source_text) { - (Some(text), Some(source_text)) => { - Some(SelectionPolishPreviewPayload { text, source_text }) - } - _ => None, - } -} - -#[tauri::command] -pub async fn get_selection_polish_preview( - core: CoreState<'_>, -) -> Result, String> { - // 面板懒创建后会自己拉一次;只有后端确实处在「润色结果」模式时才给负载, - // 免把残留的选区快照当成新请求(面板会被意外切到润色模式)。 - if !crate::selection_polish_preview_pending() { - return Ok(None); - } - let snapshot = core - .services() - .selection - .snapshot() - .await - .map_err(|error| error.message)?; - Ok(map_selection_preview(snapshot)) -} - -#[tauri::command] -pub async fn confirm_selection_polish_preview( - core: CoreState<'_>, - text: String, -) -> Result<(), String> { - crate::clear_selection_polish_preview_pending(); - let snapshot = core - .services() - .selection - .snapshot() - .await - .map_err(|error| error.message)?; - let session_id = snapshot - .session_id - .ok_or_else(|| "selection preview is not active".to_string())?; - core.services() - .selection - .confirm(session_id, Some(text)) - .await - .map_err(|error| error.message) -} - -#[tauri::command] -pub async fn cancel_selection_polish_preview(core: CoreState<'_>) -> Result<(), String> { - crate::clear_selection_polish_preview_pending(); - core.services() - .selection - .cancel(None) - .await - .map_err(|error| error.message) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn selection_preview_wire_uses_legacy_camel_case_fields() { - let payload = map_selection_preview(openless_core::SelectionSnapshot { - phase: openless_core::SelectionPhase::Preview, - session_id: Some(openless_core::SessionId::new()), - source_text: Some("source".to_string()), - preview_text: Some("preview".to_string()), - instruction: None, - insert_outcome: None, - revert_outcome: None, - }) - .expect("preview snapshot should map to the legacy payload"); - - assert_eq!( - serde_json::to_value(payload).unwrap(), - serde_json::json!({ "text": "preview", "sourceText": "source" }) - ); - } -} diff --git a/openless-all/app/src-tauri/src/commands/selection_voice.rs b/openless-all/app/src-tauri/src/commands/selection_voice.rs index e0a5dbcab..6b0b23317 100644 --- a/openless-all/app/src-tauri/src/commands/selection_voice.rs +++ b/openless-all/app/src-tauri/src/commands/selection_voice.rs @@ -79,6 +79,9 @@ pub async fn confirm_selection_voice_intent_prompt( coord: CoordinatorState<'_>, intent: String, ) -> Result<(), String> { + if !matches!(intent.as_str(), "edit" | "compose") { + return Err("selectionVoiceInvalidIntent".into()); + } let session_id = core .services() .selection_voice diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 6d36d82c5..c6dc40479 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -21,7 +21,6 @@ use crate::persistence::{ }; use crate::qa_adapter::TauriQaHostContext; -use crate::qa_hotkey::{QaHotkeyError, QaHotkeyEvent, QaHotkeyMonitor}; use crate::types::{ CapsulePayload, CapsuleState, CapsuleStyle, HotkeyCapability, HotkeyStatus, HotkeyStatusState, }; @@ -31,7 +30,6 @@ mod capsule_focus; mod dictation; mod hotkey_loops; mod native_dictation_key; -mod qa; mod restore_runtime; #[cfg(all(not(mobile), target_os = "windows"))] pub(crate) mod selection_voice_session; @@ -64,7 +62,6 @@ pub(super) fn qa_event_target() -> &'static str { #[cfg(any(debug_assertions, test))] use dictation::{handle_pressed, handle_released}; use dictation::{handle_pressed_edge, handle_released_edge, handle_trigger_combined}; -use qa::handle_qa_hotkey_pressed; /// Window size of the vocabulary suggestion card (logical points). /// @@ -578,7 +575,6 @@ struct Inner { /// explicit runtime target out of order. settings_host_gate: Mutex<()>, hotkey_resume_started: AtomicBool, - overlay_qa_handoff: tokio::sync::Mutex<()>, inserter: TextInserter, /// Whether the suggestion card is currently occupying the capsule window. /// @@ -630,7 +626,6 @@ struct Inner { /// Selection voice QA (issue #118): global shortcut listener parallel to /// the dictation hotkey (global-hotkey crate). `None` means the feature is /// off or installation has not succeeded yet. - qa_hotkey: Mutex>, coding_agent_modifier_hotkey: Mutex>, coding_agent_combo_hotkey: Mutex>, /// State from the most recent emit_capsule, for introspection/tests only @@ -780,7 +775,6 @@ impl Coordinator { hotkey_runtime_target: Mutex::new(hotkey_runtime_target), settings_host_gate: Mutex::new(()), hotkey_resume_started: AtomicBool::new(false), - overlay_qa_handoff: tokio::sync::Mutex::new(()), inserter: TextInserter::new(), vocab_card_visible: AtomicBool::new(false), vocab_card_epoch: crate::vocab_card_placement::PresentationEpoch::default(), @@ -809,7 +803,6 @@ impl Coordinator { voice_edit_host: Arc::new(Mutex::new( voice_edit_session::VoiceEditHostState::default(), )), - qa_hotkey: Mutex::new(None), coding_agent_modifier_hotkey: Mutex::new(None), coding_agent_combo_hotkey: Mutex::new(None), last_capsule_state: Mutex::new(None), @@ -922,7 +915,6 @@ impl Coordinator { hotkey_runtime_target: Mutex::new(hotkey_runtime_target), settings_host_gate: Mutex::new(()), hotkey_resume_started: AtomicBool::new(false), - overlay_qa_handoff: tokio::sync::Mutex::new(()), inserter: TextInserter::new(), vocab_card_visible: AtomicBool::new(false), vocab_card_epoch: crate::vocab_card_placement::PresentationEpoch::default(), @@ -949,7 +941,6 @@ impl Coordinator { voice_edit_host: Arc::new( Mutex::new(voice_edit_session::VoiceEditHostState::default()), ), - qa_hotkey: Mutex::new(None), coding_agent_modifier_hotkey: Mutex::new(None), coding_agent_combo_hotkey: Mutex::new(None), last_capsule_state: Mutex::new(None), @@ -1062,8 +1053,7 @@ impl Coordinator { || self.inner.selection_polish_capsule_active.load(Ordering::SeqCst) || self.inner.vocab_card_visible.load(Ordering::SeqCst) || self.inner.qa_context.is_panel_visible() - || self.inner.backend.less_computer_active_session().is_some() - || crate::recording_companion::claimed(); + || self.inner.backend.less_computer_active_session().is_some(); if !crate::capsule_feedback::can_present_feedback( *self.inner.last_capsule_state.lock(), self.inner.backend.snapshot().dictation.phase, @@ -1278,7 +1268,6 @@ impl Coordinator { if inner.backend.ensure_runtime_ready().is_ok() { let coord = Coordinator { inner }; coord.start_hotkey_listener(); - coord.start_qa_hotkey_listener(); #[cfg(not(mobile))] coord.start_selection_polish_hotkey_listener(); coord.start_combo_hotkey_listener(); @@ -1320,13 +1309,6 @@ impl Coordinator { self.inner.hotkey.lock().take(); } - /// Starts the QA hotkey supervisor (issue #118), parallel to - /// `start_hotkey_listener`: a daemon thread retries registration every 3s - /// on failure (the user may have changed the combo). - pub fn start_qa_hotkey_listener(&self) { - qa::install_recording_companion(&self.inner); - } - pub fn stop_coding_agent_hotkey_listener(&self) { take_coding_agent_hotkeys_on_main_thread(&self.inner); } @@ -1335,27 +1317,6 @@ impl Coordinator { update_coding_agent_hotkey_binding_now(&self.inner) } - pub fn stop_qa_hotkey_listener(&self) { - // QaHotkeyMonitor::drop bottoms out in Carbon RemoveEventHotKey on - // macOS, which requires the main thread. The RunEvent::Exit callback is - // not guaranteed to run on the AppKit main thread; a drop that lands on - // a tokio worker triggers macOS dispatch_assert_queue_fail SIGTRAP. - // Wrap it in run_on_main_thread so the drop happens on the main - // thread; when the AppHandle is already None, drop directly (worst case - // is a crash at exit time anyway). See issue #169. - let inner = Arc::clone(&self.inner); - if self - .inner - .host - .run_on_main_thread(move || { - inner.qa_hotkey.lock().take(); - }) - .is_err() - { - self.inner.qa_hotkey.lock().take(); - } - } - #[cfg(not(mobile))] pub fn start_selection_polish_hotkey_listener(&self) { let inner = Arc::clone(&self.inner); @@ -1474,88 +1435,6 @@ impl Coordinator { /// 用户在设置里改了 QA 组合键时调用。先持久化(由 prefs.set 完成), /// 然后通知活着的 monitor 重新注册;monitor 不存在时 supervisor 会自然 /// 在下一次循环里读到新的 prefs。 - pub(crate) fn update_qa_hotkey_binding(&self) { - let target = hotkey_runtime_target(&self.inner); - let Some(binding) = target.qa else { - // The user disabled the feature -> drop the monitor directly. The - // drop must happen on the main thread, or Carbon unregister fails - // / is UB. - let inner_clone = Arc::clone(&self.inner); - if self - .inner - .host - .run_on_main_thread(move || { - inner_clone.qa_hotkey.lock().take(); - }) - .is_err() - { - self.inner.qa_hotkey.lock().take(); - } - log::info!("[coord] QA hotkey 已关闭"); - self.update_modifier_shortcut_bindings(); - return; - }; - if crate::shortcut_binding::legacy_modifier_trigger(&binding).is_some() { - let inner_clone = Arc::clone(&self.inner); - if self - .inner - .host - .run_on_main_thread(move || { - inner_clone.qa_hotkey.lock().take(); - }) - .is_err() - { - self.inner.qa_hotkey.lock().take(); - } - self.update_modifier_shortcut_bindings(); - log::info!("[coord] QA hotkey uses modifier-only listener"); - return; - } - self.update_modifier_shortcut_bindings(); - // The global-hotkey crate's manager.register/unregister must run on - // the main thread; off-thread, Carbon registration appears to succeed - // but events are never dispatched. - let inner_clone = Arc::clone(&self.inner); - let binding_for_main = binding.clone(); - if self - .inner - .host - .run_on_main_thread(move || { - // Path 1: a monitor already exists -> swap the binding on the - // main thread. - if let Some(monitor) = inner_clone.qa_hotkey.lock().as_ref() { - if let Err(e) = monitor.update_binding(binding_for_main.clone()) { - log::warn!("[coord] update QA hotkey binding 失败: {e}"); - } - return; - } - // Path 2: not installed yet -> reinstall on the main thread - // (the supervisor also retries, but this way the hotkey takes - // effect as soon as the set_qa_hotkey command returns). - let (tx, rx) = mpsc::channel::(); - match QaHotkeyMonitor::start(binding_for_main, tx) { - Ok(monitor) => { - *inner_clone.qa_hotkey.lock() = Some(monitor); - log::info!( - "[coord] QA hotkey listener installed on main thread (via update)" - ); - let bridge_inner = Arc::clone(&inner_clone); - std::thread::Builder::new() - .name("openless-qa-hotkey-bridge".into()) - .spawn(move || qa_hotkey_bridge_loop(bridge_inner, rx)) - .ok(); - } - Err(e) => { - log::warn!("[coord] update QA hotkey binding 失败: {e}"); - } - } - }) - .is_err() - { - log::warn!("[coord] update QA hotkey binding: AppHandle 未 bind,跳过"); - } - } - pub(crate) fn update_translation_hotkey_binding(&self) { if let Err(e) = self.try_update_translation_hotkey_binding() { log::warn!("[coord] update translation hotkey binding 失败: {e}"); @@ -1642,19 +1521,6 @@ impl Coordinator { } } - /// Label of the current QA shortcut for the frontend Settings (e.g. - /// "Cmd+Shift+;"). Returns an empty string when `qa_hotkey == None`; the UI - /// shows "not enabled" for it. - pub fn qa_hotkey_label(&self) -> String { - self.inner - .backend - .get_preferences() - .qa_hotkey - .as_ref() - .map(|b| b.display_label()) - .unwrap_or_default() - } - /// After saving, syncs style, window size, and click region without /// waiting for the next recording state event. pub fn sync_capsule_style_from_preferences(&self) { @@ -1759,10 +1625,6 @@ impl Coordinator { let previous = &change.previous; let next = &change.next; *self.inner.hotkey_runtime_target.lock() = next.clone(); - if previous.dictation != next.dictation || previous.qa_companion_key != next.qa_companion_key { - crate::recording_companion::configure(next.dictation.clone(), next.qa_companion_key.clone()); - } - if previous.translation != next.translation { self.try_update_translation_hotkey_binding()?; } @@ -1777,9 +1639,6 @@ impl Coordinator { self.try_update_native_dictation_binding()?; self.update_modifier_shortcut_bindings(); } - if previous.qa != next.qa { - self.update_qa_hotkey_binding(); - } if previous.switch_style != next.switch_style { self.update_switch_style_hotkey_binding(); } @@ -1913,85 +1772,7 @@ impl Coordinator { switch_to_previous_style(&self.inner); } - pub async fn open_qa_from_overlay(&self) -> Result<(), String> { - log::info!("[coord] overlay QA open requested"); - self.inner - .backend - .services() - .qa - .start_question() - .await - .map_err(|error| error.message) - } - - pub async fn finalize_qa_from_overlay(&self) -> Result<(), String> { - let Ok(_handoff) = self.inner.overlay_qa_handoff.try_lock() else { - return Ok(()); - }; - let backend = &self.inner.backend; - let snapshot = backend.snapshot().dictation; - if let Some(session_id) = snapshot.session_id { - if !matches!( - snapshot.phase, - openless_core::DictationPhase::Starting | openless_core::DictationPhase::Recording - ) { - return Err("当前语音仍在处理中,请稍后再试".into()); - } - // Capture the source before either transcription or the QA panel - // can change focus. Only owned text/app metadata crosses threads. - let capture = tauri::async_runtime::spawn_blocking(|| { - let (capture, _) = crate::selection::resolve_selection_workspace_capture(); - capture.map(|value| (value.text, value.source_app)) - }) - .await - .map_err(|error| format!("capture QA selection: {error}"))?; - let result = backend - .stop_dictation_for_qa(session_id) - .await - .map_err(|error| error.message)?; - let (selection_text, selection_source_app) = match capture { - Some((text, app)) => (Some(text), app), - None => (None, None), - }; - return backend - .services() - .qa - .submit_captured_text(openless_core::QaInput { - text: result.raw_text, - selection_text, - selection_source_app, - }) - .await - .map_err(|error| error.message); - } - let qa = backend - .services() - .qa - .snapshot() - .await - .map_err(|error| error.message)?; - if let (openless_core::QaPhase::Recording, Some(session_id)) = (qa.phase, qa.session_id) { - backend - .services() - .qa - .stop_recording(session_id) - .await - .map_err(|error| error.message) - } else { - self.open_qa_from_overlay().await - } - } - - /// QA toggle for the CLI entry point: reuses the modifier-only QA hotkey - /// edge handler. Same semantics as `handle_qa_hotkey_pressed` — Idle → - /// open the panel / Recording → finalize / Processing → ignore. A fallback - /// entry point when the desktop shortcut forwards to CLI. - pub async fn cli_toggle_qa_panel(&self) { - handle_qa_hotkey_pressed(&self.inner).await; - } - pub fn set_shortcut_recording_active(&self, active: bool) { - crate::recording_companion::suspend(active); self.inner .shortcut_recording_active .store(active, Ordering::SeqCst); diff --git a/openless-all/app/src-tauri/src/coordinator/dictation_core.rs b/openless-all/app/src-tauri/src/coordinator/dictation_core.rs index 17012a0f4..5e3bb1043 100644 --- a/openless-all/app/src-tauri/src/coordinator/dictation_core.rs +++ b/openless-all/app/src-tauri/src/coordinator/dictation_core.rs @@ -73,14 +73,6 @@ pub(super) async fn handle_pressed_edge( pressed_at: std::time::Instant, press_id: u64, ) { - if crate::recording_companion::claimed() || inner.qa_context.is_panel_visible() { - // Result/recording windows never hijack the recording key into the old - // second-stage follow-up flow. Only the explicit companion chord asks. - if let Ok(qa) = inner.backend.services().qa.snapshot().await { - if crate::recording_companion::claimed() || matches!(qa.phase, - openless_core::QaPhase::Recording | openless_core::QaPhase::Thinking | openless_core::QaPhase::AwaitingApproval) { return; } - } - } match dispatch( inner, openless_core::DictationHotkeyEdge::Pressed { @@ -100,11 +92,6 @@ pub(super) async fn handle_released_edge( released_at: std::time::Instant, press_id: u64, ) { - if inner.qa_context.is_panel_visible() { - if let Ok(qa) = inner.backend.services().qa.snapshot().await { - if matches!(qa.phase, openless_core::QaPhase::Recording | openless_core::QaPhase::Thinking | openless_core::QaPhase::AwaitingApproval) { return; } - } - } match dispatch( inner, openless_core::DictationHotkeyEdge::Released { diff --git a/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs b/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs index 780b4a509..326278117 100644 --- a/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs +++ b/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs @@ -176,149 +176,6 @@ pub(super) fn hotkey_supervisor_loop(inner: Arc) { } } -// ─────────────────────────── QA hotkey supervisor ─────────────────────────── - -fn take_qa_hotkey_on_main_thread(inner: &Arc) { - let main = Arc::clone(inner); - if let Err(error) = inner.host.run_on_main_thread(move || { - main.qa_hotkey.lock().take(); - }) { - log::warn!("[qa] cannot dispatch hotkey removal: {error}"); - } -} - -pub(super) fn qa_hotkey_supervisor_loop(inner: Arc) { - let mut attempts: u32 = 0; - loop { - if inner.shutdown.load(Ordering::SeqCst) { - return; - } - let registration = - crate::issue1266_debug::lock_host(&inner.settings_host_gate, DebugKind::QaHotkey); - if inner.backend.ensure_runtime_ready().is_err() { - drop(registration); - std::thread::sleep(std::time::Duration::from_secs(1)); - continue; - } - - // QA disabled: sleep until the runtime target changes; changes arrive - // via the explicit settings effect. - let binding = match hotkey_runtime_target(&inner).qa { - Some(b) => b, - None => { - take_qa_hotkey_on_main_thread(&inner); - drop(registration); - std::thread::sleep(std::time::Duration::from_secs(5)); - continue; - } - }; - if crate::shortcut_binding::legacy_modifier_trigger(&binding).is_some() { - take_qa_hotkey_on_main_thread(&inner); - if let Some(monitor) = inner.hotkey.lock().as_ref() { - let (qa_trigger, selection_polish_trigger, translation_trigger) = - modifier_shortcut_triggers(&inner); - monitor.update_modifier_shortcuts( - qa_trigger, - selection_polish_trigger, - translation_trigger, - ); - } - drop(registration); - std::thread::sleep(std::time::Duration::from_secs(5)); - continue; - } - - if inner.qa_hotkey.lock().is_some() { - // Already registered -> do not reinstall; recheck after 5s (binding - // changes go through the manual update path). - drop(registration); - std::thread::sleep(std::time::Duration::from_secs(5)); - continue; - } - - // On macOS the global-hotkey crate uses Carbon RegisterEventHotKey and - // requires the manager to be constructed on the main thread; otherwise - // register() looks Ok but events are never dispatched (the step missed - // by the first version of issue #118 / PR #119, leaving hotkey presses - // completely dead). Run QaHotkeyMonitor::start on the main thread via - // run_on_main_thread and receive the result back on the channel. - let (tx, rx) = mpsc::channel::(); - let (init_tx, init_rx) = mpsc::sync_channel::>(0); - let binding_for_main = binding.clone(); - if inner - .host - .run_on_main_thread(move || { - let result = QaHotkeyMonitor::start(binding_for_main, tx); - let _ = init_tx.send(result); - }) - .is_err() - { - drop(registration); - std::thread::sleep(std::time::Duration::from_secs(1)); - continue; - } - - // run_on_main_thread is fire-and-forget; wait for the main thread to - // send the result back, capped at 5s so a busy main thread cannot block - // the supervisor forever. - let init_result = match init_rx.recv_timeout(std::time::Duration::from_secs(5)) { - Ok(r) => r, - Err(_) => { - drop(init_rx); - attempts += 1; - if attempts <= 3 || attempts % 10 == 0 { - log::warn!( - "[coord] QA hotkey 第 {attempts} 次注册超时(主线程未回执);3s 后重试" - ); - } - drop(registration); - std::thread::sleep(std::time::Duration::from_secs(3)); - continue; - } - }; - - match init_result { - Ok(monitor) => { - *inner.qa_hotkey.lock() = Some(monitor); - log::info!( - "[coord] QA hotkey listener installed on main thread (after {} attempt(s))", - attempts + 1 - ); - let inner_clone = Arc::clone(&inner); - std::thread::Builder::new() - .name("openless-qa-hotkey-bridge".into()) - .spawn(move || qa_hotkey_bridge_loop(inner_clone, rx)) - .ok(); - attempts = 0; - } - Err(e) => { - attempts += 1; - if attempts <= 3 || attempts % 10 == 0 { - log::warn!("[coord] QA hotkey 第 {attempts} 次注册失败: {e}; 3s 后重试"); - } - drop(registration); - std::thread::sleep(std::time::Duration::from_secs(3)); - } - } - } -} - -pub(super) fn qa_hotkey_bridge_loop(inner: Arc, rx: mpsc::Receiver) { - while let Ok(evt) = rx.recv() { - if inner.shortcut_recording_active.load(Ordering::SeqCst) { - continue; - } - let inner_cloned = Arc::clone(&inner); - match evt { - QaHotkeyEvent::Pressed => { - inner - .host - .spawn(async move { handle_qa_hotkey_pressed(&inner_cloned).await }); - } - } - } -} - // ─────────────────────── Selection Polish hotkey ─────────────────────── // Selection polish is a desktop (Windows-first) workflow; mobile registers no // global hotkey. @@ -2026,10 +1883,6 @@ pub(super) fn modifier_shortcut_triggers( Option, ) { let target = hotkey_runtime_target(inner); - let qa_trigger = target - .qa - .as_ref() - .and_then(crate::shortcut_binding::legacy_modifier_trigger); let translation_trigger = if is_builtin_translation_shift(&target.translation) { None } else { @@ -2039,7 +1892,7 @@ pub(super) fn modifier_shortcut_triggers( .selection_polish .as_ref() .and_then(crate::shortcut_binding::legacy_modifier_trigger); - (qa_trigger, selection_polish_trigger, translation_trigger) + (None, selection_polish_trigger, translation_trigger) } /// Decides "will translation actually happen" here, not on the read side: @@ -2228,11 +2081,7 @@ pub(super) fn hotkey_bridge_loop(inner: Arc, rx: mpsc::Receiver { - inner.host.block_on(async { - handle_qa_hotkey_pressed(&inner_cloned).await; - }); - } + HotkeyEvent::QaShortcutPressed => {} #[cfg(not(mobile))] HotkeyEvent::SelectionPolishShortcutPressed => { handle_selection_workspace_hotkey_pressed(&inner_cloned); @@ -2267,15 +2116,6 @@ pub(super) fn reset_shortcut_held_state(inner: &Arc) { monitor.reset_held_state(); } let target = hotkey_runtime_target(inner); - if let Some(binding) = target.qa.as_ref() { - if crate::shortcut_binding::legacy_modifier_trigger(binding).is_none() { - if let Some(monitor) = inner.qa_hotkey.lock().as_ref() { - if let Err(e) = monitor.update_binding(binding.clone()) { - log::warn!("[coord] reset QA hotkey latch failed: {e}"); - } - } - } - } if !is_builtin_translation_shift(&target.translation) && crate::shortcut_binding::legacy_modifier_trigger(&target.translation).is_none() { @@ -2490,7 +2330,6 @@ pub(crate) mod less_computer_test_support { less_computer_voice: Mutex::new(None), settings_host_gate: Mutex::new(()), hotkey_resume_started: AtomicBool::new(false), - overlay_qa_handoff: tokio::sync::Mutex::new(()), inserter: TextInserter::new(), vocab_card_visible: AtomicBool::new(false), vocab_card_epoch: crate::vocab_card_placement::PresentationEpoch::default(), @@ -2518,7 +2357,6 @@ pub(crate) mod less_computer_test_support { voice_edit_host: Arc::new( Mutex::new(voice_edit_session::VoiceEditHostState::default()), ), - qa_hotkey: Mutex::new(None), coding_agent_modifier_hotkey: Mutex::new(None), coding_agent_combo_hotkey: Mutex::new(None), last_capsule_state: Mutex::new(None), diff --git a/openless-all/app/src-tauri/src/coordinator/qa.rs b/openless-all/app/src-tauri/src/coordinator/qa.rs deleted file mode 100644 index 302cbdd5c..000000000 --- a/openless-all/app/src-tauri/src/coordinator/qa.rs +++ /dev/null @@ -1,50 +0,0 @@ -use super::Inner; -use std::sync::Arc; - -pub(super) fn install_recording_companion(inner: &Arc) { - let target = inner.hotkey_runtime_target.lock().clone(); - crate::recording_companion::configure(target.dictation, target.qa_companion_key); - let weak = Arc::downgrade(inner); - crate::recording_companion::install(Arc::new(move || { - let Some(inner) = weak.upgrade() else { - return; - }; - if inner - .shortcut_recording_active - .load(std::sync::atomic::Ordering::SeqCst) - { - return; - } - let task = Arc::clone(&inner); - let _ = inner.host.spawn(async move { - handle_qa_hotkey_pressed(&task).await; - }); - })); -} - -/// One chord owns the complete question: start → record → finish. Processing -/// never queues another question. Later turns retain the conversation until dismissal. -pub(super) async fn handle_qa_hotkey_pressed(inner: &Arc) { - let Ok(_handoff) = inner.overlay_qa_handoff.try_lock() else { - return; - }; - let qa = &inner.backend.services().qa; - let Ok(snapshot) = qa.snapshot().await else { - return; - }; - let result = match (snapshot.phase, snapshot.session_id) { - (openless_core::QaPhase::Recording, Some(session_id)) => { - qa.stop_recording(session_id).await - } - (openless_core::QaPhase::Thinking | openless_core::QaPhase::AwaitingApproval, _) => return, - _ => { - inner - .backend - .start_voice_question_from_recording_key() - .await - } - }; - if let Err(error) = result { - log::warn!("[ask] question hotkey failed: {error}"); - } -} diff --git a/openless-all/app/src-tauri/src/coordinator/restore_runtime.rs b/openless-all/app/src-tauri/src/coordinator/restore_runtime.rs index 88fbeeb42..8b074705d 100644 --- a/openless-all/app/src-tauri/src/coordinator/restore_runtime.rs +++ b/openless-all/app/src-tauri/src/coordinator/restore_runtime.rs @@ -163,7 +163,6 @@ fn reconcile_hotkeys( target: openless_core::HotkeyRuntimeTarget, ) -> Result<(), BackendError> { *coord.inner.hotkey_runtime_target.lock() = target.clone(); - crate::recording_companion::configure(target.dictation.clone(), target.qa_companion_key.clone()); let trigger = crate::shortcut_binding::legacy_modifier_trigger(&target.dictation); let binding = crate::types::HotkeyBinding { trigger: trigger.unwrap_or(crate::types::HotkeyTrigger::Custom), @@ -206,7 +205,6 @@ fn reconcile_hotkeys_on_main( // Release old combinations together before applying an absolute target; // this also permits swapping two bindings during a restore. inner.combo_hotkey.lock().take(); - inner.qa_hotkey.lock().take(); inner.translation_hotkey.lock().take(); inner.switch_style_hotkey.lock().take(); inner.open_app_hotkey.lock().take(); @@ -257,20 +255,6 @@ fn reconcile_hotkeys_on_main( .map_err(|error| error.to_string())?; *inner.combo_hotkey.lock() = Some(monitor); } - if let Some(binding) = target.qa.as_ref().filter(|binding| { - crate::shortcut_binding::legacy_modifier_trigger(binding).is_none() - && !is_unconfigured_shortcut(binding) - }) { - let (send, receive) = mpsc::channel(); - let monitor = - QaHotkeyMonitor::start(binding.clone(), send).map_err(|error| error.to_string())?; - let owned = Arc::clone(inner); - std::thread::Builder::new() - .name("openless-qa-hotkey-bridge".into()) - .spawn(move || qa_hotkey_bridge_loop(owned, receive)) - .map_err(|error| error.to_string())?; - *inner.qa_hotkey.lock() = Some(monitor); - } if !is_builtin_translation_shift(&target.translation) && crate::shortcut_binding::legacy_modifier_trigger(&target.translation).is_none() && !is_unconfigured_shortcut(&target.translation) diff --git a/openless-all/app/src-tauri/src/coordinator/selection_voice_session.rs b/openless-all/app/src-tauri/src/coordinator/selection_voice_session.rs index db6e330ae..41a131039 100644 --- a/openless-all/app/src-tauri/src/coordinator/selection_voice_session.rs +++ b/openless-all/app/src-tauri/src/coordinator/selection_voice_session.rs @@ -615,6 +615,9 @@ async fn continue_selection_voice_disposition( inner: &Arc, disposition: SelectionVoiceDisposition, ) -> Result { + if matches!(&disposition, SelectionVoiceDisposition::Question { .. }) { + return Err("随便问暂不可用,未更改选中的内容".into()); + } let route = inner .backend .services() diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index a72765215..388211b86 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -5705,23 +5705,16 @@ impl HostActions for TauriHostActions { match action { HostAction::ShowMain | HostAction::FocusMain => crate::show_main_window(&app), HostAction::ShowDictationFeedback | HostAction::HideDictationFeedback => {} - HostAction::ShowSelectionPreview => crate::show_selection_polish_preview(&app), - HostAction::HideSelectionPreview => crate::hide_selection_polish_preview(&app), + HostAction::ShowSelectionPreview => { + return Err(BackendError::new(BackendErrorCode::Unsupported, "Selection polish preview is unavailable")); + } + HostAction::HideSelectionPreview => {} HostAction::ShowQa => { - let was_visible = self.qa_context.is_panel_visible() - && app - .get_webview_window("qa") - .is_some_and(|window| window.is_visible().unwrap_or(false)); - self.qa_context.set_voice_surface_active(true); - self.qa_context.prepare_show(); - if !was_visible { - crate::show_qa_window(&app, "idle"); - } + return Err(BackendError::new(BackendErrorCode::Unsupported, "Ask is unavailable")); } HostAction::HideQa => { self.qa_context.set_voice_surface_active(false); self.qa_context.clear(); - crate::hide_qa_window(&app); } HostAction::ShowLessComputer => { crate::show_less_computer_window(&app).map_err(map_tauri_error)?; diff --git a/openless-all/app/src-tauri/src/hotkey.rs b/openless-all/app/src-tauri/src/hotkey.rs index 8c200a5d5..38a639906 100644 --- a/openless-all/app/src-tauri/src/hotkey.rs +++ b/openless-all/app/src-tauri/src/hotkey.rs @@ -768,7 +768,6 @@ mod platform { match event_type { TAP_DISABLED_BY_TIMEOUT | TAP_DISABLED_BY_USER_INPUT => { - crate::recording_companion::reset_held(); if let Some(tap) = *ctx.handles.tap.lock().unwrap() { unsafe { CGEventTapEnable(tap, true) }; } diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index 52e621c03..7b91ce796 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -49,7 +49,6 @@ mod native_audio_cue; #[cfg(not(mobile))] mod native_capsule; mod qa_adapter; -mod recording_companion; mod tauri_coordinator_host; // 托盘麦克风设备变更监听:macOS CoreAudio / Windows MMDevice 原生通知(空闲零唤醒), // 仅桌面端。详见 issue #470。 @@ -87,10 +86,7 @@ mod permissions; mod persistence; mod polish; #[cfg(not(mobile))] -mod qa_hotkey; -#[cfg(mobile)] -#[path = "mobile_stubs/qa_hotkey.rs"] -mod qa_hotkey; +mod ask_window; mod recorder; #[cfg(not(mobile))] mod remote_server; @@ -141,44 +137,14 @@ const LOG_ROTATE_LIMIT_BYTES: u64 = 10 * 1024 * 1024; #[cfg(target_os = "macos")] const OPENLESS_BUNDLE_ID: &str = "com.openless.app"; -/// Positions the QA window at the bottom-center of the screen on first show only, so the -/// user-dragged position survives hide → show cycles. See issue #118 v2. -static QA_WINDOW_POSITIONED: AtomicBool = AtomicBool::new(false); -/// 「润色结果」模式的事件名。该模式复用选区助手(qa)面板,不再有独立预览窗。 -#[cfg(not(any(target_os = "android", target_os = "ios")))] -pub(crate) const SELECTION_POLISH_PREVIEW_SHOWN: &str = "selection-polish-preview:shown"; -/// 让面板自行决定退出「润色结果」模式(它可能正处在提问对话中)。 -#[cfg(not(any(target_os = "android", target_os = "ios")))] -pub(crate) const SELECTION_POLISH_PREVIEW_HIDE: &str = "selection-polish-preview:hide"; -/// 是否处于「润色结果」模式。`get_selection_polish_preview` 只在它为 true 时 -/// 返回负载:面板懒创建后挂载时也会拉一次,不能把残留的快照当成新请求。 -static SELECTION_POLISH_PREVIEW_PENDING: AtomicBool = AtomicBool::new(false); - -pub(crate) fn selection_polish_preview_pending() -> bool { - SELECTION_POLISH_PREVIEW_PENDING.load(Ordering::SeqCst) -} - -pub(crate) fn clear_selection_polish_preview_pending() { - SELECTION_POLISH_PREVIEW_PENDING.store(false, Ordering::SeqCst); -} - #[cfg(target_os = "macos")] static LESS_COMPUTER_WINDOW_POSITIONED: AtomicBool = AtomicBool::new(false); /// 聊天面板退场动画的世代计数:hide 先发 `chat-panel:closing` 让前端播 220ms /// 退场动画、240ms 后才真正 hide;期间再次 show 会推进世代,作废挂起的 hide。 -static QA_PANEL_EPOCH: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); static LESS_COMPUTER_PANEL_EPOCH: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); static VOICE_EDIT_PANEL_EPOCH: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); -#[cfg(not(any(target_os = "android", target_os = "ios")))] -static QA_WINDOW_CREATE_LOCK: std::sync::OnceLock> = - std::sync::OnceLock::new(); - -#[cfg(not(any(target_os = "android", target_os = "ios")))] -fn qa_window_create_lock() -> &'static parking_lot::Mutex<()> { - QA_WINDOW_CREATE_LOCK.get_or_init(|| parking_lot::Mutex::new(())) -} #[cfg(not(mobile))] static TRAY_MICROPHONE_WATCHER_STOPPING: AtomicBool = AtomicBool::new(false); @@ -391,6 +357,7 @@ macro_rules! app_invoke_handler_desktop { commands::cancel_voice_edit_session, commands::get_voice_edit_state, commands::voice_edit_window_open, + ask_window::ask_window_open, commands::voice_edit_window_close, commands::set_capsule_transcript_visible, #[cfg(target_os = "macos")] @@ -400,12 +367,6 @@ macro_rules! app_invoke_handler_desktop { commands::inject_hotkey_click_for_dev, #[cfg(debug_assertions)] commands::run_selection_polish_for_dev, - #[cfg(not(mobile))] - commands::get_selection_polish_preview, - #[cfg(not(mobile))] - commands::confirm_selection_polish_preview, - #[cfg(not(mobile))] - commands::cancel_selection_polish_preview, commands::repolish, commands::list_style_packs, commands::create_style_pack_from_template, @@ -440,8 +401,6 @@ macro_rules! app_invoke_handler_desktop { commands::reorder_channels, commands::record_channel_test, commands::set_active_omni_provider, - commands::get_qa_hotkey_label, - commands::set_qa_hotkey, commands::set_selection_polish_hotkey, #[cfg(all(not(mobile), target_os = "windows"))] commands::get_selection_voice_intent_prompt, @@ -463,16 +422,6 @@ macro_rules! app_invoke_handler_desktop { commands::set_open_app_hotkey, commands::set_quick_note_hotkey, commands::set_style_pack_hotkeys, - commands::qa_window_dismiss, - commands::qa_cancel_turn, - commands::qa_stop_recording, - commands::qa_window_set_expanded, - commands::qa_get_snapshot, - commands::qa_toggle_recording, - commands::qa_submit_text, - commands::deepseek_tts_start, - commands::deepseek_tts_stop, - commands::qa_set_edit_instruction_mode, commands::chat_panel_focus_keyboard, commands::validate_combo_hotkey, commands::set_combo_hotkey, @@ -702,16 +651,6 @@ macro_rules! app_invoke_handler_mobile { $crate::commands::get_voice_edit_state, $crate::commands::voice_edit_window_open, $crate::commands::voice_edit_window_close, - $crate::commands::qa_window_dismiss, - $crate::commands::qa_cancel_turn, - $crate::commands::qa_stop_recording, - $crate::commands::qa_window_set_expanded, - $crate::commands::qa_get_snapshot, - $crate::commands::qa_toggle_recording, - $crate::commands::qa_submit_text, - $crate::commands::deepseek_tts_start, - $crate::commands::deepseek_tts_stop, - $crate::commands::qa_set_edit_instruction_mode, $crate::commands::repolish, $crate::commands::list_style_packs, $crate::commands::create_style_pack_from_template, @@ -960,11 +899,7 @@ fn run_desktop() { let _ = capsule.hide(); } - // QA / Less Computer windows are lazily created (not eagerly declared in - // tauri.conf.json): they are built on first use (ensure_qa_window / - // ensure_less_computer_window), so there is no extra WebKit process while idle - // — saving 3 resident webviews. Positioning and the QA drag fix are applied in - // the creation/show paths. + // Auxiliary windows are created only on demand. // Main-window material: Mica on Windows. macOS deliberately applies no // NSVisualEffectView: the React shell (WindowChrome, --ol-window-bg) paints a fully @@ -1156,7 +1091,6 @@ fn run_desktop() { let coordinator = app.state::>(); coordinator.tauri_host().stop_native_capsule(); coordinator.stop_hotkey_listener(); - coordinator.stop_qa_hotkey_listener(); coordinator.stop_selection_polish_hotkey_listener(); coordinator.stop_coding_agent_hotkey_listener(); coordinator.stop_combo_hotkey_listener(); @@ -1190,7 +1124,6 @@ fn observe_issue1266_lifecycle(event: &RunEvent) { }; let kind = match label.as_str() { "main" => Kind::MainWindow, - "qa" => Kind::QaWindow, "capsule" => Kind::CapsuleWindow, "less-computer" => Kind::LessWindow, _ => Kind::OtherWindow, @@ -2124,7 +2057,6 @@ pub(crate) fn show_main_window(app: &AppHandle) { /// Async actions spawn on the Tauri runtime and never block the callback thread: /// - ToggleDictation enters the shared Core facade; CancelDictation releases the Less Host /// capture first, then cancels Core -/// - ToggleQa forwards to handle_qa_hotkey_pressed (same semantics as pressing the QA hotkey) fn dispatch_cli_intent(app: &AppHandle, intent: cli::CliIntent) { match intent { cli::CliIntent::ToggleDictation => { @@ -2162,20 +2094,7 @@ fn dispatch_cli_intent(app: &AppHandle, intent: cli::CliIntent) { } }); } - cli::CliIntent::ToggleQa => { - let coordinator = app - .try_state::>() - .map(|state| Arc::clone(&*state)); - let Some(coordinator) = coordinator else { - log::warn!("[cli] coordinator not yet managed; dropping QA intent"); - return; - }; - let coord = Arc::clone(&coordinator); - tauri::async_runtime::spawn(async move { - log::info!("[cli] toggle-qa: dispatching to qa hotkey handler"); - coord.cli_toggle_qa_panel().await; - }); - } + cli::CliIntent::ToggleQa => log::info!("[cli] Ask is unavailable"), } } @@ -2316,18 +2235,6 @@ fn wait_for_app_activation(app: &AppHandle) { #[cfg(not(target_os = "macos"))] fn wait_for_app_activation(_app: &AppHandle) {} -/// QA records through the shared capsule and keeps its separate answer card open. -const QA_WINDOW_WIDTH: f64 = 480.0; -const QA_WINDOW_HEIGHT: f64 = 80.0; -const QA_WINDOW_EXPANDED_WIDTH: f64 = 720.0; -const QA_WINDOW_EXPANDED_HEIGHT: f64 = 380.0; -/// Reserve room for the independent recording capsule and its context badge. -const QA_WINDOW_BOTTOM_INSET: f64 = 96.0; -/// Expanded answers sit higher so the lower composer does not crowd the recording capsule. -const QA_WINDOW_EXPANDED_BOTTOM_INSET: f64 = 132.0; -const QA_RESULT_BOTTOM_LIFT: f64 = 40.0; -/// Bottom margin reserved for the macOS Dock (same source as the capsule). - #[derive(Clone, Copy, Debug, PartialEq)] struct LogicalMonitorFrame { x: f64, @@ -2846,225 +2753,7 @@ fn clamp_to_monitor( (clamped_x, clamped_y) } -/// Places the QA window at the bottom-center of the screen just above the capsule. Called -/// once during Tauri startup and before each show, so positions stay correct after the user -/// switches monitors. -fn position_qa_window(window: &tauri::WebviewWindow) -> tauri::Result<()> { - let Some((frame, scale)) = chat_window_work_area(window, true)? else { - return Ok(()); - }; - let (x, y) = bottom_center_position( - frame, - QA_WINDOW_WIDTH, - QA_WINDOW_HEIGHT, - QA_WINDOW_BOTTOM_INSET, - ); - window.set_position(tauri::PhysicalPosition::new( - (x * scale).round() as i32, - (y * scale).round() as i32, - ))?; - window.set_size(tauri::PhysicalSize::new( - (QA_WINDOW_WIDTH * scale).round() as u32, - (QA_WINDOW_HEIGHT * scale).round() as u32, - ))?; - Ok(()) -} - -// Preserve a dragged window's centre and baseline; repeated result events must not drift. -fn qa_resize_origin( - x: f64, - y: f64, - old_width: f64, - old_height: f64, - width: f64, - height: f64, - was_expanded: bool, - expanded: bool, -) -> (f64, f64) { - let lift = if expanded { QA_RESULT_BOTTOM_LIFT } else { 0.0 } - - if was_expanded { - QA_RESULT_BOTTOM_LIFT - } else { - 0.0 - }; - ( - x + (old_width - width) / 2.0, - y + old_height - height - lift, - ) -} - -/// Called on the native main thread by the restricted QA command. The compact -/// state has a genuinely small native frame, so hidden content cannot eat clicks. -pub(crate) fn set_qa_window_expanded( - app: &AppHandle, - expanded: bool, -) -> Result<(), String> { - let window = app - .get_webview_window("qa") - .ok_or_else(|| "qa_window_unavailable".to_string())?; - let area = chat_window_work_area(&window, false).map_err(|_| "qa_geometry_unavailable")?; - let scale = area.map(|(_, scale)| scale).unwrap_or( - window - .scale_factor() - .map_err(|_| "qa_geometry_unavailable")?, - ); - let position = window - .inner_position() - .map_err(|_| "qa_geometry_unavailable")? - .to_logical::(scale); - let current_size = window - .inner_size() - .map_err(|_| "qa_geometry_unavailable")? - .to_logical::(scale); - let was_expanded = current_size.height > QA_WINDOW_HEIGHT + 1.0; - let mut width = if expanded { - QA_WINDOW_EXPANDED_WIDTH - } else { - QA_WINDOW_WIDTH - }; - let mut height = if expanded { - QA_WINDOW_EXPANDED_HEIGHT - } else { - QA_WINDOW_HEIGHT - }; - if let Some((frame, _)) = area { - width = width.min((frame.width - 32.0).max(240.0)); - height = height.min((frame.height - 64.0).max(QA_WINDOW_HEIGHT)); - } - let (mut x, mut y) = qa_resize_origin( - position.x, - position.y, - current_size.width, - current_size.height, - width, - height, - was_expanded, - expanded, - ); - if let Some((frame, _)) = area { - x = x.clamp( - frame.x + 16.0, - (frame.x + frame.width - width - 16.0).max(frame.x + 16.0), - ); - let bottom_inset = if expanded { - QA_WINDOW_EXPANDED_BOTTOM_INSET - } else { - QA_WINDOW_BOTTOM_INSET - }; - y = y.clamp( - frame.y + 32.0, - (frame.y + frame.height - height - bottom_inset).max(frame.y + 32.0), - ); - } - window - .set_position(tauri::PhysicalPosition::new( - (x * scale).round() as i32, - (y * scale).round() as i32, - )) - .map_err(|_| "qa_position_failed")?; - window - .set_size(tauri::PhysicalSize::new( - (width * scale).round() as u32, - (height * scale).round() as u32, - )) - .map_err(|_| "qa_resize_failed")?; - Ok(()) -} - -/// Shows the QA window and emits a state event (the frontend subscribes to `qa:state`). -/// `content_kind` is an opaque string ("loading" / "answer" / "idle", etc.); the frontend -/// React view decides which to render. **Never** steals focus from the frontmost app -/// (keeps the Cmd+C fallback able to read the selection from the original app). -pub(crate) fn show_qa_window(app: &AppHandle, content_kind: &str) { - #[cfg(target_os = "android")] - { - match crate::android::jni::android::with_android_env(|env, context| { - crate::android::jni::android::open_qa_host(env, context) - }) { - Ok(()) => log::info!("[qa] android requested WarmupActivity foreground for QA"), - Err(error) => log::warn!("[qa] android failed to foreground WarmupActivity: {error}"), - } - log::info!("[qa] android publish qa:state to main kind={content_kind}"); - tauri_events::publish( - app, - None, - openless_core::BackendEventKind::QaState(openless_core::QaStateEvent::simple( - openless_core::QaStateKind::Idle, - )), - ); - return; - } - - let Some(window) = ensure_qa_window(app) else { - log::info!("[qa] show 跳过:qa 窗口不存在 (content_kind={content_kind})"); - return; - }; - // Center only on first show; afterwards keep the user-dragged position. - if !QA_WINDOW_POSITIONED.load(Ordering::Relaxed) { - if let Err(e) = position_qa_window(&window) { - log::warn!("[qa] position before first show failed: {e}"); - } - QA_WINDOW_POSITIONED.store(true, Ordering::Relaxed); - } - // macOS: don't use window.show() (it does makeKeyAndOrderFront, pushing OpenLess to - // frontmost; the subsequent capture_selection AX read / Cmd+C fallback would then run - // against OpenLess's own webview → can't grab the original app's selection). Use - // orderFrontRegardless so the window is visible but **not** key; the user's app stays - // frontmost and AX can still read the selection. Standard Spotlight / Raycast approach. - // - // ⚠️ Critical: every NSWindow operation must run on the main thread; macOS 26 asserts - // this hard (violations SIGTRAP immediately). show_qa_window is often called from a - // tokio worker (qa_hotkey_bridge_loop), so raw ObjC msg_send must be dispatched via - // `app.run_on_main_thread`. See issue #118 v2. - #[cfg(target_os = "macos")] - { - let window_clone = window.clone(); - let _ = app.run_on_main_thread(move || { - use objc2::msg_send; - use objc2::runtime::AnyObject; - match window_clone.ns_window() { - Ok(handle) => { - let ns = handle as *mut AnyObject; - if ns.is_null() { - log::warn!("[qa] ns_window null; falling back to window.show()"); - let _ = window_clone.show(); - } else { - unsafe { - let _: () = msg_send![ns, orderFrontRegardless]; - } - } - } - Err(e) => { - log::warn!("[qa] ns_window unavailable: {e}; falling back to window.show()"); - let _ = window_clone.show(); - } - } - }); - } - #[cfg(target_os = "windows")] - if !show_qa_window_no_activate(&window) { - log::warn!("[qa] show_no_activate failed; falling back to window.show()"); - if let Err(e) = window.show() { - log::warn!("[qa] show fallback failed: {e}"); - } - } - #[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] - if let Err(e) = window.show() { - log::warn!("[qa] show failed: {e}"); - } - // Cancel the pending exit-hide (fast close-open) and replay the entrance animation. - QA_PANEL_EPOCH.fetch_add(1, Ordering::SeqCst); - let _ = app.emit_to("qa", "chat-panel:shown", serde_json::json!({})); - tauri_events::publish( - app, - None, - openless_core::BackendEventKind::QaState(openless_core::QaStateEvent::simple( - openless_core::QaStateKind::Idle, - )), - ); -} - -/// Converts the chat-panel floating windows (qa / less-computer) into non-activating +/// Converts the floating preview and Less Computer windows into non-activating /// NSPanels (macOS; same technique as the capsule). /// /// Goal: the panel can become key window **without OpenLess being activated** (Spotlight @@ -3094,9 +2783,6 @@ fn make_chat_window_panel_macos(window: &tauri::WebviewWindow // Floating level (NSFloatingWindowLevel): above normal windows, below the // menu bar / capsule (25). panel.set_level(3); - if tag == "qa" { - panel.set_hides_on_deactivate(false); - } panel.set_collection_behaviour( NSWindowCollectionBehavior::NSWindowCollectionBehaviorFullScreenAuxiliary | NSWindowCollectionBehavior::NSWindowCollectionBehaviorCanJoinAllSpaces, @@ -3154,67 +2840,6 @@ fn make_chat_window_draggable_macos( } } -/// Lazily creates the QA window: it used to be created eagerly in tauri.conf.json (keeping -/// a resident WebKit process); now it is built on first show — nonexistent while idle → -/// saves one resident webview. Config matches the old tauri.conf qa block item by item -/// ("center": false ⇒ **don't** call .center(); "focus": false ⇒ focused(false)). -/// Key: make_qa_window_draggable_macos used to run once at startup; it must be re-applied -/// at creation time here, otherwise the lazily created QA window can't be dragged on macOS. -#[cfg(not(any(target_os = "android", target_os = "ios")))] -fn ensure_qa_window(app: &AppHandle) -> Option> { - // ShowQa can arrive concurrently from independent Core workflows. Keep the existing-window - // lookup and first build in one critical section so two callers cannot create two qa HWNDs. - let _creation_guard = qa_window_create_lock().lock(); - if let Some(w) = app.get_webview_window("qa") { - return Some(w); - } - let built = - WebviewWindowBuilder::new(app, "qa", WebviewUrl::App("index.html?window=qa".into())) - .title("OpenLess QA") - .inner_size(QA_WINDOW_WIDTH, QA_WINDOW_HEIGHT) - .decorations(false) - .transparent(true) - .shadow(true) - .always_on_top(true) - .skip_taskbar(true) - .resizable(false) - .focused(false) - .visible(false) - .accept_first_mouse(true) - .build(); - match built { - Ok(w) => { - // ⚠️ NSWindow operations must run on the main thread (macOS 26 hard - // requirement). ensure_qa_window is often entered from a tokio worker (hotkey - // bridge loop); calling setMovable directly on the worker thread makes AppKit - // throw an NSException from _postWindowNeedsToResetDragMargins and crash the - // whole app (root cause of four same-stack crashes on 2026-07-03) — dispatch - // back to the main thread. - #[cfg(target_os = "macos")] - { - let w_clone = w.clone(); - let _ = app.run_on_main_thread(move || { - make_chat_window_panel_macos(&w_clone, "qa"); - make_chat_window_draggable_macos(&w_clone, "qa"); - }); - } - Some(w) - } - Err(e) => { - log::warn!("[qa] lazy window create failed: {e}"); - None - } - } -} - -// Mobile routes QA to the main window (show_qa_window returns early on Android); Android's -// WebviewWindowBuilder lacks the desktop methods, so this stub just returns the existing -// window (compiles, but unreachable at runtime). -#[cfg(any(target_os = "android", target_os = "ios"))] -fn ensure_qa_window(app: &AppHandle) -> Option> { - app.get_webview_window("qa") -} - /// Lazily creates the Less Computer window. On macOS it is additionally converted into a /// focus-stealing-free NSPanel. #[cfg(target_os = "macos")] @@ -3326,42 +2951,6 @@ fn hide_chat_window_animated( }); } -/// Hides the QA window. Shared by commands::qa_window_dismiss and coordinator session teardown. -pub(crate) fn hide_qa_window(app: &AppHandle) { - #[cfg(target_os = "android")] - { - let _ = app.emit_to("main", "qa:dismiss", serde_json::json!({})); - return; - } - - #[cfg(not(any(target_os = "android", target_os = "ios")))] - hide_chat_window_animated(app, "qa", &QA_PANEL_EPOCH); -} - -/// 「润色结果」不再有独立窗口:复用选区助手(qa)面板切到润色模式。核心在 -/// 「预览确认」输出模式下照旧只发 `HostAction::ShowSelectionPreview`。 -#[cfg(not(any(target_os = "android", target_os = "ios")))] -pub(crate) fn show_selection_polish_preview(app: &AppHandle) { - SELECTION_POLISH_PREVIEW_PENDING.store(true, Ordering::SeqCst); - show_qa_window(app, "polish-preview"); - // 面板可能是本次才懒创建(刚挂载,订阅还没装上)——它在挂载时也会自己 - // 拉一次负载(见 `get_selection_polish_preview` 的 pending 门禁)。 - let _ = app.emit_to("qa", SELECTION_POLISH_PREVIEW_SHOWN, ()); -} - -#[cfg(any(target_os = "android", target_os = "ios"))] -pub(crate) fn show_selection_polish_preview(_app: &AppHandle) {} - -/// 退出「润色结果」模式。是否顺带关掉面板交给前端:面板可能正处在提问对话中, -/// 不能被润色流程的收尾动作一起关掉(与 egui 侧 `HideSelectionPreview` 一致)。 -pub(crate) fn hide_selection_polish_preview(app: &AppHandle) { - clear_selection_polish_preview_pending(); - #[cfg(any(target_os = "android", target_os = "ios"))] - let _ = app; - #[cfg(not(any(target_os = "android", target_os = "ios")))] - let _ = app.emit_to("qa", SELECTION_POLISH_PREVIEW_HIDE, ()); -} - const VOICE_EDIT_WINDOW_WIDTH: f64 = 480.0; const VOICE_EDIT_WINDOW_HEIGHT: f64 = 620.0; @@ -3719,46 +3308,6 @@ pub(crate) fn hide_less_computer_glow(app: &AppHandle) { #[cfg(not(target_os = "macos"))] pub(crate) fn hide_less_computer_glow(_app: &AppHandle) {} -/// Hands focus back to the QA window after the selection is captured (second half of the -/// Windows focus-dance). Called by begin_qa_session when capture_selection finishes; -/// no-op on non-Windows. issue #466. -#[cfg(target_os = "windows")] -pub(crate) fn refocus_qa_window(app: &AppHandle) { - if let Some(window) = app.get_webview_window("qa") { - let _ = show_qa_window_no_activate(&window); - } -} - -#[cfg(not(target_os = "windows"))] -pub(crate) fn refocus_qa_window(_app: &AppHandle) {} - -#[cfg(target_os = "windows")] -fn show_qa_window_no_activate(window: &tauri::WebviewWindow) -> bool { - // The name is historical; the actual behavior is now "show + focus" — the QA webview - // must really get keyboard focus so ESC reaches the React listener and the X button's - // first click isn't swallowed by the OS as an activation click. - // - // Why Tauri's show() / set_focus() instead of Win32 SetForegroundWindow + SetFocus: - // - SetFocus(host_hwnd) alone doesn't guarantee the WebView2 child receives keyboard - // events; WebView2 child windows have their own focus model. Tauri's internal - // webview-specific path actually delivers focus to the webview. - // - SetForegroundWindow can be rejected under Win11 focus-stealing prevention. - // Tauri 2.x has fallbacks in its cross-platform abstraction (temporary SPI - // adjustment / input queue attach). - // - // Trade-off vs issue #164 "QA window must not steal the frontmost app's focus": the - // window briefly becomes foreground when it appears, but begin_qa_session's focus-dance - // temporarily returns focus to the user's original app before capturing the selection - // (see the same-issue comment in coordinator.rs), and refocus_qa_window takes it back - // afterwards — the selection path still works; issue #164 being briefly violated for - // "the frame the QA window appears" is the cost of the #466 fix. - if window.show().is_err() { - return false; - } - let _ = window.set_focus(); - true -} - /// Pure multi-monitor capsule helpers (tests run on every CI host). Window creation /// inside the module stays `cfg(windows)`. mod capsule_mirrors; @@ -4286,21 +3835,6 @@ mod tests { assert_eq!(capsule_visual_height(true), 140.0); } - #[test] - fn qa_resize_preserves_centre_baseline_and_user_position() { - let expanded = - super::qa_resize_origin(-700.0, 700.0, 480.0, 80.0, 720.0, 380.0, false, true); - assert_eq!(expanded, (-820.0, 360.0)); - assert_eq!( - super::qa_resize_origin(expanded.0, expanded.1, 720.0, 380.0, 720.0, 380.0, true, true), - expanded - ); - assert_eq!( - super::qa_resize_origin(expanded.0, expanded.1, 720.0, 380.0, 480.0, 80.0, true, false), - (-700.0, 700.0) - ); - } - #[test] fn logical_monitor_frame_preserves_negative_origin() { let frame = logical_monitor_frame(-2560, 720, 5120, 2880, 2.0); diff --git a/openless-all/app/src-tauri/src/mobile_stubs/qa_hotkey.rs b/openless-all/app/src-tauri/src/mobile_stubs/qa_hotkey.rs deleted file mode 100644 index b94c7acbf..000000000 --- a/openless-all/app/src-tauri/src/mobile_stubs/qa_hotkey.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! Mobile stub — QA hotkeys are unavailable on Android/iOS. - -use std::sync::mpsc::Sender; - -use crate::types::ShortcutBinding; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum QaHotkeyEvent { - Pressed, -} - -#[derive(Debug, thiserror::Error)] -pub enum QaHotkeyError { - #[error("注册 QA 快捷键失败: {0}")] - RegisterFailed(String), -} - -pub struct QaHotkeyMonitor; - -impl QaHotkeyMonitor { - pub fn start( - _binding: ShortcutBinding, - _tx: Sender, - ) -> Result { - Err(QaHotkeyError::RegisterFailed( - "QA hotkeys are not available on mobile".into(), - )) - } - - pub fn update_binding(&self, _binding: ShortcutBinding) -> Result<(), QaHotkeyError> { - Ok(()) - } -} diff --git a/openless-all/app/src-tauri/src/mouse_dictation.rs b/openless-all/app/src-tauri/src/mouse_dictation.rs index 57fd204bc..3b1294d6d 100644 --- a/openless-all/app/src-tauri/src/mouse_dictation.rs +++ b/openless-all/app/src-tauri/src/mouse_dictation.rs @@ -232,7 +232,6 @@ fn match_modifier_states(required: &[String], down: [bool; 4]) -> bool { /// Dispatch a Mouse4 / Mouse5 edge into the active monitor (if any). pub fn handle_button(primary: &str, pressed: bool) { - crate::recording_companion::event(primary, pressed); let Some(normalized) = normalize_mouse_primary(primary) else { return; }; diff --git a/openless-all/app/src-tauri/src/qa_hotkey.rs b/openless-all/app/src-tauri/src/qa_hotkey.rs deleted file mode 100644 index 4d4504ca9..000000000 --- a/openless-all/app/src-tauri/src/qa_hotkey.rs +++ /dev/null @@ -1,264 +0,0 @@ -//! Global hotkey listener dedicated to the selection-based voice QA flow. -//! -//! 与 `hotkey.rs`(modifier-only 听写热键)平行——QA 用的是组合键 -//! `Cmd+Shift+;` / `Ctrl+Shift+;`,所以走 `global-hotkey` crate(macOS 内部 -//! 用 Carbon `RegisterEventHotKey`,Windows 用 `RegisterHotKey`)。 -//! -//! Produces only `QaHotkeyEvent::Pressed` edge events; the toggle / recording lifecycle is -//! interpreted by the coordinator (first press -> start QA; second press -> end). -//! -//! Shares the process-wide manager / event receiver via `global_hotkey_runtime`. - -use std::sync::mpsc::{Receiver, Sender}; -use std::sync::Arc; - -use global_hotkey::{GlobalHotKeyEvent, HotKeyState}; -use parking_lot::Mutex; - -use crate::global_hotkey_runtime::{GlobalHotkeyRuntime, RegisteredHotkey}; -use crate::shortcut_binding::{parse_global_hotkey, ShortcutBindingError}; -use crate::types::ShortcutBinding; - -#[derive(Debug, Clone, Copy)] -pub enum QaHotkeyEvent { - /// The user pressed the configured QA chord (toggle mode: first press starts, second ends). - Pressed, -} - -#[derive(Debug, thiserror::Error)] -pub enum QaHotkeyError { - #[error("不支持的修饰键: {0}")] - UnsupportedModifier(String), - #[error("不支持的主键: {0}")] - UnsupportedKey(String), - #[error("注册全局快捷键失败: {0}")] - RegisterFailed(String), - #[error("初始化全局快捷键管理器失败: {0}")] - ManagerInitFailed(String), -} - -/// QA global hotkey listener. Unregisters on `Drop`. -/// -/// Uses the `global-hotkey` crate internally; the forwarding thread holds a shared `Sender`. -pub struct QaHotkeyMonitor { - inner: Arc, -} - -struct Inner { - /// Handle of the currently registered hotkey; used to unregister. - registered: Mutex>, - tx: Sender, -} - -// global-hotkey 0.6's GlobalHotKeyManager internally holds HHOOK / window handles and other -// `*mut c_void` on Windows, and the crate does not mark it Send/Sync. These handles are OS -// process-level resources whose cross-thread access the OS itself synchronizes; coordinator.rs -// needs to put `Arc` (indirectly holding QaHotkeyMonitor) into async_runtime::spawn, -// which requires Send. Mark manually. The same applies to GlobalHotKeyManager on macOS, which -// uses Carbon EventHotKey internally. -// Same approach as the existing unsafe impl Send/Sync for hotkey.rs::CallbackContext. -unsafe impl Send for Inner {} -unsafe impl Sync for Inner {} - -impl QaHotkeyMonitor { - /// Start listening and register one hotkey. `tx` receives `QaHotkeyEvent::Pressed` on each - /// press edge. - /// - /// **Note**: the `global-hotkey` crate requires the manager to be constructed on the main - /// thread on macOS. The caller must ensure the call is initiated from the main thread (the - /// coordinator's supervisor thread hops to the main thread via - /// `AppHandle::run_on_main_thread` before spawning this monitor). - /// This function does not assert the main thread — unit / integration tests never reach the - /// manager creation line either. - pub fn start( - binding: ShortcutBinding, - tx: Sender, - ) -> Result { - let runtime = GlobalHotkeyRuntime::shared() - .map_err(|e| QaHotkeyError::ManagerInitFailed(e.to_string()))?; - - let hotkey = parse_binding(&binding)?; - let (registered, rx) = runtime - .register(hotkey) - .map_err(|e| QaHotkeyError::RegisterFailed(e.to_string()))?; - - // Start the forwarding thread: the runtime already dispatches by hotkey id; the id - // check stays as a defensive line so a future accidental re-connection to the - // process-wide event stream cannot leak into other shortcuts. - let hotkey_id = registered.hotkey().id(); - let tx_for_thread = tx.clone(); - std::thread::Builder::new() - .name("openless-qa-hotkey-forward".into()) - .spawn(move || forward_loop(hotkey_id, rx, tx_for_thread)) - .map_err(|e| QaHotkeyError::RegisterFailed(format!("spawn forward thread: {e}")))?; - - Ok(Self { - inner: Arc::new(Inner { - registered: Mutex::new(Some(registered)), - tx, - }), - }) - } - - /// Replace the currently registered hotkey (when the user changes the chord in settings). - pub fn update_binding(&self, binding: ShortcutBinding) -> Result<(), QaHotkeyError> { - let next = parse_binding(&binding)?; - let mut current = self.inner.registered.lock(); - if let Some(prev) = current.as_ref() { - if prev.hotkey() == next { - return Ok(()); - } - } - let runtime = GlobalHotkeyRuntime::shared() - .map_err(|e| QaHotkeyError::ManagerInitFailed(e.to_string()))?; - let (registered, rx) = runtime - .register(next) - .map_err(|e| QaHotkeyError::RegisterFailed(e.to_string()))?; - let hotkey_id = registered.hotkey().id(); - // Keep event forwarding alive for the replacement registration. - std::thread::Builder::new() - .name("openless-qa-hotkey-forward".into()) - .spawn({ - let tx = self.inner.tx.clone(); - move || forward_loop(hotkey_id, rx, tx) - }) - .map_err(|e| QaHotkeyError::RegisterFailed(format!("spawn forward thread: {e}")))?; - *current = Some(registered); - Ok(()) - } -} - -impl Drop for QaHotkeyMonitor { - fn drop(&mut self) { - self.inner.registered.lock().take(); - } -} - -fn forward_loop(hotkey_id: u32, rx: Receiver, tx: Sender) { - while let Ok(event) = rx.recv() { - if event.id() != hotkey_id { - continue; - } - if !matches!(event.state(), HotKeyState::Pressed) { - continue; - } - if let Err(e) = tx.send(QaHotkeyEvent::Pressed) { - log::warn!("[qa-hotkey] 事件投递失败: {e}"); - break; - } - } - log::info!("[qa-hotkey] 转发线程退出"); -} - -fn parse_binding( - binding: &ShortcutBinding, -) -> Result { - parse_global_hotkey(binding).map_err(|e| match e { - ShortcutBindingError::UnsupportedModifier(m) => QaHotkeyError::UnsupportedModifier(m), - ShortcutBindingError::UnsupportedKey(k) => QaHotkeyError::UnsupportedKey(k), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use global_hotkey::hotkey::{Code, Modifiers}; - - #[test] - fn parse_default_binding() { - let binding = ShortcutBinding::default_qa(); - let parsed = parse_binding(&binding).expect("default binding parses"); - assert!(parsed.mods.contains(Modifiers::SHIFT)); - assert_eq!(parsed.key, Code::Semicolon); - } - - #[test] - fn parse_letter_binding() { - let binding = ShortcutBinding { - primary: "k".into(), - modifiers: vec!["cmd".into(), "alt".into()], - }; - let parsed = parse_binding(&binding).expect("letter binding parses"); - assert_eq!(parsed.key, Code::KeyK); - #[cfg(target_os = "windows")] - assert!(parsed.mods.contains(Modifiers::CONTROL)); - #[cfg(not(target_os = "windows"))] - assert!(parsed.mods.contains(Modifiers::SUPER)); - assert!(parsed.mods.contains(Modifiers::ALT)); - } - - #[test] - fn unsupported_modifier_rejected() { - let binding = ShortcutBinding { - primary: ";".into(), - modifiers: vec!["hyper".into()], - }; - assert!(matches!( - parse_binding(&binding), - Err(QaHotkeyError::UnsupportedModifier(_)) - )); - } - - #[test] - fn empty_primary_rejected() { - let binding = ShortcutBinding { - primary: "".into(), - modifiers: vec!["cmd".into()], - }; - assert!(matches!( - parse_binding(&binding), - Err(QaHotkeyError::UnsupportedKey(_)) - )); - } - - #[test] - fn cmd_modifier_normalizes_per_platform() { - let binding = ShortcutBinding { - primary: ";".into(), - modifiers: vec!["cmd".into(), "shift".into()], - }; - let parsed = parse_binding(&binding).expect("binding parses"); - - #[cfg(target_os = "windows")] - { - assert!(parsed.mods.contains(Modifiers::CONTROL)); - assert!(!parsed.mods.contains(Modifiers::SUPER)); - } - - #[cfg(not(target_os = "windows"))] - { - assert!(parsed.mods.contains(Modifiers::SUPER)); - } - } - - #[test] - fn forward_loop_ignores_unrelated_hotkey_ids() { - let (event_tx, event_rx) = std::sync::mpsc::channel(); - let (out_tx, out_rx) = std::sync::mpsc::channel(); - - event_tx - .send(GlobalHotKeyEvent { - id: 41, - state: HotKeyState::Pressed, - }) - .unwrap(); - event_tx - .send(GlobalHotKeyEvent { - id: 42, - state: HotKeyState::Released, - }) - .unwrap(); - event_tx - .send(GlobalHotKeyEvent { - id: 42, - state: HotKeyState::Pressed, - }) - .unwrap(); - drop(event_tx); - - forward_loop(42, event_rx, out_tx); - - assert!(matches!(out_rx.recv().unwrap(), QaHotkeyEvent::Pressed)); - assert!(out_rx.try_recv().is_err()); - } -} diff --git a/openless-all/app/src-tauri/src/recording_companion.rs b/openless-all/app/src-tauri/src/recording_companion.rs deleted file mode 100644 index b25f75d7d..000000000 --- a/openless-all/app/src-tauri/src/recording_companion.rs +++ /dev/null @@ -1,218 +0,0 @@ -//! A recording shortcut plus one auxiliary key. Uses the existing physical-key -//! tap, preserving left/right keys, Fn and custom multi-key recording bindings. -use crate::types::ShortcutBinding; -use std::collections::HashSet; -use std::sync::{Arc, Mutex, OnceLock}; - -type Action = Arc; -#[derive(Default)] -struct Route { - state: ChordState, - action: Option, - suspended: bool, -} -#[derive(Default)] -struct ChordState { - recording: Option, - auxiliary: String, - held: HashSet, - consumed: bool, -} -static ROUTE: OnceLock> = OnceLock::new(); -fn route() -> &'static Mutex { - ROUTE.get_or_init(|| Mutex::new(Route::default())) -} -fn key(value: &str) -> String { - match value.to_ascii_lowercase().as_str() { - "rightoption" | "rightalt" | "alt-right" => "altright".into(), - "leftoption" | "leftalt" | "alt-left" => "altleft".into(), - "rightcontrol" | "ctrl-right" => "ctrlright".into(), - "leftcontrol" | "ctrl-left" => "ctrlleft".into(), - "rightcommand" | "rightsuper" | "cmd-right" | "super-right" => "cmdright".into(), - "leftcommand" | "leftsuper" | "cmd-left" | "super-left" => "cmdleft".into(), - "rightshift" | "shift-right" => "shiftright".into(), - "leftshift" | "shift-left" => "shiftleft".into(), - "super" | "command" => "cmd".into(), - "control" => "ctrl".into(), - "option" => "alt".into(), - " " => "space".into(), - other => other.into(), - } -} -fn matches(required: &str, physical: &str) -> bool { - required == physical - || matches!(required, "cmd" | "ctrl" | "alt" | "shift") && physical.starts_with(required) -} -impl ChordState { - fn sync_fn(&mut self, pressed: bool) { - if pressed { - self.held.insert("fn".into()); - } else { - self.held.remove("fn"); - } - } - fn event(&mut self, physical: &str, pressed: bool) -> (bool, bool) { - let physical = key(physical); - let auxiliary = key(&self.auxiliary); - let was_down = self.held.contains(&physical); - if pressed { - self.held.insert(physical.clone()); - } else { - self.held.remove(&physical); - } - if physical == auxiliary && !pressed { - let consume = std::mem::take(&mut self.consumed); - return (consume, false); - } - if physical != auxiliary || !pressed { - return (false, false); - } - if self.consumed { - return (true, false); - } - if was_down || auxiliary.is_empty() { - return (false, false); - } - let Some(binding) = &self.recording else { - return (false, false); - }; - let mut required: Vec = binding.modifiers.iter().map(|value| key(value)).collect(); - if !binding.primary.eq_ignore_ascii_case("ModifierChord") { - required.push(key(&binding.primary)); - } - if required.is_empty() - || required - .iter() - .any(|value| value.is_empty() || *value == auxiliary) - { - return (false, false); - } - let complete = required - .iter() - .all(|required| self.held.iter().any(|held| matches(required, held))) - && self.held.iter().all(|held| { - held == &auxiliary || required.iter().any(|required| matches(required, held)) - }); - if complete { - self.consumed = true; - } - (complete, complete) - } -} -pub(crate) fn configure(recording: ShortcutBinding, auxiliary: String) { - let mut route = route().lock().unwrap(); - route.state = ChordState { - recording: Some(recording), - auxiliary, - ..Default::default() - }; -} -pub(crate) fn install(action: Action) { - route().lock().unwrap().action = Some(action); -} -pub(crate) fn suspend(suspended: bool) { - let mut route = route().lock().unwrap(); - route.suspended = suspended; - route.state.held.clear(); - route.state.consumed = false; -} -pub(crate) fn reset_held() { - let mut route = route().lock().unwrap(); - route.state.held.clear(); - route.state.consumed = false; -} -/// Reconcile Fn from the current keyboard event, even if its modifier edge was missed. -pub(crate) fn sync_fn(pressed: bool) { - let mut route = route().lock().unwrap(); - route.state.sync_fn(pressed); -} -pub(crate) fn claimed() -> bool { - route().lock().unwrap().state.consumed -} -/// Returns true only for a claimed auxiliary key, including its autorepeat and release. -pub(crate) fn event(physical: &str, pressed: bool) -> bool { - let (consume, action) = { - let mut route = route().lock().unwrap(); - if route.suspended { - return false; - } - let (consume, fire) = route.state.event(physical, pressed); - (consume, if fire { route.action.clone() } else { None }) - }; - if let Some(action) = action { - action(); - } - consume -} - -#[cfg(test)] -mod tests { - use super::*; - fn state(primary: &str, modifiers: &[&str]) -> ChordState { - ChordState { - recording: Some(ShortcutBinding { - primary: primary.into(), - modifiers: modifiers.iter().map(|v| v.to_string()).collect(), - }), - auxiliary: "Space".into(), - ..Default::default() - } - } - #[test] - fn event_snapshot_consumes_fn_space_but_not_a_standalone_space() { - let mut s = state("Fn", &[]); - s.sync_fn(true); - assert_eq!(s.event("Space", true), (true, true)); - s.sync_fn(false); - assert_eq!(s.event("Space", false), (true, false)); - s.held.insert("fn".into()); - s.sync_fn(false); - assert_eq!(s.event("Space", true), (false, false)); - } - #[test] - fn side_specific_recording_is_the_only_anchor_and_repeats_are_suppressed() { - let mut s = state("RightAlt", &[]); - s.event("LeftAlt", true); - assert_eq!(s.event("Space", true), (false, false)); - s.event("Space", false); - s.event("LeftAlt", false); - s.event("RightAlt", true); - assert_eq!(s.event("Space", true), (true, true)); - assert_eq!(s.event("Space", true), (true, false)); - s.event("RightAlt", false); - assert_eq!(s.event("Space", false), (true, false)); - assert_eq!(s.event("Space", true), (false, false)); - } - #[test] - fn fn_and_custom_recording_chords_keep_the_entire_base() { - let mut s = state("Fn", &[]); - s.event("Fn", true); - assert_eq!(s.event("Space", true), (true, true)); - let mut s = state("D", &["cmd", "shift"]); - s.event("LeftCommand", true); - s.event("LeftShift", true); - assert_eq!(s.event("Space", true), (false, false)); - s.event("Space", false); - s.event("D", true); - assert_eq!(s.event("Space", true), (true, true)); - } - #[test] - fn mouse_recording_anchors_and_updated_auxiliary_keys_work() { - let mut s = state("Mouse4", &[]); - s.auxiliary = "Q".into(); - s.event("Mouse4", true); - assert_eq!(s.event("Space", true), (false, false)); - s.event("Space", false); - assert_eq!(s.event("Q", true), (true, true)); - } - #[test] - fn extra_modifiers_and_auxiliary_pressed_before_base_do_not_trigger() { - let mut s = state("RightControl", &[]); - s.event("Space", true); - s.event("RightControl", true); - assert_eq!(s.event("Space", true), (false, false)); - s.event("Space", false); - s.event("LeftShift", true); - assert_eq!(s.event("Space", true), (false, false)); - } -} diff --git a/openless-all/app/src-tauri/src/side_aware_combo.rs b/openless-all/app/src-tauri/src/side_aware_combo.rs index fc8016936..05d59d383 100644 --- a/openless-all/app/src-tauri/src/side_aware_combo.rs +++ b/openless-all/app/src-tauri/src/side_aware_combo.rs @@ -193,17 +193,6 @@ pub enum SideModifier { ShiftRight, } -impl SideModifier { - fn recording_key(self) -> &'static str { - match self { - Self::CmdLeft => "LeftCommand", Self::CmdRight => "RightCommand", - Self::CtrlLeft => "LeftControl", Self::CtrlRight => "RightControl", - Self::AltLeft => "LeftAlt", Self::AltRight => "RightAlt", - Self::ShiftLeft => "LeftShift", Self::ShiftRight => "RightShift", - } - } -} - pub struct SideAwareComboMonitor; impl SideAwareComboMonitor { @@ -381,12 +370,10 @@ pub mod platform { pub fn dispatch_vk(vk_code: u32, pressed: bool) -> bool { if let Some(side) = side_from_vk(vk_code) { - crate::recording_companion::event(side.recording_key(), pressed); handle_side_modifier(side, pressed); return false; } if let Some(primary) = primary_from_vk(vk_code) { - if crate::recording_companion::event(&primary, pressed) { return true; } handle_primary_key(&primary, pressed); } false @@ -682,17 +669,12 @@ pub mod platform { // `CGEventSourceKeyState` (Apple's standard way to resolve side). None => keycode_is_down(keycode), }; - crate::recording_companion::event(side.recording_key(), down); handle_side_modifier(side, down); - } else if keycode == 63 { - crate::recording_companion::event("Fn", flags & (1 << 23) != 0); } return false; } - crate::recording_companion::sync_fn(flags & (1 << 23) != 0); if let Some(primary) = primary_from_keycode(keycode) { - if crate::recording_companion::event(&primary, pressed) { return true; } handle_primary_key(&primary, pressed); } false diff --git a/openless-all/app/src/api/ask-window.ts b/openless-all/app/src/api/ask-window.ts new file mode 100644 index 000000000..e6c8f3948 --- /dev/null +++ b/openless-all/app/src/api/ask-window.ts @@ -0,0 +1,5 @@ +import { invokeOrPreview } from '../runtime/transport'; + +export function openAskWindow(): Promise { + return invokeOrPreview('ask_window_open', undefined, () => undefined); +} diff --git a/openless-all/app/src/api/qa.ts b/openless-all/app/src/api/qa.ts deleted file mode 100644 index 9d780a94d..000000000 --- a/openless-all/app/src/api/qa.ts +++ /dev/null @@ -1,60 +0,0 @@ -export * from '../lib/ipc/qa'; -export { chatPanelFocusKeyboard } from '../lib/ipc/chat-panel'; - -import type { QaStatePayload } from './contracts'; -import { isTauriHost } from '../runtime/environment'; -import { invokeOrPreview, listenTauriEvent, requireBackendReady } from '../runtime/transport'; - -export interface QaRecordingLevel { - sessionId: string; - level: number; -} - -export function listenQaStateChanges( - onState: (state: QaStatePayload) => void, -): Promise<() => void> { - return listenTauriEvent('qa:state', onState); -} - -export function listenQaLevelChanges( - onLevel: (recording: QaRecordingLevel) => void, -): Promise<() => void> { - return listenTauriEvent('qa:level', onLevel); -} - -export function listenQaDismissed(onDismissed: () => void): Promise<() => void> { - return listenTauriEvent('qa:dismiss', onDismissed); -} - -export function listenQaWindowShown(onShown: () => void): Promise<() => void> { - return listenTauriEvent('chat-panel:shown', onShown); -} - -export function listenQaWindowClosing(onClosing: () => void): Promise<() => void> { - return listenTauriEvent('chat-panel:closing', onClosing); -} - -export function listenEmbeddedQaShown(onShown: () => void): Promise<() => void> { - return listenQaStateChanges((state) => { - // ShowQa sends a bare idle signal. The idle dismissal snapshot carries messages. - if (state.kind === 'idle' && state.sessionId == null && state.messages === undefined) onShown(); - }); -} - -export async function startQaReadAloud( - text: string, - requestId: string, - onAudio: (bytes: number[]) => void, -): Promise { - if (!isTauriHost()) throw new Error('Read aloud requires a Tauri QA window'); - await requireBackendReady(); - const { Channel, invoke } = await import('@tauri-apps/api/core'); - const audio = new Channel(); - audio.onmessage = onAudio; - await invoke('deepseek_tts_start', { text, requestId, audio }); -} - -export async function stopQaReadAloud(requestId: string): Promise { - if (!isTauriHost()) return; - await invokeOrPreview('deepseek_tts_stop', { requestId }, () => undefined); -} diff --git a/openless-all/app/src/api/selection.ts b/openless-all/app/src/api/selection.ts index 7f9714d1a..1bde81467 100644 --- a/openless-all/app/src/api/selection.ts +++ b/openless-all/app/src/api/selection.ts @@ -1,17 +1,8 @@ -export * from '../lib/ipc/selection-polish-preview'; export * from '../lib/ipc/selection-voice-preview'; export * from '../lib/ipc/expression'; import { listenTauriEvent } from '../runtime/transport'; -export function listenSelectionPolishPreviewShown(onShown: () => void): Promise<() => void> { - return listenTauriEvent('selection-polish-preview:shown', onShown); -} - -export function listenSelectionPolishPreviewHidden(onHidden: () => void): Promise<() => void> { - return listenTauriEvent('selection-polish-preview:hide', onHidden); -} - export function listenSelectionVoiceIntentShown(onShown: () => void): Promise<() => void> { return listenTauriEvent('selection-voice-intent:shown', onShown); } diff --git a/openless-all/app/src/app/MainWindow.tsx b/openless-all/app/src/app/MainWindow.tsx index c2207d80e..f007b499d 100644 --- a/openless-all/app/src/app/MainWindow.tsx +++ b/openless-all/app/src/app/MainWindow.tsx @@ -1,5 +1,4 @@ import { lazy, Suspense } from 'react'; -import { useTranslation } from 'react-i18next'; import { HotkeySettingsProvider } from '../data/preferences'; import { useReducedMotion } from '../lib/motion'; import type { OS } from '../runtime/environment'; @@ -7,7 +6,7 @@ import { CoreStartupScreen } from './CoreStartupScreen'; import { useAndroidPanels } from './useAndroidPanels'; import { useMainStartup } from './startup'; import { useMainWindowHotkeys } from './windowHotkeys'; -import { Shell, QaPanel, VoiceEditPanel } from './destinations'; +import { Shell, VoiceEditPanel } from './destinations'; import './main-window.css'; const AutoUpdateGate = lazy(() => @@ -20,7 +19,6 @@ const Onboarding = lazy(() => ); export function MainWindow({ os, previewOnboarding }: { os: OS; previewOnboarding: boolean }) { - const { t } = useTranslation(); const { gate, caps, error, onboardingHandoff, completeOnboarding } = useMainStartup( os, previewOnboarding, @@ -33,20 +31,7 @@ export function MainWindow({ os, previewOnboarding }: { os: OS; previewOnboardin return ( }> - {panel.kind === 'qa' ? ( -
- close(panel)} - embedded - onRequestClose={() => didClose(panel)} - /> - {panel.error && ( -
- {t('qa.compact.closeError')} {panel.error} -
- )} -
- ) : panel.kind === 'voice-edit' ? ( + {panel.kind === 'voice-edit' ? (
import('./shell/FloatingShell'); export const Shell = lazy(() => loadMainShell().then((module) => ({ default: module.FloatingShell })), ); -export const QaPanel = lazy(() => - import('../features/qa/QaPanel').then((module) => ({ default: module.QaPanel })), -); export const VoiceEditPanel = lazy(() => import('../features/voice-edit/VoiceEditPanel').then((module) => ({ default: module.VoiceEditPanel, diff --git a/openless-all/app/src/app/shell/FloatingShell.tsx b/openless-all/app/src/app/shell/FloatingShell.tsx index 66a8796a9..bc797666a 100644 --- a/openless-all/app/src/app/shell/FloatingShell.tsx +++ b/openless-all/app/src/app/shell/FloatingShell.tsx @@ -40,7 +40,7 @@ import { useNavigation, type AppTab, } from './navigation'; -import { getPage, preloadPage, Overview, SelectionAsk } from './pages'; +import { getPage, preloadPage, Overview } from './pages'; interface FloatingShellProps { os?: OS; @@ -313,17 +313,11 @@ function FloatingShellBody({ style={{ display: 'flex', flexDirection: 'column', - // Keep the usage-guide content height so the outer scroll area's - // bottom-nav spacing applies. - flex: displayTab === 'selectionAsk' ? '1 0 auto' : 1, + flex: 1, minHeight: 0, }} > - {displayTab === 'selectionAsk' ? ( - openSettings('shortcuts')} /> - ) : ( - - )} +
)} diff --git a/openless-all/app/src/app/shell/navigation.ts b/openless-all/app/src/app/shell/navigation.ts index 856d9e344..59af25fa7 100644 --- a/openless-all/app/src/app/shell/navigation.ts +++ b/openless-all/app/src/app/shell/navigation.ts @@ -8,7 +8,6 @@ export type AppTab = | 'style' | 'marketplace' | 'translation' - | 'selectionAsk' | 'quickNote' | 'corrections' | 'localAsr'; @@ -40,7 +39,6 @@ interface NavItem { export const MORE_TABS: NavItem[] = [ { id: 'vocab', icon: 'vocab' }, { id: 'translation', icon: 'translate' }, - { id: 'selectionAsk', icon: 'selectionAsk' }, { id: 'quickNote', icon: 'history' }, { id: 'corrections', icon: 'filter' }, ]; @@ -75,7 +73,6 @@ export const NAV_TREE: NavNode[] = [ icon: 'selectionAsk', children: [ { id: 'corrections' }, - { id: 'selectionAsk' }, { id: 'translation' }, { id: 'quickNote' }, ], diff --git a/openless-all/app/src/app/shell/pages.tsx b/openless-all/app/src/app/shell/pages.tsx index 49ec2cfd5..072f8d7e7 100644 --- a/openless-all/app/src/app/shell/pages.tsx +++ b/openless-all/app/src/app/shell/pages.tsx @@ -17,10 +17,6 @@ const loaders = { import('../../features/translation/Translation').then((module) => ({ default: module.Translation, })), - selectionAsk: () => - import('../../features/selection-guide/SelectionAsk').then((module) => ({ - default: module.SelectionAsk, - })), quickNote: () => import('../../features/quick-note/QuickNote').then((module) => ({ default: module.QuickNote })), corrections: () => @@ -30,7 +26,6 @@ const loaders = { }; export { Overview }; -export const SelectionAsk = lazy(loaders.selectionAsk); const pages: Partial> = { // History is a primary destination on both desktop and mobile. @@ -40,7 +35,6 @@ const pages: Partial> = { style: lazy(loaders.style), marketplace: lazy(loaders.marketplace), translation: lazy(loaders.translation), - selectionAsk: SelectionAsk, quickNote: lazy(loaders.quickNote), corrections: lazy(loaders.corrections), }; diff --git a/openless-all/app/src/app/useAndroidPanels.ts b/openless-all/app/src/app/useAndroidPanels.ts index 1ae4a254d..1bb721ada 100644 --- a/openless-all/app/src/app/useAndroidPanels.ts +++ b/openless-all/app/src/app/useAndroidPanels.ts @@ -1,10 +1,4 @@ import { useEffect, useRef, useState } from 'react'; -import { - listenEmbeddedQaShown, - listenQaDismissed, - qaGetSnapshot, - qaWindowDismiss, -} from '../api/qa'; import { cancelVoiceEditSession, getVoiceEditState, @@ -39,16 +33,9 @@ export function useAndroidPanels(enabled: boolean) { }; const route = createAndroidPanelState( - async (kind, isCurrent) => { - if (kind === 'qa') { - await qaWindowDismiss(); - } else { - const session = await getVoiceEditState(); - if (!isCurrent()) return; - // Unlike a native auxiliary window the embedded panel has no host window - // to hide. Its route commits only after the session cancellation succeeds. - await cancelVoiceEditSession(session?.sessionId); - } + async (_kind, isCurrent) => { + const session = await getVoiceEditState(); + if (isCurrent()) await cancelVoiceEditSession(session?.sessionId); }, (next) => { setPanel(next); @@ -98,10 +85,6 @@ export function useAndroidPanels(enabled: boolean) { void (async () => { const observed = route.generation(); await Promise.all([ - subscribe(listenEmbeddedQaShown, () => { - void route.switchTo('qa'); - }), - subscribe(listenQaDismissed, () => route.nativeDismiss('qa')), subscribe(listenEmbeddedVoiceEditShown, () => { void route.switchTo('voice-edit'); }), @@ -109,13 +92,11 @@ export function useAndroidPanels(enabled: boolean) { ]); if (disposed || observed !== route.generation()) return; // Recreated WebViews may have missed their initial show event. - const [voice, qa] = await Promise.allSettled([getVoiceEditState(), qaGetSnapshot()]); + const [voice] = await Promise.allSettled([getVoiceEditState()]); if (disposed || observed !== route.generation()) return; const phase = voice.status === 'fulfilled' ? voice.value?.phase : undefined; if (phase && phase !== 'completed' && phase !== 'cancelled') { await route.switchTo('voice-edit'); - } else if (qa.status === 'fulfilled' && qa.value.sessionId && qa.value.kind !== 'cancelled') { - await route.switchTo('qa'); } else { releaseBack(); } diff --git a/openless-all/app/src/app/windowRoutes.tsx b/openless-all/app/src/app/windowRoutes.tsx index fd3ed32b3..8b99bda4f 100644 --- a/openless-all/app/src/app/windowRoutes.tsx +++ b/openless-all/app/src/app/windowRoutes.tsx @@ -3,7 +3,7 @@ import { Capsule } from '../features/capsule/Capsule'; import { CapsuleTranscriptOverlay } from '../features/capsule/LiveTranscriptPill'; import { AudioCueListener } from '../features/capsule/AudioCue'; import type { OS } from '../runtime/environment'; -import { QaPanel, VoiceEditPanel } from './destinations'; +import { VoiceEditPanel } from './destinations'; const MainWindow = lazy(() => import('./MainWindow').then((module) => ({ default: module.MainWindow })), @@ -18,7 +18,6 @@ export type WindowKind = | 'main' | 'capsule' | 'capsule-rail' - | 'qa' | 'voice-edit' | 'selection-voice-intent' | 'less-computer' @@ -29,7 +28,6 @@ export function resolveWindowKind(params: URLSearchParams): WindowKind { switch (kind) { case 'capsule': case 'capsule-rail': - case 'qa': case 'voice-edit': case 'selection-voice-intent': case 'less-computer': @@ -67,12 +65,6 @@ export function WindowRoutes({ ); - case 'qa': - return ( - - - - ); case 'selection-voice-intent': return ( diff --git a/openless-all/app/src/components/QaSurface.tsx b/openless-all/app/src/components/QaSurface.tsx deleted file mode 100644 index 862c6b8ac..000000000 --- a/openless-all/app/src/components/QaSurface.tsx +++ /dev/null @@ -1 +0,0 @@ -export * from '../features/qa/QaSurface'; diff --git a/openless-all/app/src/features/capsule/Capsule.tsx b/openless-all/app/src/features/capsule/Capsule.tsx index 75cbe49e3..735472c10 100644 --- a/openless-all/app/src/features/capsule/Capsule.tsx +++ b/openless-all/app/src/features/capsule/Capsule.tsx @@ -10,7 +10,6 @@ import { DEFAULT_CAPSULE_EXIT_MS, OpenLessPill } from './OpenLessCapsule'; import { CapsuleTranscriptRail, CapsuleTranslationBadge } from './LiveTranscriptPill'; import { capsuleTranscriptFontSize, visibleCapsuleTranscript } from '../../lib/capsuleTranscript'; import { getSettings } from '../../api/preferences'; -import { qaCancelTurn, qaStopRecording } from '../../api/qa'; import { cancelDictation, setCapsuleTranscriptVisible, stopDictation } from '../../api/dictation'; import { getCapsuleHostMetrics, @@ -46,7 +45,7 @@ import { // Inject the capsule keyframes once into document.head instead of rendering them in // JSX: per-frame (~60Hz) setLevel during recording would otherwise recreate this -// static
diff --git a/openless-all/app/e2e/fixtures/ask-window.tsx b/openless-all/app/e2e/fixtures/ask-window.tsx index 14c6f6847..c5f852955 100644 --- a/openless-all/app/e2e/fixtures/ask-window.tsx +++ b/openless-all/app/e2e/fixtures/ask-window.tsx @@ -2,8 +2,12 @@ import { createRoot } from 'react-dom/client'; import { AskWindow } from '../../ask-window/ask-window'; import type { AskClient, + AskEditor, AskEvent, + AskFocus, + AskRoute, AskSnapshot, + AskSubmission, AskTurn, AskVoiceEvent, AskWindowEvent, @@ -48,6 +52,7 @@ const excerpt: AttachmentInfo = { const handlers = { event: new Set<(event: AskEvent) => void>(), window: new Set<(event: AskWindowEvent) => void>(), + reader: new Set<(event: AskWindowEvent) => void>(), voice: new Set<(event: AskVoiceEvent) => void>(), }; @@ -59,6 +64,16 @@ let current: AskSnapshot = { activeTurnId: null, }; let events: AskEvent[] = []; +let focus: AskFocus = { focusEpoch: 1, focusRequest: 1 }; +let lastEditor: AskEditor | undefined; +const voiceScope = new Map>(); +const delivered = new Set(); +const shownTurns = new Set(); +let readerRoot: ReturnType; +const transcript = document.createElement('div'); +transcript.className = 'fixture-transcript'; +transcript.hidden = true; +document.body.append(transcript); function push(event: PendingEvent, live = true, replayable = true) { const packet = { @@ -73,6 +88,38 @@ function push(event: PendingEvent, live = true, replayable = true) { if (live) handlers.event.forEach((listener) => listener(packet)); } +function resolveRoute(route: AskRoute, request: SubmitAsk) { + const turnId = current.activeTurnId!; + push({ type: 'routeResolved', route, turnId }); + if (route !== 'answer' && route !== 'selectionAnswer') return; + const turn: AskTurn = { + turnId, + userMessage: { messageId: `${turnId}-input`, text: request.text }, + attachments: attachmentArchive.filter((item) => + request.attachmentIds.includes(item.attachmentId), + ), + deepThinking: request.options.deepThinking, + status: 'queued', + searchStatus: 'idle', + sources: [], + }; + push({ type: 'turnAccepted', turn }); + showTranscript(turnId); +} + +function showTranscript(turnId: string) { + if (shownTurns.has(turnId)) return; + shownTurns.add(turnId); + transcript.hidden = false; + fixture.showCount++; + if (!readerRoot) { + readerRoot = createRoot(transcript); + readerRoot.render(); + } else { + handlers.reader.forEach((listener) => listener({ type: 'activated' })); + } +} + function reset(id: string) { current = { conversationId: id, sequence: 0, turns: [], attachments: [], activeTurnId: null }; events = []; @@ -92,39 +139,61 @@ const fixture = { imageBase64, cancelled: [] as [string, string][], hideCount: 0, + hideReaderCount: 0, + showCount: 0, snapshotReads: 0, rejectNext: false, + route: 'answer' as AskRoute | null, truncate: false, liveListeners: () => handlers.event.size, + voiceSelection: () => + lastEditor?.editable ? [lastEditor.selectionStart, lastEditor.selectionEnd] : null, active: () => current.activeTurnId!, push, + resolve(route: AskRoute) { + resolveRoute(route, fixture.submits.at(-1)!); + }, + showTranscript, addInitialCards() { push({ type: 'attachmentChanged', attachment: image }); push({ type: 'attachmentChanged', attachment: pdf }); push({ type: 'attachmentChanged', attachment: excerpt }); }, activate() { - handlers.window.forEach((listener) => listener({ type: 'activated' })); + focus = { ...focus, focusRequest: focus.focusRequest + 1 }; + handlers.window.forEach((listener) => listener({ type: 'activated', ...focus })); }, voice( event: - | Omit, 'conversationId'> - | Omit, 'conversationId'>, + | { type: 'transcribed'; recordingId: string; text: string } + | { type: 'started' | 'cancelled'; recordingId: string }, ) { - handlers.voice.forEach((listener) => - listener({ ...event, conversationId: current.conversationId }), - ); + if (event.type === 'started' && lastEditor?.editable) { + voiceScope.set(event.recordingId, { + ...lastEditor, + ...focus, + recordingId: event.recordingId, + }); + } + const target = voiceScope.get(event.recordingId); + if (target) + handlers.voice.forEach((listener) => listener({ ...target, ...event } as AskVoiceEvent)); }, switch() { reset('conversation-2'); - fixture.activate(); + lastEditor = undefined; + focus = { focusEpoch: focus.focusEpoch + 1, focusRequest: focus.focusRequest }; + for (const listeners of [handlers.window, handlers.reader]) { + listeners.forEach((listener) => listener({ type: 'sessionChanged', ...focus })); + } }, }; const accepted = new Map(); +let attachmentArchive: AttachmentInfo[] = []; const client: AskClient = { async current() { - return { conversationId: current.conversationId }; + return { conversationId: current.conversationId, ...focus }; }, async newConversation() { reset('conversation-2'); @@ -151,19 +220,17 @@ const client: AskClient = { const previous = accepted.get(request.clientRequestId); if (previous) return { turnId: previous }; const turnId = `turn-${fixture.submits.length}`; - const turn: AskTurn = { - turnId, - userMessage: { messageId: `${turnId}-input`, text: request.text }, - attachments: current.attachments.filter((attachment) => - request.attachmentIds.includes(attachment.attachmentId), - ), - deepThinking: request.options.deepThinking, + const submission: AskSubmission = { + currentTurnId: turnId, status: 'queued', searchStatus: 'idle', sources: [], + copyStatus: 'notNeeded', }; + attachmentArchive = current.attachments; accepted.set(request.clientRequestId, turnId); - push({ type: 'turnAccepted', turn }); + push({ type: 'submitted', submission, attachmentIds: request.attachmentIds }); + if (fixture.route) resolveRoute(fixture.route, request); return { turnId }; }, async cancel(conversationId, turnId) { @@ -196,14 +263,44 @@ const client: AskClient = { fixture.opened.push([conversationId, turnId, sourceId]); }, async hide() { + focus = { ...focus, focusEpoch: focus.focusEpoch + 1 }; fixture.hideCount++; handlers.window.forEach((listener) => listener({ type: 'hidden' })); }, + async syncEditor(scope, editor) { + if (scope.focusEpoch !== focus.focusEpoch || scope.focusRequest !== focus.focusRequest) + return false; + lastEditor = editor; + return true; + }, + async acknowledgeVoice(id) { + const target = voiceScope.get(id); + if ( + delivered.has(id) || + !target || + target.focusEpoch !== focus.focusEpoch || + target.revision !== lastEditor?.revision || + target.mountId !== lastEditor?.mountId + ) + return false; + delivered.add(id); + return true; + }, onEvent: (receive) => subscribe(handlers.event, receive), onWindowEvent: (receive) => subscribe(handlers.window, receive), onVoiceEvent: (receive) => subscribe(handlers.voice, receive), }; +const readerClient: AskClient = { + ...client, + async hide() { + fixture.hideReaderCount++; + transcript.hidden = true; + handlers.reader.forEach((listener) => listener({ type: 'hidden' })); + }, + onWindowEvent: (receive) => subscribe(handlers.reader, receive), +}; + declare global { interface Window { askFixture: typeof fixture; diff --git a/openless-all/app/scripts/native-live-input-fixture.swift b/openless-all/app/scripts/native-live-input-fixture.swift index c243765ee..085fe0174 100644 --- a/openless-all/app/scripts/native-live-input-fixture.swift +++ b/openless-all/app/scripts/native-live-input-fixture.swift @@ -28,4 +28,13 @@ Timer.scheduledTimer(withTimeInterval: 0.3, repeats: false) { _ in field.setSelectedRange(NSRange(location: 7, length: 2)) } Timer.scheduledTimer(withTimeInterval: 30, repeats: false) { _ in app.terminate(nil) } +if CommandLine.arguments.contains("--ask-selected") { + Timer.scheduledTimer(withTimeInterval: 0.10, repeats: true) { timer in + if field.string != "before 原文 after" { + timer.invalidate() + return + } + field.setSelectedRange(NSRange(location: 7, length: 2)) + } +} app.run() diff --git a/openless-all/app/src-tauri/Cargo.lock b/openless-all/app/src-tauri/Cargo.lock index 347b89703..1767fd53d 100644 --- a/openless-all/app/src-tauri/Cargo.lock +++ b/openless-all/app/src-tauri/Cargo.lock @@ -4006,6 +4006,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -4581,6 +4592,9 @@ dependencies = [ "lopdf", "md-5 0.11.0", "micyou-dsp", + "num-bigint", + "num-rational", + "num-traits", "once_cell", "parking_lot", "regex", diff --git a/openless-all/app/src-tauri/backend-tests/Cargo.lock b/openless-all/app/src-tauri/backend-tests/Cargo.lock index 2f9d2a1f9..3ca53d765 100644 --- a/openless-all/app/src-tauri/backend-tests/Cargo.lock +++ b/openless-all/app/src-tauri/backend-tests/Cargo.lock @@ -1675,6 +1675,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1817,6 +1828,9 @@ dependencies = [ "log", "md-5", "micyou-dsp", + "num-bigint", + "num-rational", + "num-traits", "once_cell", "parking_lot", "regex", diff --git a/openless-all/app/src-tauri/backend-tests/tests/ask_entry_settings.rs b/openless-all/app/src-tauri/backend-tests/tests/ask_entry_settings.rs new file mode 100644 index 000000000..1d30bd840 --- /dev/null +++ b/openless-all/app/src-tauri/backend-tests/tests/ask_entry_settings.rs @@ -0,0 +1,4 @@ +#![cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))] + +#[path = "../../src/ask_window/entry_settings.rs"] +mod entry_settings; diff --git a/openless-all/app/src-tauri/capabilities/ask-window.json b/openless-all/app/src-tauri/capabilities/ask-window.json index a1c718313..5b2a2a38e 100644 --- a/openless-all/app/src-tauri/capabilities/ask-window.json +++ b/openless-all/app/src-tauri/capabilities/ask-window.json @@ -1,9 +1,9 @@ { "$schema": "../gen/schemas/desktop-schema.json", "identifier": "ask-window", - "description": "Title-bar controls for the standalone Ask window", + "description": "Window and event controls for the Ask composer and conversation", "platforms": ["macOS", "windows", "linux"], - "windows": ["ask-anything"], + "windows": ["ask-anything", "ask-conversation"], "permissions": [ "core:window:allow-start-dragging", "core:event:allow-listen", diff --git a/openless-all/app/src-tauri/src/ask_window/capture.rs b/openless-all/app/src-tauri/src/ask_window/capture.rs index c1210bba3..835e480fc 100644 --- a/openless-all/app/src-tauri/src/ask_window/capture.rs +++ b/openless-all/app/src-tauri/src/ask_window/capture.rs @@ -1,5 +1,4 @@ use chrono::{DateTime, Utc}; -use std::sync::atomic::{AtomicU64, Ordering}; #[derive(Debug, Clone)] pub(crate) struct AppIdentity { @@ -7,20 +6,12 @@ pub(crate) struct AppIdentity { pub(crate) bundle_id: Option, } -#[derive(Debug)] -pub(crate) enum Selection { - None, - Text(String), - TooLarge, -} - #[derive(Debug)] pub(crate) struct Invocation { pub(crate) invocation_id: uuid::Uuid, pub(crate) captured_at: DateTime, pub(crate) foreground_pid: Option, pub(crate) foreground_app: Option, - pub(crate) selection: Selection, } impl Invocation { @@ -30,41 +21,19 @@ impl Invocation { captured_at, foreground_pid: None, foreground_app: None, - selection: Selection::None, } } } -const MAX_SCALARS: usize = 4_000; -const MAX_BYTES: usize = 16 * 1024; -static INTERACTION_EPOCH: AtomicU64 = AtomicU64::new(0); - -pub(crate) fn mark_interaction() { - INTERACTION_EPOCH.fetch_add(1, Ordering::Relaxed); -} - -pub(super) fn bounded_selection(text: String) -> Selection { - if text.len() > MAX_BYTES || text.chars().take(MAX_SCALARS + 1).count() > MAX_SCALARS { - Selection::TooLarge - } else if text.trim().is_empty() { - Selection::None - } else { - Selection::Text(text) - } -} - pub(super) struct Foreground { pub(super) pid: u32, #[cfg(target_os = "windows")] pub(super) window: usize, - #[cfg(target_os = "windows")] - target: crate::selection::SelectionInsertionTarget, } pub(super) struct Trigger { pub(super) captured_at: DateTime, pub(super) foreground: Option, - interaction_epoch: u64, } impl Trigger { @@ -72,7 +41,6 @@ impl Trigger { Self { captured_at: Utc::now(), foreground, - interaction_epoch: INTERACTION_EPOCH.load(Ordering::Relaxed), } } @@ -86,13 +54,6 @@ impl Trigger { }; invocation.foreground_pid = Some(foreground.pid); invocation.foreground_app = app_identity(foreground.pid); - let still_original = || self.interaction_epoch == INTERACTION_EPOCH.load(Ordering::Relaxed); - if still_original() { - let selection = read_selection(&foreground); - if still_original() { - invocation.selection = selection; - } - } invocation } } @@ -144,24 +105,6 @@ fn app_identity(pid: u32) -> Option { }) } -#[cfg(target_os = "macos")] -fn read_selection(original: &Foreground) -> Selection { - let still_front = || foreground().is_some_and(|now| now.pid == original.pid); - if !still_front() || crate::unicode_keystroke::is_secure_input_enabled() { - return Selection::None; - } - let selection = match crate::selection::read_native_ask_selection_for_pid(original.pid) { - Ok(Some(text)) => bounded_selection(text), - Ok(None) => Selection::None, - Err(()) => Selection::TooLarge, - }; - if still_front() && !crate::unicode_keystroke::is_secure_input_enabled() { - selection - } else { - Selection::None - } -} - #[cfg(target_os = "windows")] pub(super) fn foreground() -> Option { use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId}; @@ -172,14 +115,12 @@ pub(super) fn foreground() -> Option { } let mut pid = 0; unsafe { GetWindowThreadProcessId(window, Some(&mut pid)) }; - let target = crate::selection::capture_selection_insertion_target(); if pid == 0 || unsafe { GetForegroundWindow() } != window { return None; } Some(Foreground { pid, window: window.0 as usize, - target, }) } @@ -215,43 +156,3 @@ fn app_identity(pid: u32) -> Option { }) } } - -#[cfg(target_os = "windows")] -fn read_selection(original: &Foreground) -> Selection { - let still_front = || { - use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow; - (unsafe { GetForegroundWindow().0 as usize == original.window }) - && crate::selection::selection_insertion_target_is_current(&original.target) - }; - if !still_front() { - return Selection::None; - } - let selection = crate::host_document::read_native_ask_selection_for_pid(original.pid) - .map(bounded_selection) - .unwrap_or(Selection::None); - if still_front() { - selection - } else { - Selection::None - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn selection_limits_reject_instead_of_truncating_or_trimming() { - assert!(matches!(bounded_selection(" ".into()), Selection::None)); - assert!( - matches!(bounded_selection("🙂".repeat(4000)), Selection::Text(text) if text.chars().count() == 4000) - ); - assert!(matches!( - bounded_selection("字".repeat(4001)), - Selection::TooLarge - )); - assert!( - matches!(bounded_selection(" \ntext ".into()), Selection::Text(text) if text == " \ntext ") - ); - } -} diff --git a/openless-all/app/src-tauri/src/ask_window/chat.rs b/openless-all/app/src-tauri/src/ask_window/chat.rs index 73ea3bc53..cd1738f97 100644 --- a/openless-all/app/src-tauri/src/ask_window/chat.rs +++ b/openless-all/app/src-tauri/src/ask_window/chat.rs @@ -4,7 +4,7 @@ use openless_core::ask::{self, AskAttachmentMetadata, AskTurnKey, StandaloneQaSe use openless_core::ask_attachment::{ AttachmentContent, AttachmentError, AttachmentKind, AttachmentMetadata, AttachmentPreviewSummary, AttachmentReader as _, AttachmentRegistry, AttachmentScope, - AttachmentState, ImageMime, SelectedText, + AttachmentState, ImageMime, }; use openless_core::{BackendError, BackendErrorCode, OpenLessBackend}; use parking_lot::Mutex; @@ -16,7 +16,9 @@ use tauri_plugin_shell::ShellExt; use uuid::Uuid; use super::attachment_files::{decode_dropped_image, pick_attachment_file, NativeAttachmentFile}; -use super::{context::DesktopContext, wire, LABEL}; +use super::entry_settings::{AskEntryIntentMode, AskEntrySettings, AskEntrySettingsStore}; +use super::target::{self, TargetTracker}; +use super::{context::DesktopContext, wire, CONVERSATION_LABEL, LABEL}; pub(crate) struct AskHost { service: StandaloneQaService, @@ -25,16 +27,23 @@ pub(crate) struct AskHost { attachment_gate: Mutex<()>, backend: Arc, context_preference: Option, - pending_notice: Mutex>, + entry_settings: Arc, + targets: Arc, } impl AskHost { - pub(crate) fn new(backend: Arc) -> Self { + pub(crate) fn new(app: &AppHandle, backend: Arc) -> Self { let registry = Arc::new(AttachmentRegistry::default()); let context = Arc::new(DesktopContext::default()); let context_preference = crate::persistence::data_dir() .ok() .map(|dir| dir.join("ask-app-context.json")); + let entry_settings = Arc::new(AskEntrySettingsStore::load( + app.path() + .app_local_data_dir() + .ok() + .map(|dir| dir.join(AskEntrySettingsStore::FILE_NAME)), + )); let allowed = context_preference .as_ref() .and_then(|path| std::fs::read(path).ok()) @@ -46,6 +55,10 @@ impl AskHost { }); let service = backend .create_standalone_ask(context.clone(), Arc::new(AttachmentPort(registry.clone()))); + let targets = Arc::new(TargetTracker::new()); + if let Err(error) = targets.start() { + log::warn!("[ask] external target observation unavailable: {error}"); + } Self { service, registry, @@ -53,7 +66,8 @@ impl AskHost { attachment_gate: Mutex::new(()), backend, context_preference, - pending_notice: Mutex::new(None), + entry_settings, + targets, } } @@ -93,25 +107,40 @@ impl AskHost { .map_err(attachment_error) } - pub(crate) fn register_selection( - &self, - id: Uuid, - text: String, - source_app: Option, - ) -> Result<(), BackendError> { - let _guard = self.attachment_gate.lock(); - let metadata = self - .scope(id)? - .register_selection(SelectedText::new(text, source_app).map_err(attachment_error)?) - .map_err(attachment_error)?; - self.service - .publish_attachment(id, attachment_metadata(&metadata)?, true) + async fn prepare_submission(&self) -> Result { + let committed_settings = self.entry_settings.read().map_err(|_| { + BackendError::new(BackendErrorCode::Persistence, "askEntrySettingsUnavailable") + })?; + let intent_mode = match committed_settings.intent_mode { + AskEntryIntentMode::SemanticHybrid => ask::AskIntentMode::SemanticHybrid, + AskEntryIntentMode::SeparateModel => ask::AskIntentMode::SeparateModel, + AskEntryIntentMode::MainModelTool => ask::AskIntentMode::MainModelTool, + }; + let locked = self.targets.capture().await; + let selection = match locked.selection { + target::ExternalSelection::None => ask::ExternalSelection::None, + target::ExternalSelection::Selected(text) => ask::ExternalSelection::Selected(text), + target::ExternalSelection::Unavailable => ask::ExternalSelection::Unavailable, + }; + let writer = locked.write_target.map(|target| { + Arc::new(TargetWriter { + tracker: self.targets.clone(), + target, + }) as Arc + }); + Ok(ask::AskPreparation { + intent_mode, + selection, + writer, + clipboard: Some(Arc::new(Clipboard)), + }) } pub(crate) fn stop(&self) { let _guard = self.attachment_gate.lock(); let current = self.current(); self.service.end_all(); + self.targets.stop(); self.registry.clear(); if let Some(id) = current { self.context.close_conversation(id); @@ -131,11 +160,35 @@ impl AskHost { return; }; if host.current() == Some(event.conversation_id) { - let _ = app.emit_to( - EventTarget::webview_window(LABEL), - "ask:event", - wire::event(&event), - ); + let packet = wire::event(&event); + for label in [LABEL, CONVERSATION_LABEL] { + let _ = app.emit_to( + EventTarget::webview_window(label), + "ask:event", + &packet, + ); + } + if matches!( + &event.kind, + ask::AskEventKind::RouteResolved { + route: ask::AskRoute::Answer | ask::AskRoute::SelectionAnswer, + } + ) { + if let (Some(turn_id), Some(desktop)) = + (event.turn_id, app.try_state::>()) + { + let desktop = desktop.inner().clone(); + let conversation_id = event.conversation_id; + tauri::async_runtime::spawn(async move { + if let Err(error) = desktop + .show_conversation_for_chat(conversation_id, turn_id) + .await + { + log::warn!("[ask] conversation window failed: {error}"); + } + }); + } + } } } Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, @@ -172,6 +225,58 @@ impl AskHost { } } +struct TargetWriter { + tracker: Arc, + target: target::WriteTarget, +} + +impl ask::AskWriter for TargetWriter { + fn write( + &self, + operation: ask::AskWriteOperation, + text: String, + claim: ask::AskWriteClaim, + ) -> futures_util::future::BoxFuture<'_, ask::AskWriteOutcome> { + let operation = match operation { + ask::AskWriteOperation::Insert => target::WriteOperation::Insert, + ask::AskWriteOperation::Replace => target::WriteOperation::Replace, + }; + Box::pin(async move { + match self + .tracker + .deliver(&self.target, operation, text, claim) + .await + { + target::WriteOutcome::Written => ask::AskWriteOutcome::Written, + target::WriteOutcome::NotWritten => ask::AskWriteOutcome::NotWritten, + target::WriteOutcome::Unknown => ask::AskWriteOutcome::Unknown, + target::WriteOutcome::Declined => ask::AskWriteOutcome::Declined, + } + }) + } +} + +struct Clipboard; + +impl ask::AskClipboard for Clipboard { + fn copy(&self, text: String) -> futures_util::future::BoxFuture<'_, Result<(), BackendError>> { + Box::pin(async move { + let copy = tauri::async_runtime::spawn_blocking(move || { + crate::insertion::copy_text_to_clipboard(&text) + }) + .await; + if matches!(copy, Ok(Ok(()))) { + Ok(()) + } else { + Err(BackendError::new( + BackendErrorCode::Platform, + "Could not copy the Ask result.", + )) + } + }) + } +} + struct AttachmentPort(Arc); impl ask::AttachmentReader for AttachmentPort { @@ -291,26 +396,62 @@ fn authorize(window: &WebviewWindow) -> Result<(), Value> { } } -type AskState<'a> = State<'a, Arc>; -type IpcResult = Result; +fn authorize_reader(window: &WebviewWindow) -> Result<(), Value> { + if window.label() == CONVERSATION_LABEL { + Ok(()) + } else { + authorize(window) + } +} -#[derive(serde::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SubmitEnvelope { - #[serde(flatten)] - request: ask::AskSubmit, - voice_recording_id: Option, +pub(crate) fn restricted_command_allowed(label: &str, command: &str) -> bool { + if label != LABEL && label != CONVERSATION_LABEL { + return true; + } + let reader = matches!( + command, + "ask_current" + | "ask_snapshot" + | "ask_events" + | "ask_cancel" + | "ask_open_source" + | "ask_window_hide" + ); + reader + || label == LABEL + && matches!( + command, + "ask_new_conversation" + | "ask_end_conversation" + | "ask_submit" + | "ask_add_attachment" + | "ask_add_dropped_image" + | "ask_remove_attachment" + | "ask_get_attachment_preview" + | "ask_sync_editor" + | "ask_ack_voice" + ) } +type AskState<'a> = State<'a, Arc>; +type IpcResult = Result; + #[tauri::command] pub fn ask_current(window: WebviewWindow, host: AskState<'_>) -> IpcResult { - authorize(&window)?; - Ok(json!({ + authorize_reader(&window)?; + let mut status = json!({ "conversationId": host.current(), "themeMode": host.backend.get_preferences().theme_mode, "platform": std::env::consts::OS, - "notice": host.pending_notice.lock().take(), - })) + }); + if window.label() == LABEL { + if let Some(desktop) = window.try_state::>() { + let (epoch, request) = desktop.composer_focus(); + status["focusEpoch"] = json!(epoch); + status["focusRequest"] = json!(request); + } + } + Ok(status) } #[tauri::command] @@ -323,11 +464,6 @@ pub fn ask_new_conversation(window: WebviewWindow, host: AskState<'_>) -> IpcRes if let Some(desktop) = window.try_state::>() { desktop.set_conversation(Some(conversation_id.to_string())); } - let _ = window.emit_to( - EventTarget::webview_window(LABEL), - "ask:window", - json!({"type":"activated"}), - ); Ok(json!({"conversationId": conversation_id})) } @@ -347,17 +483,12 @@ pub fn ask_end_conversation( if let Some(desktop) = window.try_state::>() { desktop.set_conversation(None); } - let _ = window.emit_to( - EventTarget::webview_window(LABEL), - "ask:window", - json!({"type":"activated"}), - ); Ok(Value::Null) } #[tauri::command] pub fn ask_snapshot(window: WebviewWindow, host: AskState<'_>, conversation_id: Uuid) -> IpcResult { - authorize(&window)?; + authorize_reader(&window)?; host.require_current(conversation_id).map_err(wire::error)?; host.service .snapshot(conversation_id) @@ -372,7 +503,7 @@ pub fn ask_events( conversation_id: Uuid, after_sequence: u64, ) -> IpcResult { - authorize(&window)?; + authorize_reader(&window)?; host.require_current(conversation_id).map_err(wire::error)?; let replay = host .service @@ -387,44 +518,16 @@ pub fn ask_events( pub async fn ask_submit( window: WebviewWindow, host: AskState<'_>, - request: SubmitEnvelope, + request: ask::AskSubmit, ) -> IpcResult { authorize(&window)?; - let SubmitEnvelope { - request, - voice_recording_id, - } = request; host.require_current(request.conversation_id) .map_err(wire::error)?; let conversation_id = request.conversation_id; let used_attachments = request.attachment_ids.clone(); - let desktop = window - .try_state::>() - .map(|state| state.inner().clone()); - let proof = voice_recording_id.map(|id| (id, request.clone())); let turn_id = host .service - .submit_guarded(request, || { - proof.as_ref().is_none_or(|(id, request)| { - desktop.as_ref().is_some_and(|desktop| { - desktop.current_voice_delivery().is_some_and(|delivery| { - let editor = delivery.target.editor; - delivery.recording_id == *id - && delivery.mode == super::voice::Mode::Submit - && editor.conversation_id == request.conversation_id.to_string() - && request.client_request_id == id.to_string() - && delivery.text.trim() == request.text - && editor.attachment_ids.iter().eq(request - .attachment_ids - .iter() - .map(Uuid::to_string) - .collect::>() - .iter()) - && editor.options.deep_thinking == request.options.deep_thinking - }) - }) - }) - }) + .submit_prepared(request, || host.prepare_submission()) .await .map_err(wire::error)?; let _guard = host.attachment_gate.lock(); @@ -443,7 +546,7 @@ pub fn ask_cancel( conversation_id: Uuid, turn_id: Uuid, ) -> IpcResult { - authorize(&window)?; + authorize_reader(&window)?; host.require_current(conversation_id).map_err(wire::error)?; host.service .cancel(AskTurnKey { @@ -535,7 +638,7 @@ pub fn ask_open_source( turn_id: Uuid, source_id: String, ) -> IpcResult { - authorize(&window)?; + authorize_reader(&window)?; host.require_current(conversation_id).map_err(wire::error)?; let url = host .service @@ -562,6 +665,13 @@ pub(crate) struct DesktopPorts { } impl super::Integration for DesktopPorts { + fn auto_voice_on_open(&self) -> bool { + self.ask + .entry_settings + .read() + .is_ok_and(|settings| settings.auto_voice_on_open) + } + fn prepare_open( &self, invocation: super::capture::Invocation, @@ -612,19 +722,6 @@ impl super::Integration for DesktopPorts { } } } - let source_app = invocation.foreground_app.and_then(|source| source.name); - let selection_result = match invocation.selection { - super::capture::Selection::Text(text) => { - ask.register_selection(conversation, text, source_app) - } - super::capture::Selection::TooLarge => { - Err(attachment_error(AttachmentError::SelectionTooLarge)) - } - super::capture::Selection::None => Ok(()), - }; - if let Err(error) = selection_result { - *ask.pending_notice.lock() = Some(wire::error(error)); - } Ok(()) }) } @@ -653,7 +750,7 @@ pub fn ask_window_hide( window: WebviewWindow, desktop: State<'_, Arc>, ) -> IpcResult { - authorize(&window)?; + authorize_reader(&window)?; desktop .hide(&window) .map_err(|message| json!({"code":"platform","message":message}))?; @@ -664,18 +761,19 @@ pub fn ask_window_hide( pub fn ask_sync_editor( window: WebviewWindow, desktop: State<'_, Arc>, + focus_epoch: u64, + focus_request: u64, editor: super::voice::Editor, ) -> IpcResult { authorize(&window)?; - if editor.mount_id.len() > 128 - || editor.editor_id.len() > 128 - || editor.attachment_ids.len() > 4 - { + if editor.mount_id.len() > 128 || editor.editor_id.len() > 128 { return Err(json!({"code":"invalid_argument","message":"Invalid Ask editor state."})); } - Ok(json!( - desktop.update_composer(desktop.focus_epoch(), Some(editor)) - )) + Ok(json!(desktop.update_composer( + focus_epoch, + focus_request, + editor + ))) } #[tauri::command] @@ -735,25 +833,77 @@ pub fn ask_set_context_permission( Ok(Value::Null) } +fn settings_reply(result: Result) -> IpcResult { + result + .map(|settings| json!(settings)) + .map_err(|_| json!({"code":"persistence","message":"Could not access Ask settings."})) +} + +#[tauri::command] +pub fn ask_get_entry_settings(window: WebviewWindow, host: AskState<'_>) -> IpcResult { + if window.label() != "main" { + return Err( + json!({"code":"permission_denied","message":"Open Ask settings in the main window."}), + ); + } + settings_reply(host.entry_settings.read()) +} + +#[tauri::command] +pub async fn ask_set_entry_settings( + window: WebviewWindow, + host: AskState<'_>, + settings: AskEntrySettings, +) -> IpcResult { + if window.label() != "main" { + return Err( + json!({"code":"permission_denied","message":"Open Ask settings in the main window."}), + ); + } + let store = host.entry_settings.clone(); + let result = tauri::async_runtime::spawn_blocking(move || store.write(settings)) + .await + .unwrap_or_else(|error| Err(error.to_string())); + settings_reply(result) +} + #[cfg(test)] mod tests { use super::*; #[test] - fn voice_submission_carries_separate_recording_and_conversation_ids() { - let conversation = Uuid::new_v4(); - let recording = Uuid::new_v4(); - let payload: SubmitEnvelope = serde_json::from_value(json!({ - "conversationId": conversation, - "clientRequestId": recording, - "voiceRecordingId": recording, - "text": "hi", - "attachmentIds": [], - "options": { "deepThinking": false } - })) - .unwrap(); - assert_eq!(payload.request.conversation_id, conversation); - assert_eq!(payload.request.client_request_id, recording.to_string()); - assert_eq!(payload.voice_recording_id.unwrap().as_uuid(), recording); + fn conversation_window_can_read_and_cancel_but_cannot_submit_or_change_settings() { + for command in [ + "ask_current", + "ask_snapshot", + "ask_events", + "ask_cancel", + "ask_open_source", + "ask_window_hide", + ] { + assert!(restricted_command_allowed(CONVERSATION_LABEL, command)); + assert!(restricted_command_allowed(LABEL, command)); + } + for command in [ + "ask_submit", + "ask_new_conversation", + "ask_end_conversation", + "ask_add_attachment", + "ask_get_attachment_preview", + "ask_sync_editor", + "ask_ack_voice", + ] { + assert!(!restricted_command_allowed(CONVERSATION_LABEL, command)); + assert!(restricted_command_allowed(LABEL, command)); + } + for command in [ + "ask_get_entry_settings", + "ask_set_entry_settings", + "copy_text_to_clipboard", + ] { + assert!(!restricted_command_allowed(CONVERSATION_LABEL, command)); + assert!(!restricted_command_allowed(LABEL, command)); + assert!(restricted_command_allowed("main", command)); + } } } diff --git a/openless-all/app/src-tauri/src/ask_window/entry_settings.rs b/openless-all/app/src-tauri/src/ask_window/entry_settings.rs new file mode 100644 index 000000000..c9c2c3077 --- /dev/null +++ b/openless-all/app/src-tauri/src/ask_window/entry_settings.rs @@ -0,0 +1,239 @@ +use std::{ + fs, + io::{ErrorKind, Write}, + path::{Path, PathBuf}, +}; + +use parking_lot::{Mutex, RwLock}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum AskEntryIntentMode { + #[default] + SemanticHybrid, + SeparateModel, + MainModelTool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct AskEntrySettings { + pub(crate) auto_voice_on_open: bool, + pub(crate) intent_mode: AskEntryIntentMode, +} + +impl Default for AskEntrySettings { + fn default() -> Self { + Self { + auto_voice_on_open: true, + intent_mode: AskEntryIntentMode::default(), + } + } +} + +#[derive(Deserialize)] +#[serde(remote = "AskEntrySettings", rename_all = "camelCase")] +struct StoredSettings { + #[serde(default = "default_auto_voice_on_open")] + auto_voice_on_open: bool, + #[serde(default)] + intent_mode: AskEntryIntentMode, +} + +fn default_auto_voice_on_open() -> bool { + AskEntrySettings::default().auto_voice_on_open +} + +fn decode_stored(bytes: &[u8]) -> serde_json::Result { + let mut decoder = serde_json::Deserializer::from_slice(bytes); + let settings = StoredSettings::deserialize(&mut decoder)?; + decoder.end()?; + Ok(settings) +} + +fn persist(path: &Path, settings: AskEntrySettings) -> anyhow::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()); + let parent = + parent.ok_or_else(|| anyhow::anyhow!("Ask entry settings directory is missing"))?; + fs::create_dir_all(parent)?; + let mut file = tempfile::NamedTempFile::new_in(parent)?; + file.write_all(&serde_json::to_vec(&settings)?)?; + file.as_file().sync_all()?; + file.persist(path)?; + Ok(()) +} + +pub(crate) struct AskEntrySettingsStore { + path: Option, + committed: RwLock>, + writes: Mutex<()>, +} + +impl AskEntrySettingsStore { + pub(crate) const FILE_NAME: &str = "ask-entry-settings.json"; + + pub(crate) fn load(path: Option) -> Self { + let committed = match path.as_deref() { + Some(path) => match fs::read(path) { + Ok(bytes) => decode_stored(&bytes) + .map_err(|error| format!("Invalid Ask entry settings: {error}")), + Err(error) if error.kind() == ErrorKind::NotFound => { + Ok(AskEntrySettings::default()) + } + Err(error) => Err(format!("Could not read Ask entry settings: {error}")), + }, + None => Err("Ask entry settings storage is unavailable.".into()), + }; + + Self { + path, + committed: RwLock::new(committed), + writes: Mutex::new(()), + } + } + + pub(crate) fn read(&self) -> Result { + self.committed.read().clone() + } + + pub(crate) fn write(&self, settings: AskEntrySettings) -> Result { + let _writer = self.writes.lock(); + let path = self + .path + .as_deref() + .ok_or("Ask entry settings storage is unavailable.")?; + persist(path, settings) + .map_err(|error| format!("Could not save Ask entry settings: {error}"))?; + *self.committed.write() = Ok(settings); + Ok(settings) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_file_and_missing_keys_use_the_canonical_defaults() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join(AskEntrySettingsStore::FILE_NAME); + assert_eq!( + AskEntrySettingsStore::load(Some(path.clone())) + .read() + .unwrap(), + AskEntrySettings::default() + ); + fs::write(&path, br#"{"autoVoiceOnOpen":false}"#).unwrap(); + assert_eq!( + AskEntrySettingsStore::load(Some(path.clone())) + .read() + .unwrap(), + AskEntrySettings { + auto_voice_on_open: false, + intent_mode: AskEntryIntentMode::SemanticHybrid, + } + ); + fs::write(&path, br#"{"intentMode":"mainModelTool"}"#).unwrap(); + assert_eq!( + AskEntrySettingsStore::load(Some(path)).read().unwrap(), + AskEntrySettings { + auto_voice_on_open: true, + intent_mode: AskEntryIntentMode::MainModelTool, + } + ); + assert!(serde_json::from_str::(r#"{"autoVoiceOnOpen":false}"#).is_err()); + } + + #[test] + fn writes_replace_the_full_value_and_reload_with_the_canonical_wire_names() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join(AskEntrySettingsStore::FILE_NAME); + let store = AskEntrySettingsStore::load(Some(path.clone())); + for (settings, mode) in [ + (AskEntrySettings::default(), "semanticHybrid"), + ( + AskEntrySettings { + auto_voice_on_open: false, + intent_mode: AskEntryIntentMode::SeparateModel, + }, + "separateModel", + ), + ( + AskEntrySettings { + auto_voice_on_open: true, + intent_mode: AskEntryIntentMode::MainModelTool, + }, + "mainModelTool", + ), + ] { + assert_eq!(store.write(settings).unwrap(), settings); + assert_eq!(store.read().unwrap(), settings); + assert_eq!( + serde_json::from_slice::(&fs::read(&path).unwrap()).unwrap(), + serde_json::json!({"autoVoiceOnOpen": settings.auto_voice_on_open, "intentMode": mode}) + ); + assert_eq!( + AskEntrySettingsStore::load(Some(path.clone())) + .read() + .unwrap(), + settings + ); + } + } + + #[test] + fn failures_do_not_publish_uncommitted_values() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join(AskEntrySettingsStore::FILE_NAME); + fs::write(&path, br#"{"intentMode":"unknown"}"#).unwrap(); + assert!(AskEntrySettingsStore::load(Some(path.clone())) + .read() + .is_err()); + fs::remove_file(&path).unwrap(); + let store = AskEntrySettingsStore::load(Some(path.clone())); + fs::create_dir(&path).unwrap(); + assert!(store + .write(AskEntrySettings { + auto_voice_on_open: false, + ..AskEntrySettings::default() + }) + .is_err()); + assert_eq!(store.read().unwrap(), AskEntrySettings::default()); + } + + #[test] + fn overlapping_writes_leave_disk_and_memory_at_the_same_commit() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join(AskEntrySettingsStore::FILE_NAME); + let store = AskEntrySettingsStore::load(Some(path.clone())); + let barrier = std::sync::Barrier::new(2); + let settings = [ + AskEntrySettings { + auto_voice_on_open: false, + intent_mode: AskEntryIntentMode::SeparateModel, + }, + AskEntrySettings { + auto_voice_on_open: true, + intent_mode: AskEntryIntentMode::MainModelTool, + }, + ]; + std::thread::scope(|scope| { + for value in settings { + let (store, barrier) = (&store, &barrier); + scope.spawn(move || { + barrier.wait(); + assert_eq!(store.write(value).unwrap(), value); + }); + } + }); + let committed = store.read().unwrap(); + assert!(settings.contains(&committed)); + assert_eq!( + AskEntrySettingsStore::load(Some(path)).read().unwrap(), + committed + ); + } +} diff --git a/openless-all/app/src-tauri/src/ask_window/hotkey.rs b/openless-all/app/src-tauri/src/ask_window/hotkey.rs index 0a0374da9..e80a4b4df 100644 --- a/openless-all/app/src-tauri/src/ask_window/hotkey.rs +++ b/openless-all/app/src-tauri/src/ask_window/hotkey.rs @@ -288,9 +288,9 @@ enum Origin { Ask(Option), } -enum Chord { - Open(capture::Trigger), - Voice(Option), +struct Chord { + trigger: Option, + autostart: bool, } enum Message { @@ -346,25 +346,28 @@ impl Native { return false; }; let foreground = capture::foreground(); - let chord = match &held.stroke.origin { - Origin::Ask(original) => { + let trigger = match &held.stroke.origin { + Origin::Ask(_) => { if !desktop.is_native_target(foreground.as_ref()) { return false; } - Chord::Voice(original.clone()) + None } Origin::Ordinary => { if desktop.is_native_target(foreground.as_ref()) { return false; } - Chord::Open(capture::Trigger::at_key_event(foreground)) + Some(capture::Trigger::at_key_event(foreground)) } }; + let chord = Chord { + trigger, + autostart: !desktop.voice_is_pending(), + }; held.chord_taken = tx.send(Message::Chord(held.stroke.id, chord)).is_ok(); held.chord_taken }); if down && !handled { - capture::mark_interaction(); self.abort_for_companion(tx); } return if handled { @@ -379,7 +382,7 @@ impl Native { } if down && (self.escape_captured - || (!disabled && desktop.owns_capsule() && desktop.is_focused())) + || (!disabled && desktop.voice_is_pending() && desktop.is_focused())) { if !self.escape_captured { let _ = tx.send(Message::Escape); @@ -422,7 +425,6 @@ impl Native { }; if !matches_binding { if modifier == 0 { - capture::mark_interaction(); self.abort_for_companion(tx); } return Disposition::Stock; @@ -519,6 +521,7 @@ async fn dispatch(desktop: Weak, mut receiver: mpsc::UnboundedReceiver< && !stroke.binding.fn_tap && matches!(stroke.binding.mode, HotkeyMode::Hold | HotkeyMode::Auto) && !active + && (matches!(stroke.origin, Origin::Ordinary) || !desktop.voice_is_pending()) { if let Some(runtime) = RUNTIME.get() { let tx = runtime.tx.clone(); @@ -536,7 +539,7 @@ async fn dispatch(desktop: Weak, mut receiver: mpsc::UnboundedReceiver< completed: false, }; if preexisting_grace { - next.start(&desktop, &edges); + next.start(&desktop, &edges, false); } current = Some(next); } @@ -548,7 +551,7 @@ async fn dispatch(desktop: Weak, mut receiver: mpsc::UnboundedReceiver< if !matches!(state.stroke.origin, Origin::Ordinary) || !desktop.ports.ordinary_dictation_is_active() { - state.start(&desktop, &edges); + state.start(&desktop, &edges, false); } } } @@ -566,7 +569,7 @@ async fn dispatch(desktop: Weak, mut receiver: mpsc::UnboundedReceiver< { continue; } - state.start(&desktop, &edges); + state.start(&desktop, &edges, true); state.release(&desktop, &edges, at); } Message::Chord(id, chord) => { @@ -576,24 +579,23 @@ async fn dispatch(desktop: Weak, mut receiver: mpsc::UnboundedReceiver< else { continue; }; - match chord { - Chord::Open(trigger) => { - state.abort(&desktop); - let desktop = desktop.clone(); - tauri::async_runtime::spawn(async move { - if let Err(error) = desktop.open_with(Some(trigger)).await { - log::warn!("[ask-window] could not open: {error}"); - } - }); + let cancel = chord.trigger.is_some() && state.started && !state.stop_only; + let autostart = chord.autostart && !desktop.voice_is_pending(); + let desktop = desktop.clone(); + tauri::async_runtime::spawn(async move { + if cancel { + desktop + .ports + .ordinary_dictation_edge(DictationHotkeyEdge::Combined { + press_id: id, + at: Instant::now(), + }) + .await; } - Chord::Voice(target) => { - if target.is_some() { - desktop.voice.intent(voice::Mode::Submit, target, id, true); - } else { - desktop.notify_frontend("ask-window:focused"); - } + if let Err(error) = desktop.open_with(chord.trigger, Some(autostart)).await { + log::warn!("[ask-window] could not open: {error}"); } - } + }); state.completed = true; } Message::Abort(id) => { @@ -615,7 +617,7 @@ async fn dispatch(desktop: Weak, mut receiver: mpsc::UnboundedReceiver< type EdgeSender = mpsc::UnboundedSender<(DictationHotkeyEdge, HotkeyMode)>; impl Dispatch { - fn start(&mut self, desktop: &Desktop, ordinary: &EdgeSender) { + fn start(&mut self, desktop: &Desktop, ordinary: &EdgeSender, toggle: bool) { if self.started { return; } @@ -629,11 +631,10 @@ impl Dispatch { at: self.stroke.at, }, ), - Origin::Ask(target) => desktop.voice.intent( - voice::Mode::Insert, + Origin::Ask(target) => desktop.voice.manual( target.clone(), self.stroke.id, - mode != HotkeyMode::Hold, + toggle && mode != HotkeyMode::Hold, ), } } diff --git a/openless-all/app/src-tauri/src/ask_window/mod.rs b/openless-all/app/src-tauri/src/ask_window/mod.rs index 18a977f98..472beb541 100644 --- a/openless-all/app/src-tauri/src/ask_window/mod.rs +++ b/openless-all/app/src-tauri/src/ask_window/mod.rs @@ -2,12 +2,14 @@ pub(crate) mod attachment_files; pub(crate) mod capture; pub(crate) mod chat; pub(crate) mod context; +pub(crate) mod entry_settings; pub(crate) mod hotkey; +pub(crate) mod target; pub(crate) mod voice; mod wire; use std::{ - collections::VecDeque, + collections::{HashSet, VecDeque}, future::Future, pin::Pin, sync::{ @@ -20,22 +22,28 @@ use std::{ use openless_core::{CapsulePayload, DictationHotkeyEdge, OpenLessBackend}; use parking_lot::Mutex; use tauri::{ - AppHandle, Emitter, EventTarget, LogicalSize, Manager, PhysicalPosition, WebviewUrl, - WebviewWindow, WebviewWindowBuilder, WindowEvent, + webview::PageLoadEvent, AppHandle, Emitter, EventTarget, LogicalSize, Manager, Monitor, + PhysicalPosition, PhysicalSize, WebviewUrl, WebviewWindow, WebviewWindowBuilder, WindowEvent, }; use capture::{Invocation, Trigger}; pub(crate) const LABEL: &str = "ask-anything"; +pub(crate) const CONVERSATION_LABEL: &str = "ask-conversation"; const PAGE: &str = "ask-window/index.html"; -const WIDTH: f64 = 800.0; -const HEIGHT: f64 = 920.0; +const WIDTH: f64 = 640.0; +const COMPOSER_HEIGHT: f64 = 300.0; +const CONVERSATION_HEIGHT: f64 = 600.0; +const WINDOW_GAP: f64 = 12.0; static OPEN: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); pub(crate) type Task = Pin + Send + 'static>>; pub(crate) trait Integration: Send + Sync { fn prepare_open(&self, invocation: Invocation) -> Task>; + fn auto_voice_on_open(&self) -> bool { + true + } fn ordinary_dictation_edge(&self, edge: DictationHotkeyEdge) -> Task<()>; fn ordinary_dictation_is_active(&self) -> bool; fn capsule(&self, frame: CapsulePayload); @@ -46,17 +54,77 @@ pub(crate) struct Desktop { ports: Arc, backend: Arc, focused: AtomicBool, - focus_epoch: AtomicU64, invocation_epoch: AtomicU64, #[cfg(target_os = "windows")] native_window: std::sync::atomic::AtomicUsize, current_conversation: Mutex>, - composer: Mutex>, + revealed_turns: Mutex>, + placement: Mutex>, + composer: Mutex, voice_result: Mutex>, recent_recordings: Mutex>, voice: voice::Controller, } +#[derive(Default)] +struct ComposerFocus { + epoch: u64, + request: u64, + auto_voice: Option, + target: Option, +} + +struct AutoVoice { + request: u64, + queued: bool, +} + +impl ComposerFocus { + fn request_focus(&mut self, chord: Option, recording: bool) { + if chord.is_some() && self.auto_voice.is_some() { + return; + } + self.request += 1; + self.auto_voice = (chord == Some(true) && !recording).then_some(AutoVoice { + request: self.request, + queued: false, + }); + } + + fn acknowledge(&mut self, epoch: u64, request: u64, editor: voice::Editor) -> Option { + if self.epoch != epoch + || self.request != request + || self.target.as_ref().is_some_and(|previous| { + editor.mount_id != previous.editor.mount_id + || editor.editor_id != previous.editor.editor_id + || editor.revision < previous.editor.revision + || (editor.revision == previous.editor.revision && editor != previous.editor) + }) + { + return None; + } + let start = self + .auto_voice + .as_mut() + .filter(|pending| !pending.queued && editor.editable) + .is_some_and(|pending| { + pending.queued = true; + true + }); + self.target = Some(voice::Target { + focus_epoch: epoch, + editor, + }); + Some(start) + } + + fn invalidate(&mut self) { + self.epoch += 1; + self.target = None; + self.auto_voice = None; + } +} + impl Desktop { pub(crate) fn install( app: AppHandle, @@ -68,12 +136,13 @@ impl Desktop { ports, backend, focused: AtomicBool::new(false), - focus_epoch: AtomicU64::new(0), invocation_epoch: AtomicU64::new(0), #[cfg(target_os = "windows")] native_window: std::sync::atomic::AtomicUsize::new(0), current_conversation: Mutex::new(None), - composer: Mutex::new(None), + revealed_turns: Mutex::new(HashSet::new()), + placement: Mutex::new(None), + composer: Mutex::new(ComposerFocus::default()), voice_result: Mutex::new(None), recent_recordings: Mutex::new(VecDeque::new()), voice: voice::Controller::new(), @@ -84,100 +153,75 @@ impl Desktop { } pub(crate) fn set_conversation(&self, conversation_id: Option) { - let mut current = self.current_conversation.lock(); - if *current != conversation_id { - *current = conversation_id; + { + let mut current = self.current_conversation.lock(); + if *current == conversation_id { + return; + } + *current = conversation_id.clone(); + self.revealed_turns.lock().clear(); self.revoke_target(); } + let (focus_epoch, focus_request) = self.composer_focus(); + for label in [LABEL, CONVERSATION_LABEL] { + let _ = self.app.emit_to( + EventTarget::webview_window(label), + "ask:window", + serde_json::json!({ + "type": "sessionChanged", "conversationId": conversation_id, + "focusEpoch": focus_epoch, "focusRequest": focus_request + }), + ); + } } - pub(crate) fn focus_epoch(&self) -> u64 { - self.focus_epoch.load(Ordering::Acquire) + pub(crate) fn composer_focus(&self) -> (u64, u64) { + let composer = self.composer.lock(); + (composer.epoch, composer.request) } pub(crate) fn update_composer( &self, - expected_focus_epoch: u64, - editor: Option, + focus_epoch: u64, + focus_request: u64, + editor: voice::Editor, ) -> bool { let conversation = self.current_conversation.lock(); - if self.focus_epoch() != expected_focus_epoch { + if conversation.as_ref() != Some(&editor.conversation_id) || !self.is_focused() { return false; } - match editor { - Some(editor) - if conversation.as_ref() == Some(&editor.conversation_id) && self.is_focused() => - { - let mut composer = self.composer.lock(); - if self.focus_epoch() != expected_focus_epoch { - return false; - } - let changed = composer.as_ref().is_some_and(|target| { - target.editor.editor_id != editor.editor_id - || target.editor.mount_id != editor.mount_id - }); - if changed { - self.voice.cancel(); - self.voice_result.lock().take(); - self.focus_epoch.fetch_add(1, Ordering::AcqRel); - } - if composer - .as_ref() - .is_some_and(|previous| !changed && editor.revision < previous.editor.revision) - { - return false; - } - *composer = Some(voice::Target { - focus_epoch: self.focus_epoch.load(Ordering::Acquire), - editor, - }); - self.voice.revalidate(); - true - } - Some(_) => false, - None => { - self.revoke_target(); - true - } + let mut composer = self.composer.lock(); + let Some(start) = composer.acknowledge(focus_epoch, focus_request, editor) else { + return false; + }; + if start { + self.voice + .autostart(focus_request, composer.target.clone().unwrap()); } + self.voice.revalidate(); + true } - pub(crate) fn current_voice_delivery(&self) -> Option { + fn current_voice_delivery(&self) -> Option { let delivery = self.voice_result.lock().clone()?; - self.target_is_current(&delivery.target, delivery.mode) - .then_some(delivery) - } - - pub(crate) fn accepts_voice_delivery( - &self, - recording_id: openless_core::SessionId, - conversation_id: &str, - mode: voice::Mode, - ) -> bool { - self.current_voice_delivery().is_some_and(|delivery| { - delivery.recording_id == recording_id - && delivery.mode == mode - && delivery.target.editor.conversation_id == conversation_id - }) + self.target_is_current(&delivery.target).then_some(delivery) } pub(crate) fn acknowledge_voice_delivery( &self, recording_id: openless_core::SessionId, ) -> bool { - let mut pending = self.voice_result.lock(); - if pending - .as_ref() - .is_some_and(|pending| pending.recording_id == recording_id) - { - pending.take(); - return true; - } - false - } - - pub(crate) fn platform(&self) -> &'static str { - std::env::consts::OS + let delivery = { + let mut pending = self.voice_result.lock(); + if !pending + .as_ref() + .is_some_and(|item| item.recording_id == recording_id) + { + return false; + } + pending.take().unwrap() + }; + self.target_is_current(&delivery.target) } pub(crate) fn owns_capsule(&self) -> bool { @@ -208,11 +252,13 @@ impl Desktop { } pub(crate) fn cancel_active_voice(&self) -> bool { + let pending = self.composer.lock().auto_voice.take().is_some(); + let delivery = self.voice_result.lock().take().is_some(); if self.owns_capsule() { self.voice.cancel(); true } else { - false + pending || delivery } } @@ -241,21 +287,44 @@ impl Desktop { } fn target_at_press(&self) -> Option { - self.composer.lock().clone() + self.composer.lock().target.clone() } - fn target_is_current(&self, target: &voice::Target, mode: voice::Mode) -> bool { + fn target_is_current(&self, target: &voice::Target) -> bool { self.is_focused() && self .composer .lock() + .target .as_ref() - .is_some_and(|current| target.matches(current, mode)) + .is_some_and(|current| target.matches(current)) + } + + fn autostart_is_pending(&self, request: u64) -> bool { + self.composer + .lock() + .auto_voice + .as_ref() + .is_some_and(|pending| pending.request == request && pending.queued) + } + + fn complete_autostart(&self, request: u64) { + let mut composer = self.composer.lock(); + if composer + .auto_voice + .as_ref() + .is_some_and(|pending| pending.request == request) + { + composer.auto_voice = None; + } + } + + fn voice_is_pending(&self) -> bool { + self.composer.lock().auto_voice.is_some() || self.owns_capsule() } fn revoke_target(&self) { - self.focus_epoch.fetch_add(1, Ordering::AcqRel); - self.composer.lock().take(); + self.composer.lock().invalidate(); self.voice_result.lock().take(); self.voice.cancel(); } @@ -267,7 +336,12 @@ impl Desktop { fn notify_frontend(&self, event: &str) { let payload = match event { - "ask-window:shown" => serde_json::json!({"type": "activated"}), + "ask-window:shown" => { + let (focus_epoch, focus_request) = self.composer_focus(); + serde_json::json!({ + "type": "activated", "focusEpoch": focus_epoch, "focusRequest": focus_request + }) + } "ask-window:voice-unavailable" => serde_json::json!({"type": "notice", "error": { "code": "voice_unavailable", "message": "Could not start voice input." }}), @@ -279,10 +353,19 @@ impl Desktop { } pub(crate) fn hide(&self, window: &WebviewWindow) -> Result<(), String> { + let label = window.label(); + if !matches!(label, LABEL | CONVERSATION_LABEL) { + return Err("Not an Ask window.".into()); + } + if label == LABEL { + self.invocation_epoch.fetch_add(1, Ordering::AcqRel); + } window.hide().map_err(|error| error.to_string())?; - self.mark_unfocused(); + if label == LABEL { + self.mark_unfocused(); + } let _ = self.app.emit_to( - EventTarget::webview_window(LABEL), + EventTarget::webview_window(label), "ask:window", serde_json::json!({"type":"hidden"}), ); @@ -294,14 +377,18 @@ impl Desktop { event: &str, recording_id: openless_core::SessionId, target: &voice::Target, - mode: voice::Mode, text: Option<&str>, ) { let mut payload = serde_json::json!({ "type": event, "recordingId": recording_id, "conversationId": target.editor.conversation_id, - "mode": mode + "focusEpoch": target.focus_epoch, + "mountId": target.editor.mount_id, + "editorId": target.editor.editor_id, + "revision": target.editor.revision, + "selectionStart": target.editor.selection_start, + "selectionEnd": target.editor.selection_end }); if let Some(text) = text { payload["text"] = serde_json::json!(text); @@ -315,7 +402,7 @@ impl Desktop { match event { WindowEvent::Focused(true) => { self.focused.store(true, Ordering::Release); - self.notify_frontend("ask-window:focused"); + self.notify_frontend("ask-window:shown"); } WindowEvent::Focused(false) | WindowEvent::Destroyed => self.mark_unfocused(), _ => {} @@ -326,16 +413,22 @@ impl Desktop { let foreground = capture::foreground(); let trigger = (!self.is_native_target(foreground.as_ref())) .then(|| Trigger::at_key_event(foreground)); - self.open_with(trigger).await + self.open_with(trigger, None).await } - async fn open_with(self: &Arc, trigger: Option) -> Result<(), String> { + async fn open_with( + self: &Arc, + trigger: Option, + chord: Option, + ) -> Result<(), String> { let generation = self.invocation_epoch.fetch_add(1, Ordering::AcqRel) + 1; let _opening = OPEN.lock().await; let is_latest = || self.invocation_epoch.load(Ordering::Acquire) == generation; if !is_latest() { return Ok(()); } + let placement = (trigger.is_some() || self.placement.lock().is_none()) + .then(|| Placement::at_cursor(&self.app)); if let Some(trigger) = trigger { let captured_at = trigger.captured_at; let invocation = match tokio::time::timeout( @@ -355,37 +448,148 @@ impl Desktop { if !is_latest() { return Ok(()); } - let window = ensure_window(&self.app, self)?; - show(window).await?; - self.notify_frontend("ask-window:shown"); - Ok(()) + let chord = chord.map(|start| start && self.ports.auto_voice_on_open()); + let window = ensure_window(&self.app, self, LABEL)?; + let desktop = self.clone(); + on_ui_thread(window, move |window| { + if desktop.invocation_epoch.load(Ordering::Acquire) != generation { + return Ok(()); + } + if let Some(placement) = placement { + placement.position(&window, LABEL)?; + if let Some(reader) = desktop.app.get_webview_window(CONVERSATION_LABEL) { + placement.position(&reader, CONVERSATION_LABEL)?; + } + *desktop.placement.lock() = Some(placement); + } + desktop + .composer + .lock() + .request_focus(chord, desktop.owns_capsule()); + if !desktop.is_focused() { + if let Err(error) = focus_composer(&window) { + desktop.composer.lock().auto_voice = None; + return Err(error); + } + } + desktop.notify_frontend("ask-window:shown"); + Ok(()) + }) + .await + } + + pub(crate) async fn show_conversation_for_chat( + self: &Arc, + conversation_id: uuid::Uuid, + turn_id: uuid::Uuid, + ) -> Result<(), String> { + let conversation_id = conversation_id.to_string(); + let _opening = OPEN.lock().await; + { + let current = self.current_conversation.lock(); + if current.as_ref() != Some(&conversation_id) + || !self.revealed_turns.lock().insert(turn_id) + { + return Ok(()); + } + } + let window = ensure_window(&self.app, self, CONVERSATION_LABEL)?; + let desktop = self.clone(); + on_ui_thread(window, move |window| { + if desktop.current_conversation.lock().as_ref() != Some(&conversation_id) { + return Ok(()); + } + let placement = desktop + .placement + .lock() + .clone() + .unwrap_or_else(|| Placement::at_cursor(&desktop.app)); + placement.position(&window, CONVERSATION_LABEL)?; + reveal_without_focus(&window)?; + let _ = desktop.app.emit_to( + EventTarget::webview_window(CONVERSATION_LABEL), + "ask:window", + serde_json::json!({"type": "activated"}), + ); + Ok(()) + }) + .await + } +} + +#[derive(Clone, Default)] +struct Placement(Option); + +impl Placement { + fn at_cursor(app: &AppHandle) -> Self { + Self( + app.cursor_position() + .ok() + .and_then(|position| { + app.monitor_from_point(position.x, position.y) + .ok() + .flatten() + }) + .or_else(|| app.primary_monitor().ok().flatten()), + ) + } + + fn position(&self, window: &WebviewWindow, label: &str) -> Result<(), String> { + let Some(monitor) = &self.0 else { + return Ok(()); + }; + let work = monitor.work_area(); + let scale = monitor.scale_factor(); + let size: LogicalSize = work.size.to_logical(scale); + let gap = WINDOW_GAP.min(size.height / 24.0); + let width = WIDTH.min(size.width - gap * 2.0); + let composer_height = COMPOSER_HEIGHT.min((size.height - gap * 3.0) * 0.5); + let conversation_height = + CONVERSATION_HEIGHT.min(size.height - composer_height - gap * 3.0); + let (height, bottom) = if label == LABEL { + (composer_height, gap) + } else { + (conversation_height, composer_height + gap * 2.0) + }; + window + .set_position(PhysicalPosition::new( + work.position.x + ((size.width - width) * scale / 2.0).round() as i32, + work.position.y + ((size.height - bottom - height) * scale).round() as i32, + )) + .and_then(|()| { + window.set_size(PhysicalSize::new( + (width * scale).floor() as u32, + (height * scale).floor() as u32, + )) + }) + .map_err(|error| error.to_string()) } } -fn ensure_window(app: &AppHandle, desktop: &Arc) -> Result { - if let Some(window) = app.get_webview_window(LABEL) { +fn ensure_window( + app: &AppHandle, + desktop: &Arc, + label: &str, +) -> Result { + if let Some(window) = app.get_webview_window(label) { return Ok(window); } - let monitor = app - .cursor_position() - .ok() - .and_then(|position| { - app.monitor_from_point(position.x, position.y) - .ok() - .flatten() - }) - .or_else(|| app.primary_monitor().ok().flatten()); - let scale = monitor - .as_ref() - .map(|monitor| { - let work: LogicalSize = - monitor.work_area().size.to_logical(monitor.scale_factor()); - (work.width / WIDTH).min(work.height / HEIGHT).min(1.0) + let composer = label == LABEL; + let page_owner = Arc::downgrade(desktop); + let window = WebviewWindowBuilder::new(app, label, WebviewUrl::App(PAGE.into())) + .title(if composer { + "OpenLess — Ask Anything" + } else { + "OpenLess — Ask Conversation" }) - .unwrap_or(1.0); - let window = WebviewWindowBuilder::new(app, LABEL, WebviewUrl::App(PAGE.into())) - .title("OpenLess — Ask Anything") - .inner_size(WIDTH * scale, HEIGHT * scale) + .inner_size( + WIDTH, + if composer { + COMPOSER_HEIGHT + } else { + CONVERSATION_HEIGHT + }, + ) .decorations(false) .transparent(true) .shadow(false) @@ -394,19 +598,25 @@ fn ensure_window(app: &AppHandle, desktop: &Arc) -> Result) -> Result Result<(), String> { +async fn on_ui_thread( + window: WebviewWindow, + action: impl FnOnce(WebviewWindow) -> Result<(), String> + Send + 'static, +) -> Result<(), String> { let (tx, rx) = tokio::sync::oneshot::channel(); let target = window.clone(); window .run_on_main_thread(move || { - let result = target.show().and_then(|()| { - #[cfg(target_os = "macos")] - { - use objc2::{ - msg_send, - runtime::{AnyClass, AnyObject, Bool}, - }; - unsafe { - if let Some(class) = AnyClass::get("NSApplication") { - let app: *mut AnyObject = msg_send![class, sharedApplication]; - let _: () = msg_send![app, activateIgnoringOtherApps: Bool::YES]; - } - } - } - target.unminimize()?; - target.set_focus() - }); - let _ = tx.send(result.map_err(|error| error.to_string())); + let _ = tx.send(action(target)); }) .map_err(|error| error.to_string())?; rx.await.map_err(|error| error.to_string())? } +fn focus_composer(window: &WebviewWindow) -> Result<(), String> { + window.show().map_err(|error| error.to_string())?; + #[cfg(target_os = "macos")] + { + use objc2::{ + msg_send, + runtime::{AnyClass, AnyObject, Bool}, + }; + unsafe { + if let Some(class) = AnyClass::get("NSApplication") { + let app: *mut AnyObject = msg_send![class, sharedApplication]; + let _: () = msg_send![app, activateIgnoringOtherApps: Bool::YES]; + } + } + } + window + .unminimize() + .and_then(|()| window.set_focus()) + .map_err(|error| error.to_string()) +} + +#[cfg(target_os = "macos")] +fn reveal_without_focus(window: &WebviewWindow) -> Result<(), String> { + let handle = window.ns_window().map_err(|error| error.to_string())?; + if handle.is_null() { + return Err("Ask conversation window is unavailable.".into()); + } + unsafe { + let _: () = objc2::msg_send![ + handle.cast::(), + orderFrontRegardless + ]; + } + Ok(()) +} + +#[cfg(target_os = "windows")] +fn reveal_without_focus(window: &WebviewWindow) -> Result<(), String> { + use windows::Win32::UI::WindowsAndMessaging::{ + SetWindowPos, HWND_TOP, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SWP_SHOWWINDOW, + }; + let handle = window.hwnd().map_err(|error| error.to_string())?; + unsafe { + SetWindowPos( + windows::Win32::Foundation::HWND(handle.0), + HWND_TOP, + 0, + 0, + 0, + 0, + SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE | SWP_SHOWWINDOW, + ) + } + .map_err(|error| error.to_string()) +} + #[tauri::command] pub async fn ask_window_open(window: WebviewWindow) -> Result<(), String> { if window.label() != "main" { @@ -466,3 +720,61 @@ pub async fn ask_window_open(window: WebviewWindow) -> Result<(), String> { .ok_or("Ask Desktop is not initialized")?; desktop.open().await } + +#[cfg(test)] +mod tests { + use super::*; + + fn editor(revision: u64, editable: bool) -> voice::Editor { + voice::Editor { + conversation_id: "conversation".into(), + mount_id: "mount".into(), + editor_id: "editor".into(), + revision, + editable, + selection_start: 2, + selection_end: 2, + } + } + + #[test] + fn only_a_current_editable_chord_ack_arms_the_capsule_once() { + let mut composer = ComposerFocus::default(); + composer.request_focus(Some(true), false); + let epoch = composer.epoch; + let request = composer.request; + assert_eq!( + composer.acknowledge(epoch, request - 1, editor(1, true)), + None + ); + assert_eq!( + composer.acknowledge(epoch, request, editor(1, false)), + Some(false) + ); + composer.request_focus(Some(false), false); + assert_eq!(composer.request, request); + assert_eq!( + composer.acknowledge(epoch, request, editor(2, true)), + Some(true) + ); + assert_eq!( + composer.acknowledge(epoch, request, editor(2, true)), + Some(false) + ); + composer.invalidate(); + assert_eq!(composer.acknowledge(epoch, request, editor(3, true)), None); + } + + #[test] + fn menu_disabled_setting_and_active_capsule_can_only_refocus() { + for (chord, recording) in [(None, false), (Some(false), false), (Some(true), true)] { + let mut composer = ComposerFocus::default(); + composer.request_focus(chord, recording); + assert!(composer.auto_voice.is_none()); + assert_eq!( + composer.acknowledge(composer.epoch, composer.request, editor(1, true)), + Some(false) + ); + } + } +} diff --git a/openless-all/app/src-tauri/src/ask_window/target.rs b/openless-all/app/src-tauri/src/ask_window/target.rs new file mode 100644 index 000000000..92fbe58b3 --- /dev/null +++ b/openless-all/app/src-tauri/src/ask_window/target.rs @@ -0,0 +1,425 @@ +use std::io; +use std::ops::Range; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; +use std::sync::{mpsc, Arc, Mutex, Weak}; +use std::time::{Duration, Instant}; +use tokio::sync::oneshot; + +#[cfg(target_os = "macos")] +#[path = "target/macos.rs"] +mod native; +#[cfg(target_os = "windows")] +#[path = "target/windows.rs"] +mod native; +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +#[path = "target/unsupported.rs"] +mod native; + +const CAPTURE_TIMEOUT: Duration = Duration::from_millis(1200); +const DELIVERY_TIMEOUT: Duration = Duration::from_secs(2); +const OBSERVER_TICK: Duration = Duration::from_millis(25); +const FOCUS_REFRESH: Duration = Duration::from_millis(175); +const MAX_SELECTION_UTF16: usize = 8_000; +const MAX_SELECTION_SCALARS: usize = 4_000; +const MAX_DOCUMENT_UTF16: usize = 20_000; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum ExternalSelection { + None, + Selected(String), + Unavailable, +} + +pub(crate) struct CapturedTarget { + pub(crate) selection: ExternalSelection, + pub(crate) write_target: Option, +} + +impl CapturedTarget { + fn unavailable() -> Self { + Self { + selection: ExternalSelection::Unavailable, + write_target: None, + } + } +} + +pub(crate) struct WriteTarget(Arc<()>); + +#[derive(Clone, Copy)] +pub(crate) enum WriteOperation { + Insert, + Replace, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum WriteOutcome { + Written, + NotWritten, + Unknown, + Declined, +} + +#[derive(Default)] +pub(crate) struct TargetTracker { + running: Mutex>, +} + +struct Running { + sender: mpsc::SyncSender, + active: Arc, +} + +impl Drop for Running { + fn drop(&mut self) { + self.active.store(false, Ordering::Release); + } +} + +type Claim = Box bool + Send>; + +enum Request { + Capture(oneshot::Sender), + Deliver { + target: Arc<()>, + operation: WriteOperation, + text: String, + claim: Claim, + phase: Arc, + reply: oneshot::Sender, + }, +} + +impl TargetTracker { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn start(&self) -> io::Result<()> { + let mut running = self.running.lock().unwrap(); + if running + .as_ref() + .is_some_and(|run| run.active.load(Ordering::Acquire)) + { + return Ok(()); + } + let (sender, receiver) = mpsc::sync_channel(4); + let active = Arc::new(AtomicBool::new(true)); + let worker_active = active.clone(); + std::thread::Builder::new() + .name("ask-external-target".into()) + .spawn(move || { + run(receiver, &worker_active); + worker_active.store(false, Ordering::Release); + })?; + *running = Some(Running { sender, active }); + Ok(()) + } + + pub(crate) fn stop(&self) { + self.running.lock().unwrap().take(); + } + + fn send(&self, request: Request) -> bool { + self.running + .lock() + .unwrap() + .as_ref() + .is_some_and(|run| run.sender.try_send(request).is_ok()) + } + + pub(crate) async fn capture(&self) -> CapturedTarget { + let (reply, result) = oneshot::channel(); + if self.send(Request::Capture(reply)) { + if let Ok(Ok(captured)) = tokio::time::timeout(CAPTURE_TIMEOUT, result).await { + return captured; + } + } + CapturedTarget::unavailable() + } + + pub(crate) async fn deliver( + &self, + target: &WriteTarget, + operation: WriteOperation, + text: String, + claim: Claim, + ) -> WriteOutcome { + let guard = AbandonOnDrop(Arc::new(DeliveryPhase::default())); + let (reply, result) = oneshot::channel(); + if !self.send(Request::Deliver { + target: target.0.clone(), + operation, + text, + claim, + phase: guard.0.clone(), + reply, + }) { + return WriteOutcome::NotWritten; + } + match tokio::time::timeout(DELIVERY_TIMEOUT, result).await { + Ok(Ok(outcome)) => outcome, + _ => guard.0.abandon(), + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Foreground { + Own, + External(Key), + Unknown, +} + +struct Latest { + front: Foreground, + epoch: u64, + bound: Option, + refreshed: Instant, +} + +impl Latest { + fn new(now: Instant) -> Self { + Self { + front: Foreground::Unknown, + epoch: 0, + bound: None, + refreshed: now, + } + } + + fn refresh( + &mut self, + front: Foreground, + epoch: u64, + now: Instant, + force: bool, + bind: impl FnOnce(Key) -> Option, + ) { + match front { + Foreground::External(key) + if force + || front != self.front + || epoch != self.epoch + || now.duration_since(self.refreshed) >= FOCUS_REFRESH => + { + self.bound = bind(key); + self.refreshed = now; + } + Foreground::Unknown => self.bound = None, + Foreground::Own if self.epoch != epoch => self.bound = None, + _ => {} + } + self.epoch = epoch; + self.front = front; + } +} + +fn run(receiver: mpsc::Receiver, active: &AtomicBool) { + let epoch = Arc::new(AtomicU64::new(0)); + let Some(mut platform) = native::Platform::new(epoch.clone()) else { + return; + }; + let mut latest = Latest::new(Instant::now()); + let mut targets: Vec<(Weak<()>, native::Identity, Baseline)> = Vec::new(); + while active.load(Ordering::Acquire) { + let request = match receiver.recv_timeout(OBSERVER_TICK) { + Ok(request) => Some(request), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + if !active.load(Ordering::Acquire) { + break; + } + targets.retain(|(lease, _, _)| lease.strong_count() != 0); + + if let Some(Request::Deliver { + target, + operation, + text, + claim, + phase, + reply, + }) = request + { + let outcome = targets + .iter() + .position(|(lease, _, _)| lease.as_ptr() == Arc::as_ptr(&target)) + .map_or(WriteOutcome::NotWritten, |index| { + let (_, identity, baseline) = targets.swap_remove(index); + if !active.load(Ordering::Acquire) + || !phase.pending() + || text.is_empty() + || text.len() > MAX_DOCUMENT_UTF16 * 4 + || !baseline.permits(operation) + { + return WriteOutcome::NotWritten; + } + platform.write(&identity, &baseline, &text, || phase.claim(active, claim)) + }); + let _ = reply.send(outcome); + continue; + } + + platform.poll(); + let front = platform.foreground(); + let revision = epoch.load(Ordering::Acquire); + let capture = match request { + Some(Request::Capture(reply)) if !reply.is_closed() => Some(reply), + _ => None, + }; + latest.refresh(front, revision, Instant::now(), capture.is_some(), |key| { + platform.bind(key) + }); + platform.poll(); + let stable = |platform: &native::Platform| { + platform.foreground() == front && epoch.load(Ordering::Acquire) == revision + }; + if !stable(&platform) { + latest.bound = None; + } + if let Some(reply) = capture { + let mut captured = CapturedTarget::unavailable(); + if let Some((identity, selection, baseline)) = latest.bound.as_ref().and_then(|bound| { + platform + .capture(bound) + .map(|(selection, baseline)| (bound, selection, baseline)) + }) { + platform.poll(); + if active.load(Ordering::Acquire) && !reply.is_closed() && stable(&platform) { + captured.selection = selection; + if let Some(baseline) = baseline { + let lease = Arc::new(()); + targets.push((Arc::downgrade(&lease), identity.clone(), baseline)); + captured.write_target = Some(WriteTarget(lease)); + } + } + } + if captured.selection == ExternalSelection::Unavailable { + latest.bound = None; + } + let _ = reply.send(captured); + } + } +} + +#[derive(PartialEq, Eq)] +struct Baseline { + document: Vec, + selection: Range, +} + +impl Baseline { + fn new(document: Vec, selection: Range) -> Option { + let selected = document.get(selection.clone())?; + if document.len() > MAX_DOCUMENT_UTF16 + || char::decode_utf16(document.iter().copied()).any(|unit| unit.is_err()) + || !utf16_boundary(&document, selection.start) + || !utf16_boundary(&document, selection.end) + || selection_from_units(selected).is_none() + { + return None; + } + Some(Self { + document, + selection, + }) + } + + fn permits(&self, operation: WriteOperation) -> bool { + match operation { + WriteOperation::Insert => self.selection.is_empty(), + WriteOperation::Replace => !self.selection.is_empty(), + } + } + + fn expected(&self, text: &[u16]) -> Option> { + let len = self.document.len() - self.selection.len() + text.len(); + if len > MAX_DOCUMENT_UTF16 { + return None; + } + let mut result = Vec::with_capacity(len); + result.extend_from_slice(&self.document[..self.selection.start]); + result.extend_from_slice(text); + result.extend_from_slice(&self.document[self.selection.end..]); + Some(result) + } + + fn selected(&self) -> &[u16] { + &self.document[self.selection.clone()] + } +} + +fn utf16_boundary(units: &[u16], offset: usize) -> bool { + units + .get(offset) + .is_none_or(|unit| !(0xDC00..=0xDFFF).contains(unit)) +} + +fn selection_from_units(units: &[u16]) -> Option { + if units.is_empty() { + return Some(ExternalSelection::None); + } + if units.len() > MAX_SELECTION_UTF16 { + return None; + } + let text = String::from_utf16(units).ok()?; + (text.chars().count() <= MAX_SELECTION_SCALARS).then_some(ExternalSelection::Selected(text)) +} + +const PENDING: u8 = 0; +const AUTHORIZING: u8 = 1; +const WRITING: u8 = 2; +const DECLINED: u8 = 3; +const ABANDONED: u8 = 4; + +#[derive(Default)] +struct DeliveryPhase(AtomicU8); + +impl DeliveryPhase { + fn pending(&self) -> bool { + self.0.load(Ordering::Acquire) == PENDING + } + + fn claim(&self, active: &AtomicBool, claim: Claim) -> Result<(), WriteOutcome> { + if !active.load(Ordering::Acquire) + || self + .0 + .compare_exchange(PENDING, AUTHORIZING, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(WriteOutcome::NotWritten); + } + let (next, result) = if !claim() { + (DECLINED, Err(WriteOutcome::Declined)) + } else if !active.load(Ordering::Acquire) { + (ABANDONED, Err(WriteOutcome::NotWritten)) + } else { + (WRITING, Ok(())) + }; + self.0 + .compare_exchange(AUTHORIZING, next, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| WriteOutcome::NotWritten)?; + result + } + + fn abandon(&self) -> WriteOutcome { + match self.0.swap(ABANDONED, Ordering::AcqRel) { + WRITING => WriteOutcome::Unknown, + DECLINED => WriteOutcome::Declined, + _ => WriteOutcome::NotWritten, + } + } +} + +struct AbandonOnDrop(Arc); + +impl Drop for AbandonOnDrop { + fn drop(&mut self) { + self.0.abandon(); + } +} + +#[cfg(test)] +#[path = "target/tests.rs"] +mod tests; diff --git a/openless-all/app/src-tauri/src/ask_window/target/README.md b/openless-all/app/src-tauri/src/ask_window/target/README.md new file mode 100644 index 000000000..bdb32e470 --- /dev/null +++ b/openless-all/app/src-tauri/src/ask_window/target/README.md @@ -0,0 +1,10 @@ +# Ask external editor adapter + +Start one `TargetTracker` at desktop setup, before Ask can activate, and stop it during desktop shutdown (including restart). `capture().await` is for the winning admission only, after deduplication and busy checks. Forward `CapturedTarget.selection` to Core and wrap `write_target` in the writer port if present. `deliver().await` calls the one-shot Core claim on the native worker, after identity and baseline checks; the callback must be synchronous and should not perform I/O. Neither API needs `spawn_blocking` around it. The tracker does not send clipboard, key or UI events. + +- macOS requires Accessibility authorization and an AX text field/area exposing a verifiable focused window, control and single UTF-16 range. Range text is read directly; writes require a bounded document snapshot and a settable `AXSelectedText` on the same element. Some providers expose readonly selection but no writable range; others stop exposing their focused control while inactive or do not emit focus notifications. These are not write targets. macOS may deny restoration of a different application's focus. +- Windows reads UIA `TextPattern` selection without activation or Copy. Writes are restricted to a verified Unicode Win32 `Edit` control whose own packed `EM_GETSEL` range agrees with the UIA range, using `EM_REPLACESEL`. RichEdit, browser/WebView, custom/virtual TextPattern and ValuePattern-only providers do not have a supported range mutation here. An inaccessible virtual provider, denied foreground activation, read-only state, or a result that cannot be written verbatim on a single line does not authorize a write. +- Both adapters reject password/secure or otherwise unverifiable targets. The selected limit is 4,000 Unicode scalars / 8,000 UTF-16 units; a write baseline is at most 20,000 UTF-16 units and never leaves the native worker. Selection-only reads need no whole-document baseline. Focus events plus submit-time native identity checks invalidate stale observations; an external visit that cannot be bound is not replaced with an older target. Only the original app/window may be restored, never a different editor or range. +- Neither OS exposes an atomic compare-and-replace operation for these adapters. A provider can still mutate between the last check and dispatch. `Written` requires the exact expected document after the native call; timeout, uncertain completion, transformation or unverified post-state returns `Unknown` and is not retried. The ticket is consumed on the first delivery attempt, even if no mutation occurred. Clipboard fallback and its cancellation are Core/host policy. + +For the ignored macOS native acceptance test, compile `openless-all/app/scripts/native-live-input-fixture.swift` and launch it with `--ask-selected`. Run `real_mac_editor_replaces_only_the_verified_fixture_selection` with `OPENLESS_NATIVE_FIXTURE_PID` set to that disposable editor's PID and with Accessibility access. The test refuses to write unless the fixture is still frontmost and its expected selection is verified. diff --git a/openless-all/app/src-tauri/src/ask_window/target/macos.rs b/openless-all/app/src-tauri/src/ask_window/target/macos.rs new file mode 100644 index 000000000..2e7034f82 --- /dev/null +++ b/openless-all/app/src-tauri/src/ask_window/target/macos.rs @@ -0,0 +1,515 @@ +use super::*; +use crate::host_document::{evaluate_gate, macos::*, GateInputs}; +use block2::RcBlock; +use core_foundation::{ + array::CFArray, + base::{CFRange, CFType, TCFType}, + number::CFNumber, + runloop::{kCFRunLoopDefaultMode, CFRunLoop, CFRunLoopSource}, + string::{CFString, CFStringGetCharacters, CFStringGetLength, CFStringRef}, +}; +use objc2::{ + msg_send, + rc::{autoreleasepool, Retained}, + runtime::{AnyClass, AnyObject, Bool}, +}; +use std::{ffi::c_void, ptr, thread}; + +const AX_RANGE: i32 = 4; + +#[link(name = "ApplicationServices", kind = "framework")] +extern "C" { + fn AXIsProcessTrusted() -> bool; + fn AXUIElementGetTypeID() -> usize; + fn AXValueGetTypeID() -> usize; + fn AXUIElementIsAttributeSettable( + element: AxUiElementRef, + name: CFStringRef, + out: *mut u8, + ) -> i32; +} + +#[link(name = "AppKit", kind = "framework")] +extern "C" { + static NSWorkspaceDidActivateApplicationNotification: *const AnyObject; + static NSWorkspaceApplicationKey: *const AnyObject; +} + +const AX_UNSUPPORTED: i32 = -25205; +const AX_NO_VALUE: i32 = -25212; +const AX_MESSAGE_SECONDS: f32 = 0.065; +const FOCUS_NOTIFICATIONS: [&str; 2] = ["AXFocusedUIElementChanged", "AXFocusedWindowChanged"]; + +pub(super) struct Platform { + center: Retained, + token: Retained, + watch: Option, + epoch: Arc, +} + +#[derive(Clone)] +pub(super) struct Identity { + pid: i32, + application: Retained, + ax_app: CFType, + window: CFType, + element: CFType, +} + +struct FocusWatch { + pid: i32, + app: CFType, + source: CFRunLoopSource, + _observer: CFType, + _context: Box, +} + +struct FocusContext { + pid: i32, + epoch: Arc, +} + +unsafe extern "C" fn focus_changed( + _: AxObserverRef, + _: AxUiElementRef, + _: *const c_void, + context: *mut c_void, +) { + let context = &*context.cast::(); + if frontmost().is_some_and(|app| pid(&app) == context.pid) { + context.epoch.fetch_add(1, Ordering::AcqRel); + } +} + +impl FocusWatch { + fn new(pid: i32, epoch: Arc) -> Option { + unsafe { + let app = ax(AXUIElementCreateApplication(pid), pid)?; + let mut observer = ptr::null_mut(); + if AXObserverCreate(pid, focus_changed, &mut observer) != 0 || observer.is_null() { + return None; + } + let mut context = Box::new(FocusContext { pid, epoch }); + let observer = CFType::wrap_under_create_rule(observer.cast()); + for name in FOCUS_NOTIFICATIONS { + let name = CFString::new(name); + if AXObserverAddNotification( + observer.as_CFTypeRef().cast_mut().cast(), + raw_ax(&app), + name.as_CFTypeRef(), + ptr::from_mut(&mut *context).cast(), + ) != 0 + { + return None; + } + } + let source = AXObserverGetRunLoopSource(observer.as_CFTypeRef().cast_mut().cast()); + if source.is_null() { + return None; + } + let source = CFRunLoopSource::wrap_under_get_rule(source); + CFRunLoop::get_current().add_source(&source, kCFRunLoopDefaultMode); + Some(Self { + pid, + app, + source, + _observer: observer, + _context: context, + }) + } + } +} + +impl Drop for FocusWatch { + fn drop(&mut self) { + unsafe { CFRunLoop::get_current().remove_source(&self.source, kCFRunLoopDefaultMode) } + } +} + +fn workspace() -> Option<*mut AnyObject> { + unsafe { + let class = AnyClass::get("NSWorkspace")?; + let workspace: *mut AnyObject = msg_send![class, sharedWorkspace]; + (!workspace.is_null()).then_some(workspace) + } +} + +fn frontmost() -> Option> { + unsafe { Retained::retain(msg_send![workspace()?, frontmostApplication]) } +} + +fn pid(app: &AnyObject) -> i32 { + unsafe { msg_send![app, processIdentifier] } +} + +fn secure_input() -> bool { + crate::unicode_keystroke::is_secure_input_enabled() +} + +impl Platform { + pub(super) fn new(epoch: Arc) -> Option { + autoreleasepool(|_| unsafe { + let center: Retained = + Retained::retain(msg_send![workspace()?, notificationCenter])?; + let external_epoch = epoch.clone(); + let block = RcBlock::new(move |notification: *mut AnyObject| { + let info: *mut AnyObject = msg_send![notification, userInfo]; + let app: *mut AnyObject = msg_send![info, objectForKey: NSWorkspaceApplicationKey]; + if app + .as_ref() + .is_none_or(|app| pid(app) as u32 != std::process::id()) + { + external_epoch.fetch_add(1, Ordering::AcqRel); + } + }); + let token = Retained::retain(msg_send![ + &*center, addObserverForName: NSWorkspaceDidActivateApplicationNotification, + object: ptr::null::(), queue: ptr::null::(), usingBlock: &*block + ])?; + Some(Self { + center, + token, + watch: None, + epoch, + }) + }) + } + + pub(super) fn poll(&mut self) { + autoreleasepool(|_| unsafe { + CFRunLoop::run_in_mode(kCFRunLoopDefaultMode, Duration::ZERO, false); + }); + } + + pub(super) fn foreground(&self) -> Foreground { + autoreleasepool(|_| match frontmost().map(|app| pid(&app)) { + Some(pid) if pid as u32 == std::process::id() => Foreground::Own, + Some(pid) if pid > 0 => Foreground::External(pid), + _ => Foreground::Unknown, + }) + } + + pub(super) fn bind(&mut self, expected_pid: i32) -> Option { + autoreleasepool(|_| unsafe { + if self + .watch + .as_ref() + .is_some_and(|watch| watch.pid != expected_pid) + { + self.watch = None; + } + if secure_input() || !AXIsProcessTrusted() { + return None; + } + let application = frontmost().filter(|app| pid(app) == expected_pid)?; + let bundle_id = { + let bundle: *const AnyObject = msg_send![&*application, bundleIdentifier]; + (!bundle.is_null()) + .then(|| CFString::wrap_under_get_rule(bundle.cast()).to_string()) + }; + if evaluate_gate(&GateInputs { + bundle_id, + ..GateInputs::default() + }) + .is_some() + { + return None; + } + if self.watch.is_none() { + self.watch = FocusWatch::new(expected_pid, self.epoch.clone()); + } + let ax_app = self.watch.as_ref()?.app.clone(); + let window = element_attr(&ax_app, "AXFocusedWindow", expected_pid)?; + let element = element_attr(&ax_app, "AXFocusedUIElement", expected_pid)?; + Some(Identity { + pid: expected_pid, + application, + ax_app, + window, + element, + }) + }) + } + + pub(super) fn capture( + &self, + identity: &Identity, + ) -> Option<(ExternalSelection, Option)> { + autoreleasepool(|_| { + identity.verify()?; + let range = selected_range(&identity.element)?; + let units = text_for_range(&identity.element, range.clone(), MAX_SELECTION_UTF16)?; + let selection = selection_from_units(&units)?; + let baseline = settable(&identity.element) + .then(|| identity.baseline()) + .flatten() + .filter(|baseline| baseline.selection == range && baseline.selected() == units); + if selected_range(&identity.element)? != range { + return None; + } + identity.verify()?; + Some((selection, baseline)) + }) + } + + pub(super) fn write( + &self, + identity: &Identity, + baseline: &Baseline, + text: &str, + claim: impl FnOnce() -> Result<(), WriteOutcome>, + ) -> WriteOutcome { + autoreleasepool(|_| { + let text_units: Vec<_> = text.encode_utf16().collect(); + let Some(expected) = baseline.expected(&text_units) else { + return WriteOutcome::NotWritten; + }; + if identity.verify().is_none() + || !settable(&identity.element) + || identity.baseline().as_ref() != Some(baseline) + || !identity.restore_front() + || identity.verify().is_none() + || identity.baseline().as_ref() != Some(baseline) + { + return WriteOutcome::NotWritten; + } + let name = CFString::new("AXSelectedText"); + let value = CFString::new(text); + if identity.verify().is_none() + || self.foreground() != Foreground::External(identity.pid) + || selected_range(&identity.element).as_ref() != Some(&baseline.selection) + || secure_input() + { + return WriteOutcome::NotWritten; + } + if let Err(outcome) = claim() { + return outcome; + } + let status = unsafe { + AXUIElementSetAttributeValue( + raw_ax(&identity.element), + name.as_CFTypeRef(), + value.as_CFTypeRef(), + ) + }; + if status == 0 && document(&identity.element).as_ref() == Some(&expected) { + WriteOutcome::Written + } else { + WriteOutcome::Unknown + } + }) + } +} + +impl Drop for Platform { + fn drop(&mut self) { + unsafe { + let _: () = msg_send![&*self.center, removeObserver: &*self.token]; + } + } +} + +impl Identity { + fn verify(&self) -> Option<()> { + let terminated: Bool = unsafe { msg_send![&*self.application, isTerminated] }; + if terminated.as_bool() || secure_input() { + return None; + } + let role = attr(&self.element, "AXRole") + .ok()? + .downcast_into::()?; + if !matches!(role.to_string().as_str(), "AXTextField" | "AXTextArea") { + return None; + } + let subrole = match attr(&self.element, "AXSubrole") { + Ok(value) => Some(value.downcast_into::()?.to_string()), + Err(AX_UNSUPPORTED | AX_NO_VALUE) => None, + Err(_) => return None, + }; + if evaluate_gate(&GateInputs { + subrole, + ..GateInputs::default() + }) + .is_some() + || element_attr(&self.ax_app, "AXFocusedWindow", self.pid)? != self.window + || element_attr(&self.ax_app, "AXFocusedUIElement", self.pid)? != self.element + || element_attr(&self.element, "AXWindow", self.pid)? != self.window + { + return None; + } + Some(()) + } + + fn restore_front(&self) -> bool { + let is_front = || frontmost().is_some_and(|app| pid(&app) == self.pid); + if !is_front() { + let activated: Bool = + unsafe { msg_send![&*self.application, activateWithOptions: 2usize] }; + if !activated.as_bool() { + return false; + } + for _ in 0..12 { + if is_front() { + return true; + } + thread::sleep(Duration::from_millis(15)); + } + } + is_front() + } + + fn baseline(&self) -> Option { + let range = selected_range(&self.element)?; + let document = document(&self.element)?; + if selected_range(&self.element)? != range { + return None; + } + Baseline::new(document, range) + } +} + +fn attr(element: &CFType, name: &str) -> Result { + let name = CFString::new(name); + let mut value = ptr::null(); + let status = + unsafe { AXUIElementCopyAttributeValue(raw_ax(element), name.as_CFTypeRef(), &mut value) }; + if status != 0 { + Err(status) + } else if value.is_null() { + Err(AX_NO_VALUE) + } else { + Ok(unsafe { CFType::wrap_under_create_rule(value) }) + } +} + +unsafe fn ax(raw: AxUiElementRef, pid: i32) -> Option { + if raw.is_null() { + return None; + } + validate_ax(CFType::wrap_under_create_rule(raw.cast()), pid) +} + +fn raw_ax(value: &CFType) -> AxUiElementRef { + value.as_CFTypeRef().cast_mut().cast() +} + +fn validate_ax(value: CFType, expected_pid: i32) -> Option { + let mut pid = 0; + let raw = raw_ax(&value); + unsafe { + (value.type_of() == AXUIElementGetTypeID() + && AXUIElementGetPid(raw, &mut pid) == 0 + && pid == expected_pid + && AXUIElementSetMessagingTimeout(raw, AX_MESSAGE_SECONDS) == 0) + .then_some(value) + } +} + +fn element_attr(element: &CFType, name: &str, pid: i32) -> Option { + validate_ax(attr(element, name).ok()?, pid) +} + +fn get_range(value: &CFType) -> Option> { + let mut raw = CFRange { + location: 0, + length: 0, + }; + unsafe { + if value.type_of() != AXValueGetTypeID() + || AXValueGetValue( + value.as_CFTypeRef(), + AX_RANGE, + ptr::from_mut(&mut raw).cast(), + ) == 0 + { + return None; + } + } + let start = usize::try_from(raw.location).ok()?; + let len = usize::try_from(raw.length).ok()?; + Some(start..start.checked_add(len)?) +} + +fn selected_range(element: &CFType) -> Option> { + let range = get_range(&attr(element, "AXSelectedTextRange").ok()?)?; + match attr(element, "AXSelectedTextRanges") { + Ok(value) => { + let ranges = value.downcast_into::()?; + let first = *ranges.get(0)?; + if ranges.len() != 1 || first.is_null() { + return None; + } + let first = unsafe { CFType::wrap_under_get_rule(first) }; + if get_range(&first)? != range { + return None; + } + } + Err(AX_UNSUPPORTED | AX_NO_VALUE) => {} + Err(_) => return None, + } + Some(range) +} + +fn settable(element: &CFType) -> bool { + let name = CFString::new("AXSelectedText"); + let mut settable = 0; + unsafe { + AXUIElementIsAttributeSettable(raw_ax(element), name.as_concrete_TypeRef(), &mut settable) + == 0 + && settable != 0 + } +} + +fn document(element: &CFType) -> Option> { + let count = attr(element, "AXNumberOfCharacters") + .ok()? + .downcast_into::()? + .to_i64()?; + text_for_range(element, 0..usize::try_from(count).ok()?, MAX_DOCUMENT_UTF16) +} + +fn text_for_range(element: &CFType, range: Range, cap: usize) -> Option> { + if range.len() > cap { + return None; + } + if range.is_empty() { + return Some(Vec::new()); + } + let raw = CFRange { + location: isize::try_from(range.start).ok()?, + length: isize::try_from(range.len()).ok()?, + }; + let name = CFString::new("AXStringForRange"); + unsafe { + let parameter = AXValueCreate(AX_RANGE, ptr::from_ref(&raw).cast()); + if parameter.is_null() { + return None; + } + let parameter = CFType::wrap_under_create_rule(parameter); + let mut value = ptr::null(); + let status = AXUIElementCopyParameterizedAttributeValue( + raw_ax(element), + name.as_CFTypeRef(), + parameter.as_CFTypeRef(), + &mut value, + ); + if status != 0 || value.is_null() { + return None; + } + let string = CFType::wrap_under_create_rule(value).downcast_into::()?; + let string = string.as_concrete_TypeRef(); + if CFStringGetLength(string) != raw.length { + return None; + } + let mut units = vec![0; range.len()]; + CFStringGetCharacters( + string, + CFRange { + location: 0, + length: raw.length, + }, + units.as_mut_ptr(), + ); + Some(units) + } +} diff --git a/openless-all/app/src-tauri/src/ask_window/target/tests.rs b/openless-all/app/src-tauri/src/ask_window/target/tests.rs new file mode 100644 index 000000000..51bd113e9 --- /dev/null +++ b/openless-all/app/src-tauri/src/ask_window/target/tests.rs @@ -0,0 +1,176 @@ +use super::*; +use std::cell::Cell; + +#[cfg(target_os = "macos")] +#[tokio::test] +#[ignore = "requires Accessibility permission and the disposable native-live-input-fixture app"] +async fn real_mac_editor_replaces_only_the_verified_fixture_selection() { + let pid: i32 = std::env::var("OPENLESS_NATIVE_FIXTURE_PID") + .expect("provide the disposable native-live-input-fixture process ID") + .parse() + .unwrap(); + let tracker = TargetTracker::new(); + tracker.start().unwrap(); + let mut captured = None; + for _ in 0..12 { + if crate::selection::current_front_app_pid() != Some(pid) { + tokio::time::sleep(Duration::from_millis(80)).await; + continue; + } + let target = tracker.capture().await; + if crate::selection::current_front_app_pid() == Some(pid) + && target.selection == ExternalSelection::Selected("原文".into()) + { + if let Some(write_target) = target.write_target { + captured = Some(write_target); + break; + } + } + tokio::time::sleep(Duration::from_millis(80)).await; + } + let target = captured.expect("the original disposable selection must be writable"); + assert_eq!(crate::selection::current_front_app_pid(), Some(pid)); + let outcome = tracker + .deliver( + &target, + WriteOperation::Replace, + "Ask fixture result".into(), + Box::new(move || crate::selection::current_front_app_pid() == Some(pid)), + ) + .await; + tracker.stop(); + assert_eq!(outcome, WriteOutcome::Written); +} + +#[test] +fn own_focus_preserves_bound_editor_but_an_unreadable_external_visit_clears_it() { + let now = Instant::now(); + let mut latest = Latest::new(now); + latest.refresh(Foreground::External(1), 1, now, false, |_| Some("first")); + latest.refresh(Foreground::Own, 1, now, true, |_| panic!("do not bind Ask")); + assert_eq!(latest.bound, Some("first")); + + latest.refresh(Foreground::External(2), 2, now, false, |_| None); + latest.refresh(Foreground::Own, 2, now, true, |_| panic!("do not bind Ask")); + assert_eq!(latest.bound, None); +} + +#[test] +fn latest_external_observation_replaces_another_control_in_the_same_app() { + let now = Instant::now(); + let mut latest = Latest::new(now); + latest.refresh(Foreground::External(1), 1, now, false, |_| Some("first")); + latest.refresh(Foreground::External(1), 2, now, false, |_| Some("second")); + latest.refresh(Foreground::Own, 2, now, true, |_| panic!("do not bind Ask")); + assert_eq!(latest.bound, Some("second")); +} + +#[test] +fn unsampled_external_transition_invalidates_older_focus_and_submit_forces_refresh() { + let now = Instant::now(); + let mut latest = Latest::new(now); + latest.refresh(Foreground::External(1), 1, now, false, |_| Some("old")); + latest.refresh(Foreground::Own, 2, now, true, |_| panic!("do not bind Ask")); + assert_eq!(latest.bound, None); + latest.refresh(Foreground::External(1), 3, now, false, |_| Some("observed")); + latest.refresh(Foreground::External(1), 3, now, true, |_| Some("at send")); + assert_eq!(latest.bound, Some("at send")); +} + +#[test] +fn replacement_uses_original_utf16_offsets_not_the_first_matching_text() { + let doc: Vec<_> = "🙂red\r\nred".encode_utf16().collect(); + let baseline = Baseline::new(doc.clone(), 7..10).unwrap(); + assert!(!baseline.permits(WriteOperation::Insert)); + assert!(baseline.permits(WriteOperation::Replace)); + let expected = baseline.expected(&[b'X' as u16]).unwrap(); + assert_eq!(String::from_utf16(&expected).unwrap(), "🙂red\r\nX"); + assert!(Baseline::new(doc.clone(), 1..1).is_none()); + assert!(Baseline::new(doc.clone(), 7..11).is_none()); + assert!(Baseline::new(doc.clone(), 2..5).unwrap() != baseline); + let mut changed_away_from_range = doc; + changed_away_from_range[2] = b'X' as u16; + assert!(Baseline::new(changed_away_from_range, 7..10).unwrap() != baseline); +} + +#[test] +fn insertion_requires_a_collapsed_position_and_preserves_everything_around_it() { + let doc: Vec<_> = "🙂\nword".encode_utf16().collect(); + let caret = Baseline::new(doc, 2..2).unwrap(); + assert!(caret.permits(WriteOperation::Insert)); + assert!(!caret.permits(WriteOperation::Replace)); + let generated: Vec<_> = "❄".encode_utf16().collect(); + assert_eq!( + String::from_utf16(&caret.expected(&generated).unwrap()).unwrap(), + "🙂❄\nword" + ); +} + +#[test] +fn selection_keeps_whitespace_and_rejects_overlong_or_malformed_data() { + assert_eq!(selection_from_units(&[]), Some(ExternalSelection::None)); + assert_eq!( + selection_from_units(&" \r\n🙂 ".encode_utf16().collect::>()), + Some(ExternalSelection::Selected(" \r\n🙂 ".into())) + ); + assert_eq!(selection_from_units(&[0xdc00]), None); + assert_eq!( + selection_from_units(&vec![b'x' as u16; MAX_SELECTION_SCALARS + 1]), + None + ); +} + +#[test] +fn abandoned_and_stopped_attempts_cannot_run_the_authorization_callback() { + let phase = DeliveryPhase::default(); + let running = AtomicBool::new(true); + assert_eq!(phase.abandon(), WriteOutcome::NotWritten); + assert_eq!( + phase.claim(&running, Box::new(|| panic!("expired"))), + Err(WriteOutcome::NotWritten) + ); + let phase = DeliveryPhase::default(); + running.store(false, Ordering::Release); + assert_eq!( + phase.claim(&running, Box::new(|| panic!("stopped"))), + Err(WriteOutcome::NotWritten) + ); +} + +#[test] +fn declined_claim_does_not_write_and_a_claimed_timeout_is_unknown() { + let running = AtomicBool::new(true); + let phase = DeliveryPhase::default(); + assert_eq!( + phase.claim(&running, Box::new(|| false)), + Err(WriteOutcome::Declined) + ); + assert_eq!(phase.abandon(), WriteOutcome::Declined); + + let phase = DeliveryPhase::default(); + assert_eq!(phase.claim(&running, Box::new(|| true)), Ok(())); + assert_eq!( + phase.claim(&running, Box::new(|| panic!("double claim"))), + Err(WriteOutcome::NotWritten) + ); + assert_eq!(phase.abandon(), WriteOutcome::Unknown); +} + +#[test] +fn timeout_during_authorization_does_not_start_a_late_write() { + let phase = Arc::new(DeliveryPhase::default()); + let timing = phase.clone(); + let outcome = Cell::new(None); + let active = AtomicBool::new(true); + assert_eq!( + phase.claim( + &active, + Box::new(move || { + outcome.set(Some(timing.abandon())); + assert_eq!(outcome.take(), Some(WriteOutcome::NotWritten)); + true + }), + ), + Err(WriteOutcome::NotWritten) + ); +} diff --git a/openless-all/app/src-tauri/src/ask_window/target/unsupported.rs b/openless-all/app/src-tauri/src/ask_window/target/unsupported.rs new file mode 100644 index 000000000..61a541c7e --- /dev/null +++ b/openless-all/app/src-tauri/src/ask_window/target/unsupported.rs @@ -0,0 +1,35 @@ +use super::*; + +pub(super) struct Platform; +#[derive(Clone)] +pub(super) struct Identity; + +impl Platform { + pub(super) fn new(_: Arc) -> Option { + None + } + + pub(super) fn poll(&mut self) {} + + pub(super) fn foreground(&self) -> Foreground<()> { + Foreground::Unknown + } + + pub(super) fn bind(&mut self, _: ()) -> Option { + None + } + + pub(super) fn capture(&self, _: &Identity) -> Option<(ExternalSelection, Option)> { + None + } + + pub(super) fn write( + &self, + _: &Identity, + _: &Baseline, + _: &str, + _: impl FnOnce() -> Result<(), WriteOutcome>, + ) -> WriteOutcome { + WriteOutcome::NotWritten + } +} diff --git a/openless-all/app/src-tauri/src/ask_window/target/windows.rs b/openless-all/app/src-tauri/src/ask_window/target/windows.rs new file mode 100644 index 000000000..d3c09ed16 --- /dev/null +++ b/openless-all/app/src-tauri/src/ask_window/target/windows.rs @@ -0,0 +1,606 @@ +use super::*; +use crate::host_document::{is_blocked_process_name, uia_runtime_id}; +use std::{cell::RefCell, mem, thread}; +use windows::core::{implement, Interface, Result as WinResult, PWSTR}; +use windows::Win32::{ + Foundation::{CloseHandle, HANDLE, HWND, LPARAM, WAIT_TIMEOUT, WPARAM}, + System::{ + Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }, + Threading::{ + OpenProcess, QueryFullProcessImageNameW, WaitForSingleObject, PROCESS_NAME_WIN32, + PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE, + }, + }, + UI::{Accessibility::*, Input::KeyboardAndMouse::IsWindowEnabled, WindowsAndMessaging::*}, +}; + +const MESSAGE_MS: u32 = 80; +const EM_GETSEL: u32 = 0x00b0; +const EM_REPLACESEL: u32 = 0x00c2; +const EM_GETPASSWORDCHAR: u32 = 0x00d2; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) struct Window { + handle: HWND, + pid: u32, + thread: u32, +} + +#[derive(Clone)] +pub(super) struct Identity { + window: Window, + process: Arc, + root: IUIAutomationElement, + element: IUIAutomationElement, + runtime_id: Vec, + edit: Option, +} + +pub(super) struct Platform { + uia: IUIAutomation, + focus_handler: IUIAutomationFocusChangedEventHandler, + foreground_hook: HWINEVENTHOOK, + _apartment: Apartment, +} + +struct Apartment; + +impl Drop for Apartment { + fn drop(&mut self) { + unsafe { CoUninitialize() } + } +} + +struct Process(HANDLE); + +impl Process { + fn open(pid: u32) -> Option { + unsafe { + let process = Self( + OpenProcess( + PROCESS_QUERY_LIMITED_INFORMATION | PROCESS_SYNCHRONIZE, + false, + pid, + ) + .ok()?, + ); + let mut path = [0u16; 1024]; + let mut count = path.len() as u32; + QueryFullProcessImageNameW( + process.0, + PROCESS_NAME_WIN32, + PWSTR(path.as_mut_ptr()), + &mut count, + ) + .ok()?; + let path = String::from_utf16(&path[..count as usize]).ok()?; + let image = path.rsplit(['\\', '/']).next()?; + (!image.is_empty() && !is_blocked_process_name(image)).then_some(process) + } + } + + fn alive(&self) -> bool { + unsafe { WaitForSingleObject(self.0, 0) == WAIT_TIMEOUT } + } +} + +impl Drop for Process { + fn drop(&mut self) { + let _ = unsafe { CloseHandle(self.0) }; + } +} + +thread_local! { + static FOREGROUND_EPOCH: RefCell>> = const { RefCell::new(None) }; +} + +unsafe extern "system" fn foreground_changed( + _: HWINEVENTHOOK, + _: u32, + window: HWND, + _: i32, + _: i32, + _: u32, + _: u32, +) { + if Window::new(GetAncestor(window, GA_ROOT)).is_some_and(|root| root.pid == std::process::id()) + { + return; + } + FOREGROUND_EPOCH.with(|slot| { + if let Some(epoch) = slot.borrow().as_ref() { + epoch.fetch_add(1, Ordering::AcqRel); + } + }); +} + +#[implement(IUIAutomationFocusChangedEventHandler)] +struct FocusChanged(Arc); + +impl IUIAutomationFocusChangedEventHandler_Impl for FocusChanged_Impl { + fn HandleFocusChangedEvent(&self, element: Option<&IUIAutomationElement>) -> WinResult<()> { + if Window::new(unsafe { GetForegroundWindow() }) + .is_some_and(|window| window.pid == std::process::id()) + { + return Ok(()); + } + let pid = element.and_then(|element| unsafe { element.CachedProcessId().ok() }); + if pid != Some(std::process::id() as i32) { + self.0.fetch_add(1, Ordering::AcqRel); + } + Ok(()) + } +} + +impl Platform { + pub(super) fn new(epoch: Arc) -> Option { + unsafe { + CoInitializeEx(None, COINIT_MULTITHREADED).ok().ok()?; + let apartment = Apartment; + let uia: IUIAutomation = + CoCreateInstance(&CUIAutomation8, None, CLSCTX_INPROC_SERVER).ok()?; + let timeouts: IUIAutomation2 = uia.cast().ok()?; + timeouts.SetConnectionTimeout(MESSAGE_MS).ok()?; + timeouts.SetTransactionTimeout(MESSAGE_MS).ok()?; + let cache = uia.CreateCacheRequest().ok()?; + cache.AddProperty(UIA_ProcessIdPropertyId).ok()?; + let focus_handler: IUIAutomationFocusChangedEventHandler = + FocusChanged(epoch.clone()).into(); + uia.AddFocusChangedEventHandler(&cache, &focus_handler) + .ok()?; + let foreground_hook = SetWinEventHook( + EVENT_SYSTEM_FOREGROUND, + EVENT_SYSTEM_FOREGROUND, + None, + Some(foreground_changed), + 0, + 0, + WINEVENT_OUTOFCONTEXT | WINEVENT_SKIPOWNPROCESS, + ); + if foreground_hook.is_invalid() { + let _ = uia.RemoveFocusChangedEventHandler(&focus_handler); + return None; + } + FOREGROUND_EPOCH.with(|slot| *slot.borrow_mut() = Some(epoch)); + Some(Self { + uia, + focus_handler, + foreground_hook, + _apartment: apartment, + }) + } + } + + pub(super) fn poll(&mut self) { + let mut message = MSG::default(); + unsafe { + for _ in 0..32 { + if !PeekMessageW(&mut message, None, 0, 0, PM_REMOVE).as_bool() { + break; + } + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + } + } + + pub(super) fn foreground(&self) -> Foreground { + match Window::new(unsafe { GetForegroundWindow() }) { + Some(window) if window.pid == std::process::id() => Foreground::Own, + Some(window) => Foreground::External(window), + None => Foreground::Unknown, + } + } + + pub(super) fn bind(&mut self, window: Window) -> Option { + unsafe { + let process = Arc::new(Process::open(window.pid)?); + let root = self.uia.ElementFromHandle(window.handle).ok()?; + let element = self.uia.GetFocusedElement().ok()?; + let handle = element.CurrentNativeWindowHandle().ok()?; + let runtime_id = uia_runtime_id(&element).ok()?; + let edit = Window::new(handle).filter(|candidate| { + candidate.pid == window.pid + && GetAncestor(handle, GA_ROOT) == window.handle + && window.focused_child() == Some(handle) + && standard_edit(handle) + }); + let identity = Identity { + window, + process, + root, + element, + runtime_id, + edit, + }; + self.verify(&identity)?; + self.focus_matches(&identity).then_some(identity) + } + } + + pub(super) fn capture( + &self, + identity: &Identity, + ) -> Option<(ExternalSelection, Option)> { + unsafe { + self.verify(identity)?; + let pattern = identity + .element + .GetCurrentPatternAs::(UIA_TextPatternId) + .ok()?; + let first = one_range(&pattern)?; + let units = range_text(&first)?; + let selection = selection_from_units(&units)?; + let baseline = identity + .edit + .and_then(|edit| identity.write_baseline(edit, &pattern, &first, &units)); + let last = one_range(&pattern)?; + if !same_range(&first, &last)? || range_text(&last)? != units { + return None; + } + self.verify_capture(identity)?; + Some((selection, baseline)) + } + } + + pub(super) fn write( + &self, + identity: &Identity, + baseline: &Baseline, + text: &str, + claim: impl FnOnce() -> Result<(), WriteOutcome>, + ) -> WriteOutcome { + let Some(edit) = identity.edit else { + return WriteOutcome::NotWritten; + }; + let Some(state) = edit + .safe_edit_style() + .filter(|style| style & ES_READONLY == 0) + else { + return WriteOutcome::NotWritten; + }; + if text.contains('\0') || (state & ES_MULTILINE == 0 && text.contains(['\r', '\n'])) { + return WriteOutcome::NotWritten; + } + let text = text.replace("\r\n", "\n").replace('\n', "\r\n"); + let mut units: Vec<_> = text.encode_utf16().collect(); + let Some(expected) = baseline.expected(&units) else { + return WriteOutcome::NotWritten; + }; + units.push(0); + + if self.verify(identity).is_none() + || edit.baseline().as_ref() != Some(baseline) + || !identity.window.restore_front() + || !self.focus_matches(identity) + || edit.safe_edit_style() != Some(state) + || edit.baseline().as_ref() != Some(baseline) + || !self.focus_matches(identity) + || edit.selection().as_ref() != Some(&baseline.selection) + { + return WriteOutcome::NotWritten; + } + if let Err(outcome) = claim() { + return outcome; + } + if edit + .send(EM_REPLACESEL, 1, units.as_ptr() as isize) + .is_some() + && edit.document().as_ref() == Some(&expected) + { + WriteOutcome::Written + } else { + WriteOutcome::Unknown + } + } + + fn verify(&self, target: &Identity) -> Option<()> { + unsafe { + let kind = target.element.CurrentControlType().ok()?; + if !target.process.alive() + || Window::new(target.window.handle)? != target.window + || target.element.CurrentProcessId().ok()? as u32 != target.window.pid + || !self.safe_from_password(target)? + || uia_runtime_id(&target.element).ok()? != target.runtime_id + || (kind != UIA_EditControlTypeId && kind != UIA_DocumentControlTypeId) + { + return None; + } + let current_root = self.uia.ElementFromHandle(target.window.handle).ok()?; + if !self + .uia + .CompareElements(¤t_root, &target.root) + .ok()? + .as_bool() + { + return None; + } + let walker = self.uia.RawViewWalker().ok()?; + let mut node = target.element.clone(); + for _ in 0..24 { + let handle = node.CurrentNativeWindowHandle().ok()?; + if handle == target.window.handle { + return self + .uia + .CompareElements(&node, &target.root) + .ok()? + .as_bool() + .then_some(()); + } + if !handle.0.is_null() && GetAncestor(handle, GA_ROOT) != target.window.handle { + return None; + } + node = walker.GetParentElement(&node).ok()?; + } + None + } + } + + unsafe fn safe_from_password(&self, target: &Identity) -> Option { + if target.element.CurrentIsPassword().ok()?.as_bool() { + return Some(false); + } + if let Some(edit) = target.edit { + return Some(edit.safe_edit_style().is_some()); + } + let property = target + .element + .GetCurrentPropertyValueEx(UIA_IsPasswordPropertyId, true) + .ok()?; + if self.uia.CheckNotSupported(&property).ok()?.as_bool() { + return None; + } + Some(!bool::try_from(&property).ok()?) + } + + fn verify_capture(&self, target: &Identity) -> Option<()> { + self.verify(target)?; + match self.foreground() { + Foreground::Own => Some(()), + Foreground::External(window) + if window == target.window && self.focus_matches(target) => + { + Some(()) + } + _ => None, + } + } + + fn focus_matches(&self, target: &Identity) -> bool { + let matches = || unsafe { + let current = self.uia.GetFocusedElement().ok()?; + let same = self + .uia + .CompareElements(¤t, &target.element) + .ok()? + .as_bool() + && uia_runtime_id(¤t).ok()? == target.runtime_id; + Some(same && self.safe_from_password(target)?) + }; + self.foreground() == Foreground::External(target.window) + && matches() == Some(true) + && self.foreground() == Foreground::External(target.window) + && target + .edit + .is_none_or(|edit| target.window.focused_child() == Some(edit.handle)) + } +} + +impl Drop for Platform { + fn drop(&mut self) { + unsafe { + let _ = UnhookWinEvent(self.foreground_hook); + let _ = self.uia.RemoveFocusChangedEventHandler(&self.focus_handler); + } + FOREGROUND_EPOCH.with(|slot| *slot.borrow_mut() = None); + } +} + +impl Identity { + unsafe fn write_baseline( + &self, + edit: Window, + pattern: &IUIAutomationTextPattern, + range: &IUIAutomationTextRange, + selected: &[u16], + ) -> Option { + if self.window.focused_child()? != edit.handle || edit.safe_edit_style()? & ES_READONLY != 0 + { + return None; + } + let baseline = edit.baseline()?; + if baseline.selected() != selected + || uia_prefix(pattern, range, baseline.selection.start)?.as_wide() + != &baseline.document[..baseline.selection.start] + { + return None; + } + Some(baseline) + } +} + +impl Window { + fn new(handle: HWND) -> Option { + if handle.0.is_null() { + return None; + } + let mut pid = 0; + let thread = unsafe { GetWindowThreadProcessId(handle, Some(&mut pid)) }; + (thread != 0 && pid != 0).then_some(Self { + handle, + pid, + thread, + }) + } + + fn focused_child(self) -> Option { + let mut state = GUITHREADINFO { + cbSize: mem::size_of::() as u32, + ..Default::default() + }; + unsafe { GetGUIThreadInfo(self.thread, &mut state).ok()? }; + (state.hwndActive == self.handle && !state.hwndFocus.0.is_null()).then_some(state.hwndFocus) + } + + fn restore_front(self) -> bool { + let is_front = || Self::new(unsafe { GetForegroundWindow() }) == Some(self); + if !is_front() { + if Self::new(self.handle) != Some(self) + || !unsafe { SetForegroundWindow(self.handle).as_bool() } + { + return false; + } + for _ in 0..12 { + if is_front() { + return true; + } + thread::sleep(Duration::from_millis(15)); + } + } + is_front() + } + + fn safe_edit_style(self) -> Option { + if Self::new(self.handle)? != self || !standard_edit(self.handle) { + return None; + } + let style = unsafe { GetWindowLongW(self.handle, GWL_STYLE) }; + if style & ES_PASSWORD != 0 || self.send(EM_GETPASSWORDCHAR, 0, 0)? != 0 { + return None; + } + Some(if unsafe { IsWindowEnabled(self.handle).as_bool() } { + style + } else { + style | ES_READONLY + }) + } + + fn baseline(self) -> Option { + let range = self.selection()?; + let document = self.document()?; + if self.selection()? != range { + return None; + } + Baseline::new(document, range) + } + + fn document(self) -> Option> { + let count = self.send(WM_GETTEXTLENGTH, 0, 0)?; + if count > MAX_DOCUMENT_UTF16 { + return None; + } + let mut units = vec![0u16; count + 2]; + let copied = self.send(WM_GETTEXT, units.len(), units.as_mut_ptr() as isize)?; + if copied != count || self.send(WM_GETTEXTLENGTH, 0, 0)? != count { + return None; + } + units.truncate(count); + Some(units) + } + + fn selection(self) -> Option> { + let packed = self.send(EM_GETSEL, 0, 0)?; + if packed >= u32::MAX as usize { + return None; + } + let start = packed & 0xffff; + let end = packed >> 16; + (start <= end).then_some(start..end) + } + + fn send(self, message: u32, wparam: usize, lparam: isize) -> Option { + let mut result = 0; + unsafe { + let sent = SendMessageTimeoutW( + self.handle, + message, + WPARAM(wparam), + LPARAM(lparam), + SMTO_ABORTIFHUNG | SMTO_BLOCK | SMTO_ERRORONEXIT, + MESSAGE_MS, + Some(&mut result), + ); + (sent.0 != 0).then_some(result) + } + } +} + +fn standard_edit(handle: HWND) -> bool { + let mut class = [0u16; 16]; + let mut real_class = [0u16; 16]; + unsafe { + let len = GetClassNameW(handle, &mut class); + let real_len = RealGetWindowClassW(handle, &mut real_class); + IsWindowUnicode(handle).as_bool() + && len == 4 + && real_len == 4 + && class[..4].eq(&[b'E' as u16, b'd' as u16, b'i' as u16, b't' as u16]) + && class[..4] == real_class[..4] + } +} + +unsafe fn one_range(pattern: &IUIAutomationTextPattern) -> Option { + let ranges = pattern.GetSelection().ok()?; + (ranges.Length().ok()? == 1).then(|| ranges.GetElement(0).ok())? +} + +unsafe fn uia_prefix( + pattern: &IUIAutomationTextPattern, + range: &IUIAutomationTextRange, + length: usize, +) -> Option { + let prefix = pattern.DocumentRange().ok()?; + prefix + .MoveEndpointByRange( + TextPatternRangeEndpoint_End, + range, + TextPatternRangeEndpoint_Start, + ) + .ok()?; + prefix.GetText((length + 1) as i32).ok() +} + +unsafe fn same_range( + first: &IUIAutomationTextRange, + last: &IUIAutomationTextRange, +) -> Option { + Some( + first + .CompareEndpoints( + TextPatternRangeEndpoint_Start, + last, + TextPatternRangeEndpoint_Start, + ) + .ok()? + == 0 + && first + .CompareEndpoints( + TextPatternRangeEndpoint_End, + last, + TextPatternRangeEndpoint_End, + ) + .ok()? + == 0, + ) +} + +unsafe fn range_text(range: &IUIAutomationTextRange) -> Option> { + match range + .CompareEndpoints( + TextPatternRangeEndpoint_Start, + range, + TextPatternRangeEndpoint_End, + ) + .ok()? + { + 0 => Some(Vec::new()), + order if order < 0 => { + let text = range.GetText((MAX_SELECTION_UTF16 + 1) as i32).ok()?; + (!text.is_empty() && text.len() <= MAX_SELECTION_UTF16).then(|| text.as_wide().to_vec()) + } + _ => None, + } +} diff --git a/openless-all/app/src-tauri/src/ask_window/voice.rs b/openless-all/app/src-tauri/src/ask_window/voice.rs index 75c818d43..83574f142 100644 --- a/openless-all/app/src-tauri/src/ask_window/voice.rs +++ b/openless-all/app/src-tauri/src/ask_window/voice.rs @@ -11,25 +11,12 @@ use openless_core::{ BackendError, BackendErrorCode, CapsulePayload, CapsuleState, SessionId, }; use parking_lot::Mutex; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use tokio::sync::mpsc; use super::Desktop; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "camelCase")] -pub(crate) enum Mode { - Insert, - Submit, -} - -#[derive(Clone, Deserialize, Serialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub(crate) struct Options { - pub(crate) deep_thinking: bool, -} - -#[derive(Clone, Deserialize, Serialize)] +#[derive(Clone, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(crate) struct Editor { pub(crate) conversation_id: String, @@ -39,38 +26,26 @@ pub(crate) struct Editor { pub(crate) editable: bool, pub(crate) selection_start: u32, pub(crate) selection_end: u32, - pub(crate) attachment_ids: Vec, - pub(crate) options: Options, } -#[derive(Clone, Serialize)] -#[serde(rename_all = "camelCase")] +#[derive(Clone)] pub(crate) struct Target { pub(crate) focus_epoch: u64, pub(crate) editor: Editor, } impl Target { - pub(super) fn matches(&self, current: &Self, mode: Mode) -> bool { + pub(super) fn matches(&self, current: &Self) -> bool { self.focus_epoch == current.focus_epoch - && self.editor.conversation_id == current.editor.conversation_id - && self.editor.mount_id == current.editor.mount_id - && self.editor.editor_id == current.editor.editor_id - && (mode == Mode::Submit - || (self.editor.editable - && current.editor.editable - && self.editor.revision == current.editor.revision - && self.editor.selection_start == current.editor.selection_start - && self.editor.selection_end == current.editor.selection_end)) + && self.editor.editable + && self.editor == current.editor } } -#[derive(Clone, Serialize)] -#[serde(rename_all = "camelCase")] +#[derive(Clone)] pub(crate) struct Delivery { pub(crate) recording_id: SessionId, pub(crate) target: Target, - pub(crate) mode: Mode, pub(crate) text: String, } @@ -82,10 +57,8 @@ pub(super) struct Controller { enum Message { Intent { - mode: Mode, + origin: Origin, target: Option, - press: u64, - toggle: bool, }, Release { press: u64, @@ -102,11 +75,33 @@ enum Message { Finished(SessionId, Result), } +#[derive(Clone, Copy)] +enum Origin { + Chord { request: u64 }, + Manual { press: u64, toggle: bool }, +} + +impl Origin { + fn may_stop(self, active: Self) -> bool { + let Self::Manual { + press, + toggle: true, + } = self + else { + return false; + }; + !matches!(active, Self::Manual { press: active, .. } if press == active) + } + + fn is_press(self, candidate: u64) -> bool { + matches!(self, Self::Manual { press, .. } if press == candidate) + } +} + struct Active { reservation: Arc, target: Target, - mode: Mode, - press: u64, + origin: Origin, capture: Option>, finishing: bool, } @@ -132,12 +127,17 @@ impl Controller { self.owner.lock().as_ref().map(|owner| owner.id()) } - pub(super) fn intent(&self, mode: Mode, target: Option, press: u64, toggle: bool) { + pub(super) fn manual(&self, target: Option, press: u64, toggle: bool) { let _ = self.tx.send(Message::Intent { - mode, target, - press, - toggle, + origin: Origin::Manual { press, toggle }, + }); + } + + pub(super) fn autostart(&self, request: u64, target: Target) { + let _ = self.tx.send(Message::Intent { + target: Some(target), + origin: Origin::Chord { request }, }); } @@ -192,45 +192,32 @@ async fn listen(desktop: Weak, mut inbox: mpsc::UnboundedReceiver { - let Some(target) = target.filter(|target| desktop.target_is_current(target, mode)) - else { + Message::Intent { origin, target } => { + let chord = match origin { + Origin::Chord { request } if desktop.autostart_is_pending(request) => { + Some(Autostart { + desktop: &desktop, + request, + }) + } + Origin::Chord { .. } => continue, + Origin::Manual { .. } => None, + }; + let Some(target) = target.filter(|target| desktop.target_is_current(target)) else { continue; }; if let Some(recording) = active.as_mut() { - if mode == Mode::Submit - && recording.mode == Mode::Insert - && recording.press == press - && !recording.finishing - && desktop.target_is_current(&recording.target, Mode::Submit) - { - recording.mode = Mode::Submit; - desktop.notify_voice( - "started", - recording.reservation.id(), - &recording.target, - recording.mode, - None, - ); - continue; - } - if toggle - && recording.mode == mode - && recording.press != press - && desktop.target_is_current(&recording.target, mode) + if origin.may_stop(recording.origin) + && desktop.target_is_current(&recording.target) { recording.stop(&desktop); } continue; } if desktop.current_voice_delivery().is_some() - || last_start - .is_some_and(|at: Instant| at.elapsed() < Duration::from_millis(250)) + || (chord.is_none() + && last_start + .is_some_and(|at: Instant| at.elapsed() < Duration::from_millis(250))) { continue; } @@ -246,14 +233,18 @@ async fn listen(desktop: Weak, mut inbox: mpsc::UnboundedReceiver, mut inbox: mpsc::UnboundedReceiver, mut inbox: mpsc::UnboundedReceiver { recording.capture = Some(Arc::new(capture)); if recording.reservation.is_cancelled() - || !desktop.target_is_current(&recording.target, recording.mode) + || !desktop.target_is_current(&recording.target) { cancel(&desktop, &mut active, id, CapsuleState::Cancelled); } else if recording.finishing { @@ -320,8 +310,7 @@ async fn listen(desktop: Weak, mut inbox: mpsc::UnboundedReceiver { if let Some(recording) = active.as_mut().filter(|recording| { - recording.mode == Mode::Insert - && recording.press == press + recording.origin.is_press(press) && (hold || (auto && held >= Duration::from_millis(350))) }) { recording.stop(&desktop); @@ -341,7 +330,7 @@ async fn listen(desktop: Weak, mut inbox: mpsc::UnboundedReceiver { if let Some(id) = active .as_ref() - .filter(|recording| recording.press == press) + .filter(|recording| recording.origin.is_press(press)) .map(|recording| recording.reservation.id()) { cancel(&desktop, &mut active, id, CapsuleState::Cancelled); @@ -350,9 +339,7 @@ async fn listen(desktop: Weak, mut inbox: mpsc::UnboundedReceiver { if let Some(id) = active .as_ref() - .filter(|recording| { - !desktop.target_is_current(&recording.target, recording.mode) - }) + .filter(|recording| !desktop.target_is_current(&recording.target)) .map(|recording| recording.reservation.id()) { cancel(&desktop, &mut active, id, CapsuleState::Cancelled); @@ -366,7 +353,7 @@ async fn listen(desktop: Weak, mut inbox: mpsc::UnboundedReceiver, mut inbox: mpsc::UnboundedReceiver { + desktop: &'a Desktop, + request: u64, +} + +impl Drop for Autostart<'_> { + fn drop(&mut self) { + self.desktop.complete_autostart(self.request); + } +} + impl Active { fn stop(&mut self, desktop: &Desktop) { if self.finishing { @@ -440,7 +438,7 @@ fn cancel(desktop: &Desktop, active: &mut Option, id: SessionId, state: return; } if let Some(recording) = active.as_ref() { - desktop.notify_voice("cancelled", id, &recording.target, recording.mode, None); + desktop.notify_voice("cancelled", id, &recording.target, None); } frame(desktop, id, state, 0.0, 0, false); if let Some(capture) = take_owner(desktop, active, id).and_then(|recording| recording.capture) { @@ -460,26 +458,19 @@ fn complete( else { return; }; - let current = desktop.target_is_current(&recording.target, recording.mode) - && !recording.reservation.is_cancelled(); + let current = + desktop.target_is_current(&recording.target) && !recording.reservation.is_cancelled(); let state = match result { Ok(text) if current && !text.trim().is_empty() => { let mut delivery = desktop.voice_result.lock(); *delivery = Some(Delivery { recording_id: id, target: recording.target.clone(), - mode: recording.mode, text, }); drop(delivery); if let Some(delivery) = desktop.current_voice_delivery() { - desktop.notify_voice( - "transcribed", - id, - &delivery.target, - delivery.mode, - Some(&delivery.text), - ); + desktop.notify_voice("transcribed", id, &delivery.target, Some(&delivery.text)); } else { desktop.voice_result.lock().take(); } @@ -492,7 +483,7 @@ fn complete( Err(_) => CapsuleState::Error, }; if state != CapsuleState::Done { - desktop.notify_voice("cancelled", id, &recording.target, recording.mode, None); + desktop.notify_voice("cancelled", id, &recording.target, None); } frame(desktop, id, state, 0.0, 0, false); take_owner(desktop, active, id); @@ -537,16 +528,12 @@ mod tests { editable: true, selection_start: 2, selection_end: 3, - attachment_ids: vec![], - options: Options { - deep_thinking: false, - }, }, } } #[test] - fn both_delivery_modes_reject_a_different_focus_generation_or_conversation() { + fn delivery_rejects_a_different_focus_generation_or_conversation() { let original = composer(); let mut refocused = composer(); refocused.focus_epoch += 1; @@ -554,22 +541,34 @@ mod tests { other_conversation.editor.conversation_id = "another".into(); let mut replaced_view = composer(); replaced_view.editor.mount_id = "new-mount".into(); - for mode in [Mode::Insert, Mode::Submit] { - assert!(original.matches(&original, mode)); - for new_target in [&refocused, &other_conversation, &replaced_view] { - assert!(!original.matches(new_target, mode)); - } + assert!(original.matches(&original)); + for new_target in [&refocused, &other_conversation, &replaced_view] { + assert!(!original.matches(new_target)); } } #[test] - fn changed_draft_or_caret_blocks_insert_but_not_the_frozen_submit() { + fn changed_draft_or_caret_blocks_insert() { let original = composer(); let mut changed = composer(); changed.editor.selection_end += 1; - assert!(!original.matches(&changed, Mode::Insert)); + assert!(!original.matches(&changed)); + changed = composer(); changed.editor.revision += 1; - assert!(original.matches(&changed, Mode::Submit)); - assert!(!original.matches(&changed, Mode::Insert)); + assert!(!original.matches(&changed)); + } + + #[test] + fn an_autostart_can_neither_toggle_nor_accept_a_physical_release() { + let chord = Origin::Chord { request: 7 }; + let manual = Origin::Manual { + press: 7, + toggle: true, + }; + assert!(!chord.is_press(7)); + assert!(!chord.may_stop(chord)); + assert!(!chord.may_stop(manual)); + assert!(manual.may_stop(chord)); + assert!(!manual.may_stop(manual)); } } diff --git a/openless-all/app/src-tauri/src/ask_window/wire.rs b/openless-all/app/src-tauri/src/ask_window/wire.rs index 1e240745c..143e73059 100644 --- a/openless-all/app/src-tauri/src/ask_window/wire.rs +++ b/openless-all/app/src-tauri/src/ask_window/wire.rs @@ -1,7 +1,8 @@ //! Translate canonical Core events without dropping sequence numbers or reading ahead. use openless_core::ask::{ - AskActivity, AskEvent, AskEventKind, AskMessage, AskSnapshot, AskTurn, AskTurnStatus, + AskActivity, AskEvent, AskEventKind, AskMessage, AskSnapshot, AskSubmission, AskTurn, + AskTurnStatus, }; use openless_core::BackendError; use serde_json::{json, Value}; @@ -15,9 +16,29 @@ pub(super) fn error(error: BackendError) -> Value { "askImageInputUnsupported" => "The configured model does not support images in Ask.", "askRequestConflict" => "This request ID belongs to a different message.", "askConversationUnavailable" => "This conversation is no longer available.", + "askIntentInvalid" => { + "Could not determine what to do with this request. Try rephrasing it." + } + "askIntentToolUnsupported" => { + "The configured model does not support the selected intent tool." + } + "askIntentTimeout" => "Intent detection took too long. Please try again.", + "askEntrySettingsUnavailable" => { + "Could not read Ask settings. Check Settings and try again." + } _ => &error.message, }; - json!({"code": error.code, "message": message}) + let mut value = json!({"code": error.code, "message": message}); + if matches!( + error.message.as_str(), + "askIntentInvalid" + | "askIntentToolUnsupported" + | "askIntentTimeout" + | "askEntrySettingsUnavailable" + ) { + value["reason"] = json!(error.message); + } + value } fn working(status: Option) -> &'static str { @@ -57,6 +78,23 @@ fn turn(turn: &AskTurn, user: &AskMessage, assistant: &AskMessage) -> Value { }) } +fn submission(value: &AskSubmission) -> Value { + json!({ + "currentTurnId": value.current_turn_id, + "route": value.route, + "status": status(value.status, value.activity), + "activity": value.activity, + "searchStatus": value.search_status, + "searchError": value.search_error.clone().map(error), + "toolCallId": value.tool_call_id, + "sources": value.sources, + "resultText": value.result_text, + "delivery": value.delivery, + "copyStatus": value.copy_status, + "error": value.error.clone().map(error), + }) +} + pub(super) fn snapshot(snapshot: AskSnapshot) -> Value { let messages: std::collections::HashMap<_, _> = snapshot .messages @@ -74,11 +112,26 @@ pub(super) fn snapshot(snapshot: AskSnapshot) -> Value { )) }) .collect::>(); - json!({"conversationId":snapshot.conversation_id,"sequence":snapshot.sequence,"activeTurnId":snapshot.current_turn_id,"turns":turns,"attachments":snapshot.attachments,"cache":snapshot.cache}) + json!({ + "conversationId": snapshot.conversation_id, + "sequence": snapshot.sequence, + "activeTurnId": snapshot.current_turn_id, + "turns": turns, + "attachments": snapshot.attachments, + "cache": snapshot.cache, + "submission": snapshot.submission.as_ref().map(submission), + }) } pub(super) fn event(event: &AskEvent) -> Value { let mut packet = match &event.kind { + AskEventKind::Submitted { submission: value } => { + json!({"type":"submitted","submission":submission(value)}) + } + AskEventKind::RouteResolved { route } => json!({"type":"routeResolved","route":route}), + AskEventKind::Delivery { submission: value } => { + json!({"type":"delivery","submission":submission(value)}) + } AskEventKind::Accepted { user, assistant, @@ -157,4 +210,14 @@ mod tests { assert_eq!(packet["source"]["readStatus"], "blocked"); assert!(packet.get("evidence").is_none()); } + + #[test] + fn intent_error_can_be_localized_without_using_its_display_message() { + let result = error(BackendError::new( + openless_core::BackendErrorCode::Unsupported, + "askIntentToolUnsupported", + )); + assert_eq!(result["reason"], "askIntentToolUnsupported"); + assert_ne!(result["message"], result["reason"]); + } } diff --git a/openless-all/app/src-tauri/src/host_document/macos.rs b/openless-all/app/src-tauri/src/host_document/macos.rs index 8a2e2ad85..936223e4f 100644 --- a/openless-all/app/src-tauri/src/host_document/macos.rs +++ b/openless-all/app/src-tauri/src/host_document/macos.rs @@ -94,8 +94,8 @@ const EDIT_SETTLE_TIMEOUT: Duration = Duration::from_secs(5); const BASELINE_ANCHOR_TIMEOUT: Duration = Duration::from_millis(1500); #[repr(C)] -struct OpaqueAxRef(c_void); -type AxUiElementRef = *mut OpaqueAxRef; +pub(crate) struct OpaqueAxRef(c_void); +pub(crate) type AxUiElementRef = *mut OpaqueAxRef; type CFStringRef = *const c_void; type CFTypeRef = *const c_void; type CFAllocatorRef = *const c_void; @@ -119,8 +119,8 @@ const K_CF_NUMBER_CF_INDEX_TYPE: i32 = 14; /// Opaque handle for AXObserver. #[repr(C)] -struct OpaqueAxObserver(c_void); -type AxObserverRef = *mut OpaqueAxObserver; +pub(crate) struct OpaqueAxObserver(c_void); +pub(crate) type AxObserverRef = *mut OpaqueAxObserver; type AxObserverCallback = unsafe extern "C" fn( observer: AxObserverRef, @@ -131,14 +131,14 @@ type AxObserverCallback = unsafe extern "C" fn( #[link(name = "ApplicationServices", kind = "framework")] extern "C" { - fn AXUIElementCreateApplication(pid: i32) -> AxUiElementRef; - fn AXUIElementGetPid(element: AxUiElementRef, pid: *mut i32) -> AxError; - fn AXObserverCreate( + pub(crate) fn AXUIElementCreateApplication(pid: i32) -> AxUiElementRef; + pub(crate) fn AXUIElementGetPid(element: AxUiElementRef, pid: *mut i32) -> AxError; + pub(crate) fn AXObserverCreate( application: i32, callback: AxObserverCallback, observer: *mut AxObserverRef, ) -> AxError; - fn AXObserverAddNotification( + pub(crate) fn AXObserverAddNotification( observer: AxObserverRef, element: AxUiElementRef, notification: CFStringRef, @@ -149,26 +149,26 @@ extern "C" { element: AxUiElementRef, notification: CFStringRef, ) -> AxError; - fn AXObserverGetRunLoopSource(observer: AxObserverRef) -> CFRunLoopSourceRef; - fn AXUIElementSetMessagingTimeout(element: AxUiElementRef, timeout: f32) -> AxError; - fn AXUIElementCopyAttributeValue( + pub(crate) fn AXObserverGetRunLoopSource(observer: AxObserverRef) -> CFRunLoopSourceRef; + pub(crate) fn AXUIElementSetMessagingTimeout(element: AxUiElementRef, timeout: f32) -> AxError; + pub(crate) fn AXUIElementCopyAttributeValue( element: AxUiElementRef, attribute: CFStringRef, value: *mut CFTypeRef, ) -> AxError; - fn AXUIElementSetAttributeValue( + pub(crate) fn AXUIElementSetAttributeValue( element: AxUiElementRef, attribute: CFStringRef, value: CFTypeRef, ) -> AxError; - fn AXUIElementCopyParameterizedAttributeValue( + pub(crate) fn AXUIElementCopyParameterizedAttributeValue( element: AxUiElementRef, parameterized_attribute: CFStringRef, parameter: CFTypeRef, value: *mut CFTypeRef, ) -> AxError; - fn AXValueGetValue(value: AxValueRef, value_type: i32, out: *mut c_void) -> u8; - fn AXValueCreate(value_type: i32, value_ptr: *const c_void) -> AxValueRef; + pub(crate) fn AXValueGetValue(value: AxValueRef, value_type: i32, out: *mut c_void) -> u8; + pub(crate) fn AXValueCreate(value_type: i32, value_ptr: *const c_void) -> AxValueRef; } #[link(name = "CoreFoundation", kind = "framework")] diff --git a/openless-all/app/src-tauri/src/host_document/mod.rs b/openless-all/app/src-tauri/src/host_document/mod.rs index 4dbfc35e2..ac1662ee8 100644 --- a/openless-all/app/src-tauri/src/host_document/mod.rs +++ b/openless-all/app/src-tauri/src/host_document/mod.rs @@ -32,7 +32,7 @@ //! host document body. #[cfg(target_os = "macos")] -mod macos; +pub(crate) mod macos; #[cfg(target_os = "windows")] mod uia_text; #[cfg(target_os = "windows")] @@ -804,11 +804,9 @@ mod tests { } #[cfg(target_os = "windows")] -pub(crate) fn read_qa_selection() -> Option { - windows::read_qa_selection() -} +pub(crate) use windows::{is_blocked_process_name, runtime_id as uia_runtime_id}; #[cfg(target_os = "windows")] -pub(crate) fn read_native_ask_selection_for_pid(pid: u32) -> Option { - windows::read_ask_selection(pid) +pub(crate) fn read_qa_selection() -> Option { + windows::read_qa_selection() } diff --git a/openless-all/app/src-tauri/src/host_document/windows.rs b/openless-all/app/src-tauri/src/host_document/windows.rs index 1923b86df..f0151fd07 100644 --- a/openless-all/app/src-tauri/src/host_document/windows.rs +++ b/openless-all/app/src-tauri/src/host_document/windows.rs @@ -75,26 +75,10 @@ fn with_voice_edit_uia( } pub(super) fn read_qa_selection() -> Option { - read_explicit_selection(None) -} - -pub(super) fn read_ask_selection(pid: u32) -> Option { - read_explicit_selection(Some(pid)) -} - -fn read_explicit_selection(expected_pid: Option) -> Option { with_voice_edit_uia(|uia| unsafe { - let unavailable = || "selectionUnavailable".to_string(); let element = uia .GetFocusedElement() .map_err(|_| "selectionUnavailable".to_string())?; - if let Some(pid) = expected_pid { - if element.CurrentProcessId().map_err(|_| unavailable())? as u32 != pid - || !allowed_process(&element).map_err(|_| unavailable())? - { - return Ok(None); - } - } if element .CurrentIsPassword() .map_err(|_| "selectionUnavailable".to_string())? @@ -147,7 +131,7 @@ fn read_explicit_selection(expected_pid: Option) -> Option { .flatten() } -unsafe fn runtime_id(element: &IUIAutomationElement) -> Result> { +pub(crate) unsafe fn runtime_id(element: &IUIAutomationElement) -> Result> { use windows::Win32::System::Ole::{ SafeArrayDestroy, SafeArrayGetElement, SafeArrayGetLBound, SafeArrayGetUBound, }; @@ -556,7 +540,7 @@ const BLOCKED_PROCESS_NAME_FRAGMENTS: &[&str] = &[ "putty", ]; -fn is_blocked_process_name(executable_name: &str) -> bool { +pub(crate) fn is_blocked_process_name(executable_name: &str) -> bool { let lowered = executable_name.to_lowercase(); BLOCKED_PROCESS_NAME_FRAGMENTS .iter() diff --git a/openless-all/app/src-tauri/src/hotkey.rs b/openless-all/app/src-tauri/src/hotkey.rs index 55b71ccaf..62ed59dfc 100644 --- a/openless-all/app/src-tauri/src/hotkey.rs +++ b/openless-all/app/src-tauri/src/hotkey.rs @@ -703,9 +703,6 @@ mod platform { let mask: CgEventMask = (1u64 << FLAGS_CHANGED) | (1u64 << KEY_DOWN) | (1u64 << KEY_UP) - | (1u64 << 1) - | (1u64 << 3) - | (1u64 << 25) | (1u64 << SYSTEM_DEFINED); let handles = Arc::new(MacShutdownHandles { tap: std::sync::Mutex::new(None), @@ -770,7 +767,6 @@ mod platform { let ctx = unsafe { &*(user_info as *const CallbackContext) }; match event_type { - 1 | 3 | 25 => crate::ask_window::capture::mark_interaction(), TAP_DISABLED_BY_TIMEOUT | TAP_DISABLED_BY_USER_INPUT => { crate::ask_window::hotkey::reset_physical_state(); if let Some(tap) = *ctx.handles.tap.lock().unwrap() { diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index bf380a161..ac8dffe8f 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -379,6 +379,8 @@ macro_rules! app_invoke_handler_desktop { ask_window::chat::ask_ack_voice, ask_window::chat::ask_get_context_permission, ask_window::chat::ask_set_context_permission, + ask_window::chat::ask_get_entry_settings, + ask_window::chat::ask_set_entry_settings, commands::voice_edit_window_close, commands::set_capsule_transcript_visible, #[cfg(target_os = "macos")] @@ -875,7 +877,7 @@ fn run_desktop() { .manage(TrayMicrophoneDeviceCache(parking_lot::Mutex::new(Vec::new()))) .setup(move |app| { log::info!("=== OpenLess 启动 ==="); - let ask = Arc::new(ask_window::chat::AskHost::new(core_backend.clone())); + let ask = Arc::new(ask_window::chat::AskHost::new(app.handle(), core_backend.clone())); let desktop = ask_window::Desktop::install( app.handle().clone(), core_backend.clone(), @@ -1084,9 +1086,10 @@ fn run_desktop() { .invoke_handler({ let dispatch: fn(tauri::ipc::Invoke) -> bool = app_invoke_handler_desktop!(); move |invoke: tauri::ipc::Invoke| { - if invoke.message.webview().label() == ask_window::LABEL - && !invoke.message.command().starts_with("ask_") - { + if !ask_window::chat::restricted_command_allowed( + invoke.message.webview().label(), + invoke.message.command(), + ) { invoke.resolver.reject("This command is not available in Ask."); return true; } @@ -1131,12 +1134,7 @@ fn run_desktop() { } } RunEvent::Exit => { - if let Some(desktop) = app.try_state::>() { - desktop.shutdown(); - } - if let Some(ask) = app.try_state::>() { - ask.stop(); - } + stop_ask_desktop(app); TRAY_MICROPHONE_WATCHER_STOPPING.store(true, Ordering::Relaxed); crate::recorder::warm_capture_hub().stop(); let coordinator = app.state::>(); @@ -1850,6 +1848,8 @@ fn restart_app(app: AppHandle) { if let Some(coordinator) = app.try_state::>() { coordinator.tauri_host().stop_native_capsule(); } + #[cfg(not(mobile))] + stop_ask_desktop(&app); prepare_for_restart(); #[cfg(target_os = "macos")] reset_tcc_for_beta_restart(); @@ -1884,12 +1884,24 @@ fn request_quit(app: &tauri::AppHandle) { #[tauri::command] fn reset_accessibility_permission_and_restart_app(app: AppHandle) { + #[cfg(not(mobile))] + stop_ask_desktop(&app); prepare_for_restart(); #[cfg(target_os = "macos")] reset_tcc_service_for_restart("Accessibility", "accessibility recovery"); app.restart(); } +#[cfg(not(mobile))] +fn stop_ask_desktop(app: &AppHandle) { + if let Some(desktop) = app.try_state::>() { + desktop.shutdown(); + } + if let Some(ask) = app.try_state::>() { + ask.stop(); + } +} + fn prepare_for_restart() { // macOS: auto-updates leave the newly installed .app with com.apple.quarantine (no // matter how the Tauri updater unpacks, the download stream goes through LaunchServices diff --git a/openless-all/app/src-tauri/src/selection.rs b/openless-all/app/src-tauri/src/selection.rs index 85f6f35cd..c4e8e6bd8 100644 --- a/openless-all/app/src-tauri/src/selection.rs +++ b/openless-all/app/src-tauri/src/selection.rs @@ -650,11 +650,6 @@ pub(crate) fn capture_explicit_qa_selection() -> (Option, Sele (selection, target) } -#[cfg(target_os = "macos")] -pub(crate) fn read_native_ask_selection_for_pid(pid: u32) -> Result, ()> { - macos_ax::read_explicit_text(Some(pid as i32)) -} - /// 捕获当前前台应用的选区;失败统一视为无选区。 pub fn capture_selection_with_status() -> SelectionCaptureOutcome { capture_selection_with_status_diag().0 diff --git a/openless-all/app/src-tauri/src/tauri_events.rs b/openless-all/app/src-tauri/src/tauri_events.rs index 0be983357..9da304399 100644 --- a/openless-all/app/src-tauri/src/tauri_events.rs +++ b/openless-all/app/src-tauri/src/tauri_events.rs @@ -603,11 +603,13 @@ fn emit_preferences(app: &AppHandle, backend: &OpenLessBackend) { } let _ = app.emit("prefs:changed", &preferences); #[cfg(not(mobile))] - let _ = app.emit_to( - tauri::EventTarget::webview_window(crate::ask_window::LABEL), - "ask:window", - serde_json::json!({"type": "themeChanged", "themeMode": preferences.theme_mode}), - ); + for label in [crate::ask_window::LABEL, crate::ask_window::CONVERSATION_LABEL] { + let _ = app.emit_to( + tauri::EventTarget::webview_window(label), + "ask:window", + serde_json::json!({"type": "themeChanged", "themeMode": preferences.theme_mode}), + ); + } } fn transcription_notice_payload( diff --git a/openless-all/app/src/features/settings/sections/AskEntrySettingsSection.tsx b/openless-all/app/src/features/settings/sections/AskEntrySettingsSection.tsx new file mode 100644 index 000000000..df488fed0 --- /dev/null +++ b/openless-all/app/src/features/settings/sections/AskEntrySettingsSection.tsx @@ -0,0 +1,141 @@ +import { useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { PageSkeleton } from '../../../components/ui/PageSkeleton'; +import { Button } from '../../../ui/controls/Button'; +import { SettingRow, Toggle } from '../../../ui/controls/SettingRow'; +import { Card } from '../../../ui/layout/Card'; +import { SectionTitle } from '../../../ui/layout/SectionTitle'; +import { useInteraction } from '../../../ui/overlays/InteractionProvider'; +import { + readAskEntrySettings, + writeAskEntrySettings, + type AskEntrySettings, + type AskEntryIntentMode, +} from './askEntrySettings'; + +const betaModes = [ + 'separateModel', + 'mainModelTool', +] as const satisfies readonly AskEntryIntentMode[]; + +const recoveryNoticeKey = 'ask-entry-settings-recovery'; + +export function AskEntrySettingsSection() { + const { t } = useTranslation(); + const { notify } = useInteraction(); + const [settings, setSettings] = useState(); + const [saving, setSaving] = useState(false); + const writePending = useRef(false); + + useEffect(() => { + let active = true; + void readAskEntrySettings().then( + (committed) => { + if (active) setSettings(committed); + }, + () => { + if (!active) return; + setSettings((current) => current ?? 'readFailed'); + notify({ + key: recoveryNoticeKey, + state: 'failed', + message: t('askEntrySettings.loadFailed'), + }); + }, + ); + return () => { + active = false; + }; + }, [t, notify]); + + function commit(next: AskEntrySettings) { + if (writePending.current) return; + const restoring = settings === 'readFailed'; + writePending.current = true; + setSaving(true); + void writeAskEntrySettings(next) + .then( + (committed) => { + setSettings(committed); + if (restoring) { + notify({ key: recoveryNoticeKey, state: 'saved', message: t('common.saved') }); + } + }, + () => + notify({ + key: restoring ? recoveryNoticeKey : undefined, + state: 'failed', + message: t('askEntrySettings.saveFailed'), + }), + ) + .finally(() => { + writePending.current = false; + setSaving(false); + }); + } + + if (!settings) return ; + if (settings === 'readFailed') { + return ( + + + + + + ); + } + return ( + <> + + + commit({ ...settings, autoVoiceOnOpen })} + /> + + + + + {t('askEntrySettings.routing')} + + + {t(`askEntrySettings.${settings.intentMode}`)} + + {betaModes.map((mode) => { + const label = t(`askEntrySettings.${mode}`); + const enabled = settings.intentMode === mode; + return ( + + { + if (next !== enabled) { + commit({ ...settings, intentMode: next ? mode : 'semanticHybrid' }); + } + }} + /> + + + ); + })} + + + ); +} diff --git a/openless-all/app/src/features/settings/sections/AskHistorySection.tsx b/openless-all/app/src/features/settings/sections/AskHistorySection.tsx index 913509b8c..a150fad59 100644 --- a/openless-all/app/src/features/settings/sections/AskHistorySection.tsx +++ b/openless-all/app/src/features/settings/sections/AskHistorySection.tsx @@ -6,22 +6,30 @@ import { isDesktop } from '../../../lib/platform'; import { invokeOrPreview } from '../../../runtime/transport'; import { SettingRow, Toggle } from '../../../ui/controls/SettingRow'; import { Card } from '../../../ui/layout/Card'; +import { useInteraction } from '../../../ui/overlays/InteractionProvider'; -export function AskHistorySection() { +export function AskHistorySection({ compact = false }: { compact?: boolean }) { const { t } = useTranslation(); + const { notify } = useInteraction(); const { prefs, updatePrefs } = useHotkeySettings(); - const [saveFailed, setSaveFailed] = useState(false); const [contextAllowed, setContextAllowed] = useState(); const [contextSaving, setContextSaving] = useState(false); - const [contextError, setContextError] = useState(false); useEffect(() => { if (!isDesktop()) return; let cancelled = false; void invokeOrPreview('ask_get_context_permission', undefined, () => false) - .then((allowed) => { if (!cancelled) setContextAllowed(allowed); }) - .catch(() => { if (!cancelled) setContextError(true); }); - return () => { cancelled = true; }; - }, []); + .then((allowed) => { + if (!cancelled) setContextAllowed(allowed); + }) + .catch(() => { + if (!cancelled) { + notify({ state: 'failed', message: t('selectionAsk.desktopContext.saveFailed') }); + } + }); + return () => { + cancelled = true; + }; + }, [t, notify]); if (!prefs) return ; return ( @@ -29,19 +37,17 @@ export function AskHistorySection() { { - setSaveFailed(false); void updatePrefs((current) => ({ ...current, qaSaveHistory })).catch(() => - setSaveFailed(true), + notify({ state: 'failed', message: t('selectionAsk.save.historySaveFailed') }), ); }} /> - {saveFailed &&

{t('selectionAsk.save.historySaveFailed')}

} {contextAllowed !== undefined && ( { setContextSaving(true); - setContextError(false); void invokeOrPreview('ask_set_context_permission', { allowed }, () => undefined) .then(() => setContextAllowed(allowed)) - .catch(() => setContextError(true)) + .catch(() => + notify({ state: 'failed', message: t('selectionAsk.desktopContext.saveFailed') }), + ) .finally(() => setContextSaving(false)); }} /> )} - {contextError && ( -

{t('selectionAsk.desktopContext.saveFailed', 'Could not update Ask’s desktop context permission.')}

- )} ); } diff --git a/openless-all/app/src/features/settings/sections/askEntrySettings.ts b/openless-all/app/src/features/settings/sections/askEntrySettings.ts new file mode 100644 index 000000000..553c46771 --- /dev/null +++ b/openless-all/app/src/features/settings/sections/askEntrySettings.ts @@ -0,0 +1,31 @@ +import { invokeOrPreview } from '../../../runtime/transport'; + +export type AskEntryIntentMode = 'semanticHybrid' | 'separateModel' | 'mainModelTool'; + +export interface AskEntrySettings { + autoVoiceOnOpen: boolean; + intentMode: AskEntryIntentMode; +} + +function nativeOnly(): never { + throw new Error('Ask entry settings require the native main window.'); +} + +let pendingOperation: Promise = Promise.resolve(); + +function serialize(operation: () => Promise): Promise { + const result = pendingOperation.then(operation); + pendingOperation = result.then( + () => undefined, + () => undefined, + ); + return result; +} + +export function readAskEntrySettings(): Promise { + return serialize(() => invokeOrPreview('ask_get_entry_settings', undefined, nativeOnly)); +} + +export function writeAskEntrySettings(settings: AskEntrySettings): Promise { + return serialize(() => invokeOrPreview('ask_set_entry_settings', { settings }, nativeOnly)); +} diff --git a/openless-all/app/src/features/settings/shell/navigation.test.ts b/openless-all/app/src/features/settings/shell/navigation.test.ts index a14b8007e..0b8241d27 100644 --- a/openless-all/app/src/features/settings/shell/navigation.test.ts +++ b/openless-all/app/src/features/settings/shell/navigation.test.ts @@ -8,6 +8,7 @@ import { isServiceViewInactive, resolveServiceView, visibleAdvancedPages, + visibleGeneralPages, } from './navigation'; const assert = { @@ -19,6 +20,29 @@ const assert = { }, }; +for (const os of ['mac', 'win'] as const) { + const nativePages = visibleGeneralPages('desktop', os, true).map((page) => page.id); + assert.equal(nativePages.includes('askEntry'), true, `${os}: desktop has Ask Anything settings`); + assert.equal(nativePages.includes('askHistory'), false, `${os}: desktop has one Ask subpage`); +} +for (const pages of [ + visibleGeneralPages('desktop', 'mac'), + visibleGeneralPages('android', 'android', true), + visibleGeneralPages('mobile', 'mac', true), + visibleGeneralPages('harmony', 'harmony'), +]) { + assert.equal( + pages.some((page) => page.id === 'askEntry'), + false, + 'no unsupported Ask settings', + ); + assert.equal( + pages.some((page) => page.id === 'askHistory'), + true, + 'keeps legacy history settings', + ); +} + const sections = [ { id: 'general' as const, diff --git a/openless-all/app/src/features/settings/shell/navigation.ts b/openless-all/app/src/features/settings/shell/navigation.ts index b0e2301a0..3e5075c76 100644 --- a/openless-all/app/src/features/settings/shell/navigation.ts +++ b/openless-all/app/src/features/settings/shell/navigation.ts @@ -52,6 +52,12 @@ export const GENERAL_PAGES = [ titleKey: 'settings.generalPages.recordingTitle', descriptionKey: 'settings.generalPages.recordingDesc', }, + { + id: 'askEntry', + icon: 'sparkle', + titleKey: 'askEntrySettings.title', + descriptionKey: 'askEntrySettings.description', + }, { id: 'askHistory', icon: 'history', @@ -81,8 +87,15 @@ export const GENERAL_PAGES = [ export type GeneralPage = (typeof GENERAL_PAGES)[number]; export type GeneralPageId = GeneralPage['id']; -export function visibleGeneralPages(platform: PlatformKind | undefined): GeneralPage[] { +export function visibleGeneralPages( + platform: PlatformKind | undefined, + os?: OS, + native = false, +): GeneralPage[] { + const unifiedAsk = native && platform === 'desktop' && (os === 'mac' || os === 'win'); return GENERAL_PAGES.filter((page) => { + if (page.id === 'askEntry') return unifiedAsk; + if (page.id === 'askHistory') return !unifiedAsk; if (page.id === 'phone') return platform === 'desktop'; if (page.id === 'capsule') return platform == null || platform === 'desktop'; return true; diff --git a/openless-all/app/src/features/settings/shell/navigationLocale.test.ts b/openless-all/app/src/features/settings/shell/navigationLocale.test.ts index 2605ae847..3fd379dd4 100644 --- a/openless-all/app/src/features/settings/shell/navigationLocale.test.ts +++ b/openless-all/app/src/features/settings/shell/navigationLocale.test.ts @@ -7,7 +7,13 @@ import { ko } from '../../../i18n/ko'; import { de } from '../../../i18n/de'; import { fr } from '../../../i18n/fr'; import { es } from '../../../i18n/es'; -import { ADVANCED_PAGES, SETTINGS_SECTIONS, searchSettingsSections } from './navigation'; +import { askEntrySettingsEn } from '../../../i18n/askEntrySettings'; +import { + ADVANCED_PAGES, + GENERAL_PAGES, + SETTINGS_SECTIONS, + searchSettingsSections, +} from './navigation'; const retiredServiceNames = [ 'AI 服务与模型', @@ -35,6 +41,12 @@ for (const [locale, translation] of Object.entries({ })) { const i18n = createInstance(); await i18n.init({ lng: locale, fallbackLng: false, resources: { [locale]: { translation } } }); + for (const key of [ + ...GENERAL_PAGES.flatMap((page) => [page.titleKey, page.descriptionKey]), + ...Object.keys(askEntrySettingsEn).map((key) => `askEntrySettings.${key}`), + ]) { + if (!i18n.exists(key)) throw new Error(`${locale}: missing general settings label ${key}`); + } for (const key of [ 'modal.backToAdvanced', ...ADVANCED_PAGES.flatMap((page) => [page.titleKey, `modal.advancedPages.${page.id}`]), diff --git a/openless-all/app/src/features/settings/shell/tabs.tsx b/openless-all/app/src/features/settings/shell/tabs.tsx index 47acbec26..72f537e70 100644 --- a/openless-all/app/src/features/settings/shell/tabs.tsx +++ b/openless-all/app/src/features/settings/shell/tabs.tsx @@ -6,7 +6,7 @@ import { createContext, useContext, useEffect, useRef, useState } from 'react'; import { AndroidPermissionsPanel } from '@android/components/AndroidPermissionsPanel'; import { Icon } from '../../../components/Icon'; import { isRetiredMacAsrProvider } from '../../../lib/experimentalAsr'; -import { getRuntimeOS } from '../../../runtime/environment'; +import { getRuntimeOS, isTauriHost } from '../../../runtime/environment'; import { ExperimentalAsrSection, type ExperimentalAsrController, @@ -39,6 +39,7 @@ import { BetaChannelSection } from '../sections/BetaChannelSection'; import { AutoUpdateSection } from '../sections/AutoUpdateSection'; import { StartupDiagnostics } from '../sections/StartupDiagnostics'; import { AboutSection } from '../sections/AboutSection'; +import { AskEntrySettingsSection } from '../sections/AskEntrySettingsSection'; import { AskHistorySection } from '../sections/AskHistorySection'; import { getPlatformCapabilities, getCachedPlatformCapabilities } from '../../../runtime/platform'; import { listChannels, listenCredentialsChanges } from '../../../api/providers'; @@ -91,7 +92,7 @@ export function GeneralTab({ page: GeneralPageId | 'root'; onPageChange: (page: GeneralPageId | 'root') => void; }) { - const categories = visibleGeneralPages(platform); + const categories = visibleGeneralPages(platform, os, isTauriHost()); const activePage = page; if (activePage !== 'root') { return ( @@ -103,6 +104,12 @@ export function GeneralTab({ surface="microphone" /> )} + {activePage === 'askEntry' && ( + <> + + + + )} {activePage === 'askHistory' && } {activePage === 'capsule' && } {activePage === 'textInput' && ( diff --git a/openless-all/app/src/features/settings/shell/useSettingsNavigation.ts b/openless-all/app/src/features/settings/shell/useSettingsNavigation.ts index 30303a014..d3d88ad85 100644 --- a/openless-all/app/src/features/settings/shell/useSettingsNavigation.ts +++ b/openless-all/app/src/features/settings/shell/useSettingsNavigation.ts @@ -1,6 +1,6 @@ import { useEffect, useLayoutEffect, useRef, useState, type MutableRefObject } from 'react'; import { useTranslation } from 'react-i18next'; -import type { OS } from '../../../runtime/environment'; +import { isTauriHost, type OS } from '../../../runtime/environment'; import { getCachedPlatformCapabilities, getPlatformCapabilities } from '../../../runtime/platform'; import { searchSettingsSections, @@ -62,7 +62,9 @@ export function useSettingsNavigation({ : undefined; const activeGeneralPage = section === 'general' && !searching - ? visibleGeneralPages(platformCaps?.platform).find((page) => page.id === generalPage) + ? visibleGeneralPages(platformCaps?.platform, os, isTauriHost()).find( + (page) => page.id === generalPage, + ) : undefined; const showingVoiceSettings = activeAdvancedPage?.id === 'voiceEnhancement' && voiceSettingsOpen; const contentTitle = searching diff --git a/openless-all/app/src/i18n/askEntrySettings.ts b/openless-all/app/src/i18n/askEntrySettings.ts new file mode 100644 index 000000000..a09aa809a --- /dev/null +++ b/openless-all/app/src/i18n/askEntrySettings.ts @@ -0,0 +1,56 @@ +export const askEntrySettingsEn = { + title: 'Ask Anything', + description: 'Voice, intent routing, history, and app context.', + autoVoiceOnOpen: 'Auto-start voice', + autoVoiceHint: 'Start voice input when you open Ask with the recording key plus Space.', + routing: 'Intent routing', + routingHint: 'Choose how Ask decides whether to answer, write, edit, or ask for clarification.', + currentMode: 'Current mode', + semanticHybrid: 'Semantic hybrid', + separateModel: 'Separate model', + separateModelHint: 'Use a separate model request to decide the intent before generating.', + mainModelTool: 'Main model tool', + mainModelToolHint: 'Require the main model to resolve intent with a tool before responding.', + loadFailed: 'Could not load Ask settings.', + restoreDefaults: 'Restore Defaults', + restoreDefaultsHint: 'Turn on auto-start voice and restore Semantic hybrid.', + saveFailed: 'Could not save Ask settings. Your previous settings are unchanged.', +}; + +export const askEntrySettingsZhCN: typeof askEntrySettingsEn = { + title: '随便问', + description: '管理语音、意图路由、历史记录和应用上下文。', + autoVoiceOnOpen: '自动开始语音输入', + autoVoiceHint: '用录音键加空格打开「随便问」时,自动开始语音输入。', + routing: '意图路由', + routingHint: '选择「随便问」如何判断该回答、撰写、编辑,还是请你明确需求。', + currentMode: '当前模式', + semanticHybrid: '语义混合', + separateModel: '独立模型', + separateModelHint: '先用一次独立的模型请求判断意图,再生成内容。', + mainModelTool: '主模型工具', + mainModelToolHint: '要求主模型先调用工具判断意图,再生成内容。', + loadFailed: '无法读取「随便问」设置。', + restoreDefaults: '恢复默认设置', + restoreDefaultsHint: '开启自动语音输入,并恢复语义混合模式。', + saveFailed: '无法保存「随便问」设置。原有设置未更改。', +}; + +export const askEntrySettingsZhTW: typeof askEntrySettingsEn = { + title: '隨便問', + description: '管理語音、意圖路由、歷史記錄和應用程式上下文。', + autoVoiceOnOpen: '自動開始語音輸入', + autoVoiceHint: '用錄音鍵加空白鍵開啟「隨便問」時,自動開始語音輸入。', + routing: '意圖路由', + routingHint: '選擇「隨便問」如何判斷該回答、撰寫、編輯,還是請你釐清需求。', + currentMode: '目前模式', + semanticHybrid: '語意混合', + separateModel: '獨立模型', + separateModelHint: '先用一次獨立的模型請求判斷意圖,再產生內容。', + mainModelTool: '主模型工具', + mainModelToolHint: '要求主模型先呼叫工具判斷意圖,再產生內容。', + loadFailed: '無法讀取「隨便問」設定。', + restoreDefaults: '恢復預設設定', + restoreDefaultsHint: '開啟自動語音輸入,並恢復語意混合模式。', + saveFailed: '無法儲存「隨便問」設定。原有設定未變更。', +}; diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index 314e57f9f..54ca6f66b 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -4,11 +4,13 @@ import type { zhCN } from './zh-CN'; import { uiRefinementEn as uiRefinement } from './uiRefinement'; import { experimentalAsrEn as experimentalAsr } from './experimentalAsr'; +import { askEntrySettingsEn as askEntrySettings } from './askEntrySettings'; export const de: typeof zhCN = { desktopGuide, siwc, experimentalAsr, + askEntrySettings, uiRefinement, voiceEdit: { title: 'Sprachbearbeitung', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 1fc3fab3e..4efb9d94b 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -7,11 +7,13 @@ import type { zhCN } from './zh-CN'; // Type-level guarantee that en mirrors the zh-CN shape. import { uiRefinementEn as uiRefinement } from './uiRefinement'; import { experimentalAsrEn as experimentalAsr } from './experimentalAsr'; +import { askEntrySettingsEn as askEntrySettings } from './askEntrySettings'; export const en: typeof zhCN = { desktopGuide, siwc, experimentalAsr, + askEntrySettings, uiRefinement, voiceEdit: { title: 'Voice editing session', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 21fc2a02e..fb5a99ece 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -4,11 +4,13 @@ import type { zhCN } from './zh-CN'; import { uiRefinementEn as uiRefinement } from './uiRefinement'; import { experimentalAsrEn as experimentalAsr } from './experimentalAsr'; +import { askEntrySettingsEn as askEntrySettings } from './askEntrySettings'; export const es: typeof zhCN = { desktopGuide, siwc, experimentalAsr, + askEntrySettings, uiRefinement, voiceEdit: { title: 'Sesión de edición por voz', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index 425e2dee1..afc8d9566 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -4,11 +4,13 @@ import type { zhCN } from './zh-CN'; import { uiRefinementEn as uiRefinement } from './uiRefinement'; import { experimentalAsrEn as experimentalAsr } from './experimentalAsr'; +import { askEntrySettingsEn as askEntrySettings } from './askEntrySettings'; export const fr: typeof zhCN = { desktopGuide, siwc, experimentalAsr, + askEntrySettings, uiRefinement, voiceEdit: { title: 'Session de modification vocale', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index 316f81537..374500c42 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -5,11 +5,13 @@ import { siwcZhCN as siwc } from './siwc'; import { uiRefinementZhCN as uiRefinement } from './uiRefinement'; import { experimentalAsrZhCN as experimentalAsr } from './experimentalAsr'; +import { askEntrySettingsZhCN as askEntrySettings } from './askEntrySettings'; export const zhCN = { desktopGuide, siwc, experimentalAsr, + askEntrySettings, uiRefinement, voiceEdit: { title: '语音编辑会话', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index a1243728d..160087558 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -5,11 +5,13 @@ import type { zhCN } from './zh-CN'; // Traditional Chinese resources, sharing the same copy keys as the other seven locales. import { uiRefinementZhTW as uiRefinement } from './uiRefinement'; import { experimentalAsrZhTW as experimentalAsr } from './experimentalAsr'; +import { askEntrySettingsZhTW as askEntrySettings } from './askEntrySettings'; export const zhTW: typeof zhCN = { desktopGuide, siwc, experimentalAsr, + askEntrySettings, uiRefinement, voiceEdit: { title: '語音編輯工作階段', diff --git a/openless-all/ios/CoreHost/Cargo.lock b/openless-all/ios/CoreHost/Cargo.lock index bec79e086..2cb4f0f23 100644 --- a/openless-all/ios/CoreHost/Cargo.lock +++ b/openless-all/ios/CoreHost/Cargo.lock @@ -1482,6 +1482,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -1533,6 +1543,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1570,6 +1591,9 @@ dependencies = [ "lopdf", "md-5", "micyou-dsp", + "num-bigint", + "num-rational", + "num-traits", "once_cell", "parking_lot", "rand 0.8.8", From 6f0a5a23ada498ed1285e00731041ed115e7cea2 Mon Sep 17 00:00:00 2001 From: lvbaiqing Date: Fri, 20 Nov 2026 16:47:35 +0800 Subject: [PATCH 11/13] Remove obsolete Ask selection branches from the legacy Windows reader --- .../app/src-tauri/src/host_document/windows.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/openless-all/app/src-tauri/src/host_document/windows.rs b/openless-all/app/src-tauri/src/host_document/windows.rs index f0151fd07..6f536d9d8 100644 --- a/openless-all/app/src-tauri/src/host_document/windows.rs +++ b/openless-all/app/src-tauri/src/host_document/windows.rs @@ -114,17 +114,9 @@ pub(super) fn read_qa_selection() -> Option { return Ok(None); } let text = range - .GetText(if expected_pid.is_some() { 8_001 } else { 16_000 }) + .GetText(16_000) .map_err(|_| "selectionUnavailable".to_string())? .to_string(); - if expected_pid.is_some() - && !uia - .CompareElements(&element, &uia.GetFocusedElement().map_err(|_| unavailable())?) - .map_err(|_| unavailable())? - .as_bool() - { - return Ok(None); - } Ok((!text.trim().is_empty()).then_some(text)) }) .ok() From 17dca1c6db9eeb994a4d290933f3a5aa803d0865 Mon Sep 17 00:00:00 2001 From: lvbaiqing Date: Fri, 20 Nov 2026 18:13:10 +0800 Subject: [PATCH 12/13] Show only the prompt box in collapsed Ask --- docs/ask-unified-entry-contract.md | 2 +- openless-all/app/ask-window/ask-window.tsx | 94 ++++++++++++------- openless-all/app/ask-window/window.css | 9 +- openless-all/app/e2e/ask-window.spec.ts | 11 ++- .../app/src-tauri/src/ask_window/hotkey.rs | 2 +- .../app/src-tauri/src/ask_window/mod.rs | 13 +++ 6 files changed, 87 insertions(+), 44 deletions(-) diff --git a/docs/ask-unified-entry-contract.md b/docs/ask-unified-entry-contract.md index 26c7232b2..45155e424 100644 --- a/docs/ask-unified-entry-contract.md +++ b/docs/ask-unified-entry-contract.md @@ -4,7 +4,7 @@ This contract supersedes the previous standalone behavior for windows, hotkeys, ## User flow -The current recording binding plus Space opens or focuses the bottom `ask-anything` composer. Keep the existing Rare UI PromptBox, theme, draft, IME handling, thinking and attachment controls. The upper `ask-conversation` window is separate and never opened merely by the hotkey. An authoritative chat route opens it once for that turn, without taking the composer caret; routing to writing/editing/clarification does not show or hide the conversation window. Close/hide is per-window, does not cancel an accepted turn, and does not reopen the same panel on event replay. Reposition within the triggering monitor work area; retain native close button conventions and small-screen bounds. +The current recording binding plus Space opens or focuses the bottom `ask-anything` composer. Keep the existing Rare UI PromptBox, theme, draft, IME handling, thinking and attachment controls. The composer has no enclosing backdrop or close button; Escape dismisses it and cancels its pending voice input without clearing the draft. The upper `ask-conversation` window is separate and never opened merely by the hotkey. Only this conversation window has the expanded backdrop and platform-positioned close button. An authoritative chat route opens it once for that turn, without taking the composer caret; routing to writing/editing/clarification does not show or hide the conversation window. Close/hide is per-window, does not cancel an accepted turn, and does not reopen the same panel on event replay. Reposition within the triggering monitor work area and retain small-screen bounds. After a composer-focus acknowledgement for the current window/editor generation, the hotkey may launch the existing independent capsule. `autoVoiceOnOpen` defaults to true (including missing keys); it is applied only to the recording-key chord and never to a settings/menu open. Single-key dictation while the Ask composer is focused also uses the existing capsule. Both are insert-only, scoped to recording ID, conversation, editor instance/revision and focus generation; preserve unrelated draft text and current IME behavior. Repeated chords during active or starting recording are a focus-only action, never a stop or send. Losing the editor, changing conversation, cancellation, or a stale focus acknowledgement cannot send or insert late audio/transcript. No upload of Ask text or selection to ASR. diff --git a/openless-all/app/ask-window/ask-window.tsx b/openless-all/app/ask-window/ask-window.tsx index cca222384..efea89b3d 100644 --- a/openless-all/app/ask-window/ask-window.tsx +++ b/openless-all/app/ask-window/ask-window.tsx @@ -1,7 +1,7 @@ import { ToastNotificationCenter } from '../src/ui/feedback/ToastNotificationCenter'; import { AppViewport } from '../src/ui/layout/AppViewport'; import { InteractionProvider } from '../src/ui/overlays/InteractionProvider'; -import { useEffect, useRef, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { errorMessage, type AskClient, type AskFocus, type AskSnapshot } from './client'; import { useComposer } from './composer'; import { ConversationLayout, Sources, SubmissionResult } from './conversation'; @@ -112,7 +112,7 @@ export function AskWindow({ surface?: AskSurface; }) { return ( - + @@ -135,12 +135,46 @@ function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface const [platform, setPlatform] = useState(); const onWindows = platform ? platform === 'windows' : /Windows/i.test(navigator.userAgent); - function suspend() { + const suspend = useCallback(() => { shown.current = false; feed.current?.disconnect(); feed.current = undefined; setConnected(false); - } + }, []); + + const dismiss = useCallback(() => { + if (hiding.current || !shown.current) return; + hiding.current = true; + suspend(); + void client + .hide() + .catch((cause) => { + shown.current = true; + notice(errorMessage(cause, 'Could not hide Ask.'), 'error', 'ask-hide'); + reconnect.current(); + }) + .finally(() => { + hiding.current = false; + }); + }, [client, notice, suspend]); + + useEffect(() => { + if (surface !== 'composer') return; + function onEscape(event: KeyboardEvent) { + if ( + event.key !== 'Escape' || + event.defaultPrevented || + event.isComposing || + event.keyCode === 229 || + event.repeat + ) + return; + event.preventDefault(); + dismiss(); + } + window.addEventListener('keydown', onEscape); + return () => window.removeEventListener('keydown', onEscape); + }, [dismiss, surface]); useEffect(() => { let destroyed = false; @@ -264,42 +298,30 @@ function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface window.removeEventListener('drop', preventFileNavigation); document.removeEventListener('visibilitychange', onVisibilityChange); }; - }, [client, notice, surface]); + }, [client, notice, surface, suspend]); const Session = surface === 'composer' ? ComposerSession : ConversationSession; return ( <> -
- -
+ {surface === 'conversation' && ( +
+ +
+ )} {snapshot ? ( { test('only a live chat route shows the reader; gaps recover without moving or blurring the composer', async ({ page, }) => { - await expect(page.locator('#ask-root')).toHaveCSS('border-radius', '24px'); + await expect(page.locator('#ask-root')).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)'); + await expect(page.getByRole('button', { name: 'Close Ask Anything' })).toHaveCount(0); const prompt = page.getByRole('textbox', { name: 'Ask anything' }); const box = page.locator('[data-slot="prompt-box"]'); expect((await box.boundingBox())!.y).toBeGreaterThan(750); @@ -25,6 +26,7 @@ test('only a live chat route shows the reader; gaps recover without moving or bl await expect(page.getByText('Working', { exact: true })).toBeVisible(); await expect(page.locator('.user-bubble')).toHaveCount(0); await page.evaluate(() => window.askFixture.resolve('answer')); + await expect(page.locator('.ask-viewport-conversation')).toHaveCSS('border-radius', '24px'); await expect(page.locator('.user-bubble')).toHaveText('hi'); await expect(page.locator('.effort-label')).toHaveCount(0); await expect(prompt).toBeFocused(); @@ -42,7 +44,9 @@ test('only a live chat route shows the reader; gaps recover without moving or bl await expect.poll(() => page.evaluate(() => window.askFixture.liveListeners())).toBe(1); await page.evaluate(() => window.askFixture.showTranscript('turn-1')); await expect(page.getByRole('button', { name: 'Close Ask Conversation' })).toBeHidden(); - await page.getByRole('button', { name: 'Close Ask Anything' }).click(); + await prompt.dispatchEvent('keydown', { key: 'Escape', isComposing: true }); + expect(await page.evaluate(() => window.askFixture.hideCount)).toBe(0); + await prompt.press('Escape'); await expect.poll(() => page.evaluate(() => window.askFixture.liveListeners())).toBe(0); await page.evaluate(() => { const fixture = window.askFixture; @@ -157,7 +161,6 @@ test('attachment expansion keeps the bottom editor usable with a bounded viewpor const prompt = page.getByRole('textbox', { name: 'Ask anything' }); await expect(prompt).toBeFocused(); await expect(prompt).toBeInViewport(); - await expect(page.getByRole('button', { name: 'Close Ask Anything' })).toBeInViewport(); await expect(page.getByRole('button', { name: 'Attach a file' })).toBeInViewport(); }); @@ -387,7 +390,7 @@ test('write, edit, and clarify have no reader side effect and retain delivery ou await expect(page.getByText('Could not copy to the clipboard.')).toBeVisible(); await expect(prompt).toHaveValue(''); await expect(page.getByRole('button', { name: 'Sources · 1' })).toBeVisible(); - await page.getByRole('button', { name: 'Close Ask Anything' }).click(); + await retained.press('Escape'); await page.evaluate(() => window.askFixture.activate()); await expect(retained).toHaveValue('Total: $420 USD'); await page.evaluate(() => { diff --git a/openless-all/app/src-tauri/src/ask_window/hotkey.rs b/openless-all/app/src-tauri/src/ask_window/hotkey.rs index e80a4b4df..a543d166a 100644 --- a/openless-all/app/src-tauri/src/ask_window/hotkey.rs +++ b/openless-all/app/src-tauri/src/ask_window/hotkey.rs @@ -608,7 +608,7 @@ async fn dispatch(desktop: Weak, mut receiver: mpsc::UnboundedReceiver< } } Message::Escape => { - desktop.cancel_active_voice(); + desktop.dismiss_composer_from_escape(); } } } diff --git a/openless-all/app/src-tauri/src/ask_window/mod.rs b/openless-all/app/src-tauri/src/ask_window/mod.rs index 472beb541..40bd6e4ea 100644 --- a/openless-all/app/src-tauri/src/ask_window/mod.rs +++ b/openless-all/app/src-tauri/src/ask_window/mod.rs @@ -372,6 +372,19 @@ impl Desktop { Ok(()) } + fn dismiss_composer_from_escape(self: &Arc) { + let focus = self.composer_focus(); + self.cancel_active_voice(); + let desktop = self.clone(); + let _ = self.app.run_on_main_thread(move || { + if desktop.composer_focus() == focus && desktop.is_focused() { + if let Some(window) = desktop.app.get_webview_window(LABEL) { + let _ = desktop.hide(&window); + } + } + }); + } + fn notify_voice( &self, event: &str, From f363922a41f30bad93ba8ec1bf6db93fbbf8641f Mon Sep 17 00:00:00 2001 From: lvbaiqing Date: Fri, 20 Nov 2026 18:31:25 +0800 Subject: [PATCH 13/13] Expand Ask around its existing prompt box in one window --- docs/ask-unified-entry-contract.md | 6 +- openless-all/app/ask-window/ask-window.tsx | 187 ++++++++++------- openless-all/app/ask-window/client.ts | 1 + openless-all/app/ask-window/composer.tsx | 4 + openless-all/app/ask-window/conversation.tsx | 34 +-- openless-all/app/ask-window/main.tsx | 7 +- openless-all/app/ask-window/tauri-client.ts | 1 + openless-all/app/ask-window/window.css | 71 +++++-- openless-all/app/e2e/ask-window.spec.ts | 64 ++++-- openless-all/app/e2e/fixtures/ask-window.html | 8 - openless-all/app/e2e/fixtures/ask-window.tsx | 51 +---- .../src-tauri/capabilities/ask-window.json | 4 +- .../app/src-tauri/src/ask_window/chat.rs | 155 ++++++-------- .../app/src-tauri/src/ask_window/mod.rs | 198 ++++++------------ openless-all/app/src-tauri/src/lib.rs | 1 + .../app/src-tauri/src/tauri_events.rs | 12 +- 16 files changed, 384 insertions(+), 420 deletions(-) diff --git a/docs/ask-unified-entry-contract.md b/docs/ask-unified-entry-contract.md index 45155e424..6b38456d5 100644 --- a/docs/ask-unified-entry-contract.md +++ b/docs/ask-unified-entry-contract.md @@ -4,7 +4,7 @@ This contract supersedes the previous standalone behavior for windows, hotkeys, ## User flow -The current recording binding plus Space opens or focuses the bottom `ask-anything` composer. Keep the existing Rare UI PromptBox, theme, draft, IME handling, thinking and attachment controls. The composer has no enclosing backdrop or close button; Escape dismisses it and cancels its pending voice input without clearing the draft. The upper `ask-conversation` window is separate and never opened merely by the hotkey. Only this conversation window has the expanded backdrop and platform-positioned close button. An authoritative chat route opens it once for that turn, without taking the composer caret; routing to writing/editing/clarification does not show or hide the conversation window. Close/hide is per-window, does not cancel an accepted turn, and does not reopen the same panel on event replay. Reposition within the triggering monitor work area and retain small-screen bounds. +The current recording binding plus Space opens or focuses the `ask-anything` window with only the Rare UI PromptBox visible. The collapsed composer has no enclosing backdrop or close button, even if the external app has selected text. After an explicit submission is acknowledged and its authoritative route is `answer` or `selectionAnswer`, the same native window expands around the same input. The underlay starts at the measured prompt bounds and grows from the bottom on a curved transition to the original single-window 800 × 920 chat layout; reduced motion uses the final layout directly. Write, edit and clarify do not trigger expansion. Escape dismisses and cancels pending voice input without clearing the draft or cancelling an accepted turn; the expanded surface also has the platform-positioned close button. Reopening returns to the compact view, and replay cannot reopen a dismissed surface. After a composer-focus acknowledgement for the current window/editor generation, the hotkey may launch the existing independent capsule. `autoVoiceOnOpen` defaults to true (including missing keys); it is applied only to the recording-key chord and never to a settings/menu open. Single-key dictation while the Ask composer is focused also uses the existing capsule. Both are insert-only, scoped to recording ID, conversation, editor instance/revision and focus generation; preserve unrelated draft text and current IME behavior. Repeated chords during active or starting recording are a focus-only action, never a stop or send. Losing the editor, changing conversation, cancellation, or a stale focus acknowledgement cannot send or insert late audio/transcript. No upload of Ask text or selection to ASR. @@ -36,7 +36,7 @@ Core worker A exposes host-injectable lazy admission input (`intent_mode`, selec ### Projection/events and model -Continue one ordered sequence per conversation, replay and snapshot. Add a non-chat submission state (`currentTurnId`, `route`, existing activity/search state, `resultText`, `delivery`, `copyStatus`, structured error); preserve last retained result until next submission or explicit session end. UI can restore this on reopen without reenacting any external effect. Broadcast `submitted` (non-message admission) and `routeResolved { route }` for all accepted turns; the existing message `accepted`/answer delta events are exclusively chat routes after routing. Delivery progress/result is a per-turn upsert (`delivery` event, same data as snapshot), statuses `pending`, `written`, `retained`, `unknown`; clipboard is `notNeeded`, `pending`, `copied`, `failed`. Existing `activity`/`search`/`source`/`terminal` events stay conversation- and turn-scoped. Reader window can snapshot/listen/cancel/open verified sources, not submit or manage attachments. Composer can snapshot and listen; transcript shows actual incremental replies only for chat. Backend notification/window gating consumes the live chat route once by turn ID; frontend replays do not show windows. CSS/viewport details belong to worker C; coordinator owns native shared event emit/allowlist and command registration. +Continue one ordered sequence per conversation, replay and snapshot. Add a non-chat submission state (`currentTurnId`, `route`, existing activity/search state, `resultText`, `delivery`, `copyStatus`, structured error); preserve last retained result until next submission or explicit session end. UI can restore this on reopen without reenacting any external effect. Broadcast `submitted` (non-message admission) and `routeResolved { route }` for all accepted turns; the existing message `accepted`/answer delta events are exclusively chat routes after routing. Delivery progress/result is a per-turn upsert (`delivery` event, same data as snapshot), statuses `pending`, `written`, `retained`, `unknown`; clipboard is `notNeeded`, `pending`, `copied`, `failed`. Existing `activity`/`search`/`source`/`terminal` events stay conversation- and turn-scoped. The single Ask window shows incremental replies only for chat. Its expansion gate requires an acknowledged submission from the current visibility generation and that same turn’s authoritative chat route. The scoped desktop size command validates the route before resizing. A recovered route can expand the current submission even if live events lag, but recovery cannot expand a dismissed request. CSS/viewport details belong to worker C; coordinator owns native shared event emit/allowlist and command registration. `semanticHybrid` uses conservative local intent cues (not web search) and calls the configured main model for a small typed decision only when uncertain. `separateModel` makes one isolated classifier request with no transcript output. `mainModelTool` requires the configured main model to call a registered local `resolve_intent` tool before any user-facing output and resumes the same logical model conversation with the validated routing result; protocol tool continuation may use subsequent requests but not a standalone classifier session. If classifier result/choice is invalid or protocol cannot support the mode, show a localized composer error; do not silently switch model strategy. Classifier inputs are user instruction and minimum selection-presence/reference signals, not raw external text, app content, attachments or history. Tool arguments are untrusted decisions, not permission to write. A hidden classification result never becomes a chat message or renderable draft. After routing, normal question/selection and write/edit generation share existing chat provider/tool limits; edit uses the actual selection as data, body-only output for writes. Preserve private-context search isolation and existing public-only search decision if required; never expose selected text/attachment/history/target to web search. `Web Search` indicator is driven by actual tool status in the active surface, not by routing or speculative planning. @@ -44,7 +44,7 @@ Continue one ordered sequence per conversation, replay and snapshot. Add a non-c - A: Core standalone Ask runtime, prompt and provider adapters (not shared registration, settings, host or UI); tests for all routes/modes, tools and late/cancelled delivery. - B: Native external focus observer, target capture/verification and actual insert/replace in `ask_window/target*` and strictly necessary native target helpers; no app/router/voice IPC or window work. -- C: Ask window frontend, native window/voice/hotkey modules and optional Vite entries. Keep `ask_window/chat.rs`, `wire.rs`, `lib.rs`/ACL to I1. Preserve typing, attachment, ordinary dictation and focus contracts. +- C: Ask window frontend and native window/voice/hotkey modules. Keep `ask_window/chat.rs`, `wire.rs`, `lib.rs`/ACL to I1. Preserve typing, attachment, ordinary dictation and focus contracts. - D: Ask entry setting store and Settings UI/localization, targeted settings tests; no Core/router or shared native registration. I1 registers both commands and shares the same store instance with capture/admission and key routing. - I1: Cross-workstream IPC, registrations, result clipboard adapter, coordinator glue, integration/review, baseline lockfile and publication. No worker pushes or opens a PR. No archive snapshots are pushed with implementation. diff --git a/openless-all/app/ask-window/ask-window.tsx b/openless-all/app/ask-window/ask-window.tsx index efea89b3d..a44c6f481 100644 --- a/openless-all/app/ask-window/ask-window.tsx +++ b/openless-all/app/ask-window/ask-window.tsx @@ -1,7 +1,7 @@ import { ToastNotificationCenter } from '../src/ui/feedback/ToastNotificationCenter'; import { AppViewport } from '../src/ui/layout/AppViewport'; import { InteractionProvider } from '../src/ui/overlays/InteractionProvider'; -import { useCallback, useEffect, useRef, useState } from 'react'; +import { useCallback, useEffect, useRef, useState, type CSSProperties } from 'react'; import { errorMessage, type AskClient, type AskFocus, type AskSnapshot } from './client'; import { useComposer } from './composer'; import { ConversationLayout, Sources, SubmissionResult } from './conversation'; @@ -10,8 +10,6 @@ import { useAskNotice } from './notices'; import { watchTheme } from './theme'; import './window.css'; -export type AskSurface = 'composer' | 'conversation'; - type SessionProps = { client: AskClient; snapshot: AskSnapshot; @@ -20,8 +18,12 @@ type SessionProps = { focus?: AskFocus; activation: number; canDeliverVoice: () => boolean; + expanded: boolean; + captureSubmitReceipt: () => (turnId: string) => void; }; +type Expansion = { width: number; height: number; bottom: number }; + function ComposerSession(props: SessionProps) { const box = useComposer(props); const { client, snapshot } = props; @@ -29,8 +31,9 @@ function ComposerSession(props: SessionProps) { const current = submission?.currentTurnId === snapshot.activeTurnId; const chatRoute = submission?.route === 'answer' || submission?.route === 'selectionAnswer'; const [sourcesTurn, setSourcesTurn] = useState(); - const sourceTurns = - submission && !chatRoute + const sourceTurns = [ + ...(props.expanded ? snapshot.turns : []), + ...(submission && !chatRoute ? [ { turnId: submission.currentTurnId, @@ -38,17 +41,18 @@ function ComposerSession(props: SessionProps) { sources: submission.sources, }, ] - : []; + : []), + ]; return ( <> -
-
- {submission && !chatRoute && ( - - )} - {box} -
-
+ + {submission && !chatRoute && } + {box} + (); - const [cancelling, setCancelling] = useState(); - const notice = useAskNotice(); - const active = turns.some((turn) => turn.turnId === activeTurnId) ? activeTurnId : null; - return ( - <> - - {active && ( - - )} - - setSourcesTurn(undefined)} - /> - - ); -} - -export function AskWindow({ - client, - surface = 'composer', -}: { - client: AskClient; - surface?: AskSurface; -}) { +export function AskWindow({ client }: { client: AskClient }) { return ( - + - + ); } -function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface }) { +function AskScreen({ client }: { client: AskClient }) { const notice = useAskNotice(); const feed = useRef>(); const shown = useRef(true); const hiding = useRef(false); const reconnect = useRef<() => void>(() => {}); + const presentation = useRef(null); + const visibilityGeneration = useRef(0); + const expanding = useRef(); + const [receipt, setReceipt] = useState<{ conversationId: string; turnId: string }>(); + const [expansion, setExpansion] = useState(); const [snapshot, setSnapshot] = useState(); const [connected, setConnected] = useState(false); const [windowFocused, setWindowFocused] = useState(() => document.hasFocus()); @@ -142,12 +102,21 @@ function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface setConnected(false); }, []); + const clearPresentation = useCallback(() => { + visibilityGeneration.current++; + expanding.current = undefined; + setReceipt(undefined); + setExpansion(undefined); + }, []); + const dismiss = useCallback(() => { if (hiding.current || !shown.current) return; hiding.current = true; + visibilityGeneration.current++; suspend(); void client .hide() + .then(clearPresentation) .catch((cause) => { shown.current = true; notice(errorMessage(cause, 'Could not hide Ask.'), 'error', 'ask-hide'); @@ -156,10 +125,9 @@ function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface .finally(() => { hiding.current = false; }); - }, [client, notice, suspend]); + }, [clearPresentation, client, notice, suspend]); useEffect(() => { - if (surface !== 'composer') return; function onEscape(event: KeyboardEvent) { if ( event.key !== 'Escape' || @@ -174,7 +142,7 @@ function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface } window.addEventListener('keydown', onEscape); return () => window.removeEventListener('keydown', onEscape); - }, [dismiss, surface]); + }, [dismiss]); useEffect(() => { let destroyed = false; @@ -182,7 +150,7 @@ function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface let unlisten: (() => void) | undefined; const theme = watchTheme(); const updateFocus = ({ focusEpoch, focusRequest }: Partial) => { - if (surface !== 'composer' || focusEpoch === undefined || focusRequest === undefined) return; + if (focusEpoch === undefined || focusRequest === undefined) return; setFocus((previous) => previous && (previous.focusEpoch > focusEpoch || @@ -242,14 +210,17 @@ function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface void client .onWindowEvent((event) => { if (destroyed) return; - if (event.type === 'hidden') suspend(); + if (event.type === 'hidden') { + clearPresentation(); + suspend(); + } if (event.type === 'activated' || event.type === 'sessionChanged') { updateFocus(event); if (event.type === 'activated') { const wasHidden = !shown.current; shown.current = true; hiding.current = false; - if (surface === 'composer') setActivation((previous) => previous + 1); + setActivation((previous) => previous + 1); setWindowFocused(document.hasFocus()); if (wasHidden || !feed.current) connect(); else feed.current.check(); @@ -298,12 +269,68 @@ function AskScreen({ client, surface }: { client: AskClient; surface: AskSurface window.removeEventListener('drop', preventFileNavigation); document.removeEventListener('visibilitychange', onVisibilityChange); }; - }, [client, notice, surface, suspend]); + }, [clearPresentation, client, notice, suspend]); - const Session = surface === 'composer' ? ComposerSession : ConversationSession; + const route = snapshot?.submission?.route; + const routedTurn = snapshot?.submission?.currentTurnId; + const conversationId = snapshot?.conversationId; + useEffect(() => { + if ( + expansion || + !connected || + !shown.current || + !conversationId || + !receipt || + receipt.conversationId !== conversationId || + receipt.turnId !== routedTurn || + (route !== 'answer' && route !== 'selectionAnswer') || + expanding.current === routedTurn + ) + return; + const frame = presentation.current; + const box = frame?.querySelector('[data-slot="prompt-box"]'); + if (!frame || !box) return; + const start = box.getBoundingClientRect(); + const bounds = frame.getBoundingClientRect(); + const from = { width: start.width, height: start.height, bottom: bounds.bottom - start.bottom }; + const generation = visibilityGeneration.current; + expanding.current = receipt.turnId; + void (client.expandChat?.(conversationId, receipt.turnId) ?? Promise.resolve()) + .then(() => { + if (shown.current && visibilityGeneration.current === generation) setExpansion(from); + }) + .catch((cause) => { + if (shown.current && visibilityGeneration.current === generation) { + notice(errorMessage(cause, 'Could not expand Ask.'), 'error', 'ask-expand'); + } + }); + }, [client, connected, conversationId, expansion, notice, receipt, route, routedTurn]); + + function captureSubmitReceipt() { + const generation = visibilityGeneration.current; + return (turnId: string) => { + if (shown.current && visibilityGeneration.current === generation && conversationId) { + setReceipt({ conversationId, turnId }); + } + }; + } + + const expansionStyle = + expansion && + ({ + '--ask-start-width': `${expansion.width}px`, + '--ask-start-height': `${expansion.height}px`, + '--ask-start-bottom': `${Math.max(0, expansion.bottom)}px`, + } as CSSProperties); return ( - <> - {surface === 'conversation' && ( +
+ {expansion &&