From c10495aa7db3d07799ceb31dccf5da8fbc7b8c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=E1=BA=BF=20Anh=20=C4=90=E1=BA=B7ng?= Date: Sun, 11 Oct 2026 17:29:11 +0700 Subject: [PATCH 1/2] Add live browser tab streaming a shared Chrome session - Add a browser module (service + router) that runs one agent-browser session per user/project, proxies its WebSocket stream, and exposes navigate, viewport, insert-text, selection, and history endpoints. - Render streamed frames in a new BrowserTab canvas with pointer, wheel, keyboard, IME, and clipboard input, plus reconnection. - Apply session TTL, LRU eviction, origin checks, and auth so the tab and the AI agent share a single sandboxed browser. - Install agent-browser and Chrome in the image and document setup and troubleshooting. --- Dockerfile | 8 + app/core/config.py | 12 + app/core/errors.py | 6 + app/core/i18n.py | 12 + app/main.py | 8 +- app/modules/browser/__init__.py | 0 app/modules/browser/router.py | 398 ++++++++++++++++ app/modules/browser/service.py | 304 +++++++++++++ docs/installation.md | 63 +++ tests/conftest.py | 2 + tests/test_browser.py | 468 +++++++++++++++++++ web/src/components/IdeShell.tsx | 50 +- web/src/components/ide/BrowserTab.tsx | 630 ++++++++++++++++++++++++++ web/src/components/ide/NewTabMenu.tsx | 18 +- web/src/components/ide/TopTabs.tsx | 5 +- web/src/components/ide/icons.tsx | 32 ++ web/src/components/ide/tabDisplay.ts | 1 + web/src/i18n/locales/en.ts | 21 + web/src/i18n/locales/vi.ts | 21 + web/src/lib/api.ts | 89 ++++ web/src/lib/workspaceStore.ts | 28 +- 21 files changed, 2153 insertions(+), 23 deletions(-) create mode 100644 app/modules/browser/__init__.py create mode 100644 app/modules/browser/router.py create mode 100644 app/modules/browser/service.py create mode 100644 tests/test_browser.py create mode 100644 web/src/components/ide/BrowserTab.tsx diff --git a/Dockerfile b/Dockerfile index 355477f..6e6b8e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,14 @@ RUN apt-get update \ RUN curl -fsSL https://code-server.dev/install.sh | sh +# agent-browser drives a real Chrome (For Testing) for the live browser tab. +# Chrome is downloaded into the runtime user's home so the server can launch it. +RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && npm install -g agent-browser \ + && HOME=/home/remote-coder agent-browser install --with-deps \ + && rm -rf /var/lib/apt/lists/* + RUN useradd --create-home --shell /bin/bash remote-coder \ && mkdir -p \ /app/database \ diff --git a/app/core/config.py b/app/core/config.py index 5fe37f2..df47f6d 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -87,6 +87,18 @@ class Settings(BaseSettings): vscode_max_sessions: int = 4 vscode_extra_args: str = "" + browser_enabled: bool = True + browser_binary: str = "agent-browser" + browser_bind_host: str = "127.0.0.1" + browser_stream_quality: int = 60 + browser_stream_max_fps: int = 30 + browser_headless: bool = True + browser_start_timeout_seconds: int = 30 + browser_session_ttl_seconds: int = 3600 + browser_max_sessions: int = 2 + browser_allowed_domains: str = "" + browser_extra_args: str = "" + @model_validator(mode="after") def _derive_vscode_user_data_root(self) -> "Settings": if not self.vscode_user_data_root: diff --git a/app/core/errors.py b/app/core/errors.py index 9131964..8d738e1 100644 --- a/app/core/errors.py +++ b/app/core/errors.py @@ -76,6 +76,12 @@ class ErrorCode(StrEnum): VSCODE_START_FAILED = "VSCODE_START_FAILED" VSCODE_PROXY_FAILED = "VSCODE_PROXY_FAILED" VSCODE_FORBIDDEN = "VSCODE_FORBIDDEN" + BROWSER_DISABLED = "BROWSER_DISABLED" + BROWSER_START_FAILED = "BROWSER_START_FAILED" + BROWSER_PROXY_FAILED = "BROWSER_PROXY_FAILED" + BROWSER_FORBIDDEN = "BROWSER_FORBIDDEN" + BROWSER_NAVIGATION_FAILED = "BROWSER_NAVIGATION_FAILED" + BROWSER_COMMAND_FAILED = "BROWSER_COMMAND_FAILED" class FieldError(BaseModel): diff --git a/app/core/i18n.py b/app/core/i18n.py index 24d3612..957b435 100644 --- a/app/core/i18n.py +++ b/app/core/i18n.py @@ -74,6 +74,12 @@ "VSCODE_START_FAILED": "Could not start the VS Code server", "VSCODE_PROXY_FAILED": "Could not reach the VS Code server", "VSCODE_FORBIDDEN": "Cross-site request rejected", + "BROWSER_DISABLED": "The browser is disabled", + "BROWSER_START_FAILED": "Could not start the browser", + "BROWSER_PROXY_FAILED": "Could not reach the browser", + "BROWSER_FORBIDDEN": "Cross-site request rejected", + "BROWSER_NAVIGATION_FAILED": "Could not navigate the browser", + "BROWSER_COMMAND_FAILED": "Could not control the browser", }, "vi": { "INVALID_CREDENTIALS": "Tên đăng nhập hoặc mật khẩu không đúng", @@ -143,6 +149,12 @@ "VSCODE_START_FAILED": "Không thể khởi động VS Code server", "VSCODE_PROXY_FAILED": "Không thể kết nối tới VS Code server", "VSCODE_FORBIDDEN": "Từ chối yêu cầu cross-site", + "BROWSER_DISABLED": "Trình duyệt đang bị tắt", + "BROWSER_START_FAILED": "Không thể khởi động trình duyệt", + "BROWSER_PROXY_FAILED": "Không thể kết nối tới trình duyệt", + "BROWSER_FORBIDDEN": "Từ chối yêu cầu cross-site", + "BROWSER_NAVIGATION_FAILED": "Không thể điều hướng trình duyệt", + "BROWSER_COMMAND_FAILED": "Không thể điều khiển trình duyệt", }, } diff --git a/app/main.py b/app/main.py index 5846d92..2cc6a0a 100644 --- a/app/main.py +++ b/app/main.py @@ -16,6 +16,8 @@ from app.modules.ai_providers.router import router as ai_providers_router from app.modules.auth.router import router as auth_router from app.modules.auth.service import ensure_admin_user +from app.modules.browser import service as browser_service +from app.modules.browser.router import router as browser_router from app.modules.git.router import router as git_router from app.modules.health.router import router as health_router from app.modules.projects.router import router as projects_router @@ -66,6 +68,9 @@ async def _reap_idle_sessions(settings: Settings) -> None: await asyncio.to_thread( vscode_service.manager.reap_idle, settings.vscode_session_ttl_seconds ) + await asyncio.to_thread( + browser_service.manager.reap_idle, settings.browser_session_ttl_seconds + ) def create_app(settings: Settings | None = None) -> FastAPI: @@ -84,7 +89,7 @@ async def lifespan(app: FastAPI): with suppress(asyncio.CancelledError): await reaper await asyncio.to_thread(vscode_service.manager.kill_all) - + await asyncio.to_thread(browser_service.manager.kill_all) app = FastAPI( title=settings.app_name, debug=settings.debug, @@ -105,6 +110,7 @@ async def _apply_accept_language(request: Request, call_next): app.include_router(terminal_router, prefix=settings.api_prefix) app.include_router(terminal_global_router, prefix=settings.api_prefix) app.include_router(vscode_router, prefix=settings.api_prefix) + app.include_router(browser_router, prefix=settings.api_prefix) app.include_router(agents_router, prefix=settings.api_prefix) app.include_router(ai_providers_router, prefix=settings.api_prefix) app.include_router(ai_chat_router, prefix=settings.api_prefix) diff --git a/app/modules/browser/__init__.py b/app/modules/browser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/modules/browser/router.py b/app/modules/browser/router.py new file mode 100644 index 0000000..8e9a424 --- /dev/null +++ b/app/modules/browser/router.py @@ -0,0 +1,398 @@ +import asyncio +import json +import logging +from contextlib import suppress + +import websockets +from fastapi import ( + APIRouter, + HTTPException, + Query, + Request, + WebSocket, + WebSocketDisconnect, + status, +) +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy.orm import Session + +from app.core.config import Settings +from app.core.deps import DbDep, SettingsDep +from app.core.errors import ErrorCode, api_error, error_responses +from app.modules.auth.deps import CurrentUser +from app.modules.auth.models import User +from app.modules.auth.websocket import reject, same_origin, websocket_user +from app.modules.browser import service +from app.modules.projects import service as projects_service + +router = APIRouter(prefix="/projects", tags=["browser"]) + +logger = logging.getLogger(__name__) + +WS_UNAUTHORIZED = 4401 +WS_FORBIDDEN = 4403 +WS_NOT_FOUND = 4404 +WS_INTERNAL_ERROR = 4500 + + +class NavigateRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + url: str = Field(min_length=1, max_length=2048) + + +class ViewportRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + width: int = Field(ge=200, le=4000) + height: int = Field(ge=200, le=4000) + + +class InsertTextRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(min_length=1, max_length=100_000) + + +@router.post( + "/{project_id}/browser/navigate", + responses=error_responses(400, 401, 404, 502), +) +async def navigate( + request: Request, + project_id: int, + payload: NavigateRequest, + current_user: CurrentUser, + db: DbDep, + settings: SettingsDep, + session: str = Query(service.DEFAULT_SESSION), +) -> dict[str, bool]: + if not same_origin(request): + raise api_error(ErrorCode.BROWSER_FORBIDDEN, status_code=status.HTTP_403_FORBIDDEN) + + url = payload.url.strip() + if not service.is_http_url(url): + raise api_error( + ErrorCode.BROWSER_NAVIGATION_FAILED, status_code=status.HTTP_400_BAD_REQUEST + ) + + browser = await _ensure_session(db, settings, current_user, project_id, session) + db.close() + await _run(service.navigate_command(url, settings), browser) + return {"ok": True} + + +@router.post( + "/{project_id}/browser/viewport", + responses=error_responses(400, 401, 404, 502), +) +async def set_viewport( + request: Request, + project_id: int, + payload: ViewportRequest, + current_user: CurrentUser, + db: DbDep, + settings: SettingsDep, + session: str = Query(service.DEFAULT_SESSION), +) -> dict[str, bool]: + if not same_origin(request): + raise api_error(ErrorCode.BROWSER_FORBIDDEN, status_code=status.HTTP_403_FORBIDDEN) + + browser = await _ensure_session(db, settings, current_user, project_id, session) + db.close() + await _run( + service.viewport_command(payload.width, payload.height, settings), + browser, + code=ErrorCode.BROWSER_COMMAND_FAILED, + ) + return {"ok": True} + + +@router.post( + "/{project_id}/browser/insert", + responses=error_responses(400, 401, 404, 502), +) +async def insert_text( + request: Request, + project_id: int, + payload: InsertTextRequest, + current_user: CurrentUser, + db: DbDep, + settings: SettingsDep, + session: str = Query(service.DEFAULT_SESSION), +) -> dict[str, bool]: + if not same_origin(request): + raise api_error(ErrorCode.BROWSER_FORBIDDEN, status_code=status.HTTP_403_FORBIDDEN) + + browser = await _ensure_session(db, settings, current_user, project_id, session) + db.close() + await _run( + service.insert_text_command(payload.text, settings), + browser, + code=ErrorCode.BROWSER_COMMAND_FAILED, + ) + return {"ok": True} + + +@router.get( + "/{project_id}/browser/selection", + responses=error_responses(401, 404, 502), +) +async def get_selection( + request: Request, + project_id: int, + current_user: CurrentUser, + db: DbDep, + settings: SettingsDep, + session: str = Query(service.DEFAULT_SESSION), +) -> dict[str, str]: + if not same_origin(request): + raise api_error(ErrorCode.BROWSER_FORBIDDEN, status_code=status.HTTP_403_FORBIDDEN) + + browser = await _ensure_session(db, settings, current_user, project_id, session) + db.close() + try: + output = await asyncio.to_thread(browser.run, service.selection_command(settings)) + except service.BrowserError as exc: + raise api_error( + ErrorCode.BROWSER_COMMAND_FAILED, status_code=status.HTTP_502_BAD_GATEWAY + ) from exc + payload = service.parse_status(output) + data = payload.get("data") if isinstance(payload, dict) else None + text = data.get("result") if isinstance(data, dict) else None + return {"text": text if isinstance(text, str) else ""} + + +@router.post( + "/{project_id}/browser/{action}", + responses=error_responses(400, 401, 404, 502), +) +async def control( + request: Request, + project_id: int, + action: str, + current_user: CurrentUser, + db: DbDep, + settings: SettingsDep, + session: str = Query(service.DEFAULT_SESSION), +) -> dict[str, bool]: + if action not in service.NAVIGATION_ACTIONS: + raise api_error( + ErrorCode.BROWSER_NAVIGATION_FAILED, status_code=status.HTTP_400_BAD_REQUEST + ) + if not same_origin(request): + raise api_error(ErrorCode.BROWSER_FORBIDDEN, status_code=status.HTTP_403_FORBIDDEN) + + browser = await _ensure_session(db, settings, current_user, project_id, session) + db.close() + await _run(service.action_command(action, settings), browser) + return {"ok": True} + + +@router.delete( + "/{project_id}/browser/session", + responses=error_responses(401, 404), +) +async def close_session( + request: Request, + project_id: int, + current_user: CurrentUser, + db: DbDep, + session: str = Query(service.DEFAULT_SESSION), +) -> dict[str, bool]: + if not same_origin(request): + raise api_error(ErrorCode.BROWSER_FORBIDDEN, status_code=status.HTTP_403_FORBIDDEN) + + # Confirms the project exists and belongs to the user (raises 404 otherwise). + projects_service.get_project(db, current_user, project_id) + key = (current_user.id, project_id, service.safe_session(session)) + # Kill only; never start a session just to close it. + await asyncio.to_thread(service.manager.kill, key) + return {"ok": True} + + +@router.websocket("/{project_id}/browser/stream") +async def proxy_stream( + websocket: WebSocket, + project_id: int, + db: DbDep, + settings: SettingsDep, +) -> None: + if not same_origin(websocket): + await reject(websocket, WS_FORBIDDEN) + return + + user = websocket_user(websocket, db, settings) + if user is None: + await reject(websocket, WS_UNAUTHORIZED) + return + + # Accept before starting the session so a slow or failing startup reaches + # the client as a message instead of a handshake that never completes. + await websocket.accept() + + try: + browser = await _ensure_session( + db, + settings, + user, + project_id, + websocket.query_params.get("session", service.DEFAULT_SESSION), + ) + except HTTPException as exc: + await _send_error_and_close(websocket, exc) + return + except Exception: # noqa: BLE001 - unexpected failures still close cleanly + await _send_error_and_close( + websocket, + api_error( + ErrorCode.BROWSER_PROXY_FAILED, + status_code=status.HTTP_502_BAD_GATEWAY, + ), + ) + return + + db.close() + + try: + upstream = await _open_upstream(browser.host, browser.port, settings) + except (OSError, websockets.WebSocketException): + await _send_error_and_close( + websocket, + api_error( + ErrorCode.BROWSER_PROXY_FAILED, + status_code=status.HTTP_502_BAD_GATEWAY, + ), + ) + return + + browser.touch() + + sender = asyncio.create_task(_pump_client_to_server(websocket, upstream)) + receiver = asyncio.create_task(_pump_server_to_client(websocket, upstream)) + try: + await asyncio.wait({sender, receiver}, return_when=asyncio.FIRST_COMPLETED) + finally: + for task in (sender, receiver): + task.cancel() + with suppress(Exception): + await upstream.close() + + +async def _open_upstream(host: str, port: int, settings: Settings): + """Connect to the session's stream server, trying every loopback spelling. + + The server may bind IPv4 or IPv6, so a refused connection on the configured + host falls through to the other loopback forms instead of failing outright. + """ + last_error: Exception | None = None + tried: list[str] = [] + for candidate in (host, "127.0.0.1", "::1", "localhost"): + if candidate in tried: + continue + tried.append(candidate) + netloc = f"[{candidate}]" if ":" in candidate else candidate + target = f"ws://{netloc}:{port}/?pacing=push&maxFps={settings.browser_stream_max_fps}" + try: + return await websockets.connect( + target, + max_size=None, + open_timeout=settings.browser_start_timeout_seconds, + origin=f"http://{netloc}:{port}", + ) + except (OSError, websockets.WebSocketException) as exc: + last_error = exc + logger.warning("browser stream connect failed at %s: %s", target, exc) + assert last_error is not None + raise last_error + + +async def _ensure_session( + db: Session, + settings: Settings, + user: User, + project_id: int, + session: str, +) -> service.BrowserSession: + if not settings.browser_enabled: + raise api_error(ErrorCode.BROWSER_DISABLED, status_code=status.HTTP_403_FORBIDDEN) + + # Confirms the project exists and belongs to the user (raises 404 otherwise). + projects_service.get_project(db, user, project_id) + + session = service.safe_session(session) + key = (user.id, project_id, session) + name = service.session_name(user.id, project_id, session) + try: + return await asyncio.to_thread( + service.manager.ensure, key, name=name, settings=settings + ) + except service.BrowserError as exc: + raise api_error( + ErrorCode.BROWSER_START_FAILED, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR + ) from exc + + +async def _run( + command: list[str], + session: service.BrowserSession, + *, + code: ErrorCode = ErrorCode.BROWSER_NAVIGATION_FAILED, +) -> None: + try: + await asyncio.to_thread(session.run, command) + except service.BrowserError as exc: + raise api_error(code, status_code=status.HTTP_502_BAD_GATEWAY) from exc + + +def _ws_close_code(exc: Exception) -> int: + if isinstance(exc, HTTPException): + if exc.status_code == status.HTTP_401_UNAUTHORIZED: + return WS_UNAUTHORIZED + if exc.status_code == status.HTTP_404_NOT_FOUND: + return WS_NOT_FOUND + if exc.status_code == status.HTTP_403_FORBIDDEN: + return WS_FORBIDDEN + return WS_INTERNAL_ERROR + + +async def _send_error_and_close(websocket: WebSocket, exc: HTTPException) -> None: + """Report the failure reason over the accepted socket, then close.""" + detail = exc.detail if isinstance(exc.detail, dict) else {} + with suppress(RuntimeError, WebSocketDisconnect): + await websocket.send_text( + json.dumps( + { + "type": "error", + "code": detail.get("code", ErrorCode.BROWSER_PROXY_FAILED.value), + "message": detail.get("message", ""), + } + ) + ) + with suppress(RuntimeError): + await websocket.close(code=_ws_close_code(exc)) + + +async def _pump_client_to_server(websocket: WebSocket, upstream) -> None: + try: + while True: + message = await websocket.receive() + if message["type"] == "websocket.disconnect": + return + if (text := message.get("text")) is not None: + await upstream.send(text) + elif (data := message.get("bytes")) is not None: + await upstream.send(data) + except (WebSocketDisconnect, RuntimeError): + return + + +async def _pump_server_to_client(websocket: WebSocket, upstream) -> None: + try: + async for message in upstream: + if isinstance(message, str): + await websocket.send_text(message) + else: + await websocket.send_bytes(message) + except (WebSocketDisconnect, RuntimeError, websockets.WebSocketException): + return diff --git a/app/modules/browser/service.py b/app/modules/browser/service.py new file mode 100644 index 0000000..9522687 --- /dev/null +++ b/app/modules/browser/service.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import json +import os +import re +import shlex +import subprocess +import threading +import time +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass, field +from urllib.parse import urlsplit + +from app.core.config import Settings + +BrowserKey = tuple[int, int, str] + +# A runner executes an agent-browser argv with its environment and returns +# stdout. It is injectable so tests never need the real binary. +CommandRunner = Callable[[list[str], dict[str, str]], str | None] + +COMMAND_TIMEOUT_SECONDS = 30 + +NAVIGATION_ACTIONS = frozenset({"back", "forward", "reload"}) + +_SESSION_RE = re.compile(r"[^A-Za-z0-9._-]+") + +DEFAULT_SESSION = "default" + + +class BrowserError(RuntimeError): + """Raised when a browser session cannot be started or driven.""" + + +def safe_session(name: str) -> str: + """Turn a client-supplied session id into a filesystem-safe component.""" + cleaned = _SESSION_RE.sub("-", name).strip("-.") + return cleaned[:64] or DEFAULT_SESSION + + +def session_name(user_id: int, project_id: int, session: str) -> str: + return f"u{user_id}-p{project_id}-{session}" + + +def session_env(name: str, settings: Settings) -> dict[str, str]: + """Environment for every agent-browser call in one session. + + The session name is carried through the environment so the argv stays free + of repeated global options. The stream port is left to agent-browser, which + auto-assigns one; ``stream status`` reports the real port. + """ + env = os.environ.copy() + env["AGENT_BROWSER_SESSION"] = name + env["AGENT_BROWSER_STREAM_QUALITY"] = str(settings.browser_stream_quality) + return env + + +def open_command(settings: Settings) -> list[str]: + """Launch Chrome for the current session without navigating.""" + args = [settings.browser_binary, "open"] + if not settings.browser_headless: + args.append("--headed") + if settings.browser_allowed_domains.strip(): + args.extend(["--allowed-domains", settings.browser_allowed_domains]) + if settings.browser_extra_args.strip(): + args.extend(shlex.split(settings.browser_extra_args)) + return args + + +def status_command(settings: Settings) -> list[str]: + return [settings.browser_binary, "stream", "status", "--json"] + + +def stream_enable_command(settings: Settings) -> list[str]: + return [settings.browser_binary, "stream", "enable"] + + +def close_command(settings: Settings) -> list[str]: + return [settings.browser_binary, "close"] + + +def navigate_command(url: str, settings: Settings) -> list[str]: + return [settings.browser_binary, "open", url] + + +def action_command(action: str, settings: Settings) -> list[str]: + if action not in NAVIGATION_ACTIONS: + raise BrowserError(f"unsupported browser action: {action}") + return [settings.browser_binary, action] + + +def viewport_command(width: int, height: int, settings: Settings) -> list[str]: + return [settings.browser_binary, "set", "viewport", str(width), str(height)] + + +def insert_text_command(text: str, settings: Settings) -> list[str]: + # A NUL byte cannot be passed through argv; drop it defensively. + return [settings.browser_binary, "keyboard", "inserttext", text.replace("\x00", "")] + + +# Returns the current selection: the focused input/textarea's selection, or the +# document selection otherwise. +SELECTION_JS = ( + "(function(){var a=document.activeElement;" + "if(a&&(a.tagName==='INPUT'||a.tagName==='TEXTAREA')&&typeof a.selectionStart==='number')" + "{return a.value.substring(a.selectionStart,a.selectionEnd);}" + "return String(window.getSelection());})()" +) + + +def selection_command(settings: Settings) -> list[str]: + return [settings.browser_binary, "eval", SELECTION_JS, "--json"] + + +def is_http_url(url: str) -> bool: + parts = urlsplit(url) + return parts.scheme in {"http", "https"} and bool(parts.netloc) + + +def _default_runner(command: list[str], env: dict[str, str]) -> str: + try: + result = subprocess.run( + command, + env=env, + capture_output=True, + text=True, + timeout=COMMAND_TIMEOUT_SECONDS, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise BrowserError("could not run agent-browser") from exc + if result.returncode != 0: + raise BrowserError("agent-browser command failed") + return result.stdout + + +def parse_status(output: str | None) -> dict[str, object] | None: + """Extract the JSON status object from agent-browser stdout.""" + if not output: + return None + start = output.find("{") + end = output.rfind("}") + if start == -1 or end < start: + return None + try: + data = json.loads(output[start : end + 1]) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + +def _enabled_status( + runner: CommandRunner, env: dict[str, str], settings: Settings +) -> dict[str, object] | None: + """Return the stream ``data`` block when the session's stream is enabled. + + ``agent-browser stream status --json`` wraps its payload as + ``{"success": true, "data": {...}, "error": null}``, so ``enabled`` and + ``port`` live under ``data``. + """ + status = parse_status(runner(status_command(settings), env)) + if status is None: + return None + data = status.get("data") + if isinstance(data, dict) and data.get("enabled"): + return data + return None + + +@dataclass +class BrowserSession: + """A live agent-browser session serving one project over a stream port.""" + + host: str + port: int + name: str + runner: CommandRunner + settings: Settings + last_seen: float = field(default_factory=time.monotonic) + + def touch(self) -> None: + self.last_seen = time.monotonic() + + def alive(self) -> bool: + try: + return _enabled_status(self.runner, self._env(), self.settings) is not None + except BrowserError: + return False + + def run(self, command: list[str]) -> str | None: + try: + return self.runner(command, self._env()) + except BrowserError as exc: + raise BrowserError("browser command failed") from exc + + def kill(self) -> None: + try: + self.runner(close_command(self.settings), self._env()) + except BrowserError: + pass + + def _env(self) -> dict[str, str]: + return session_env(self.name, self.settings) + + +class BrowserManager: + """Keeps agent-browser sessions keyed by user and project.""" + + def __init__(self) -> None: + self._sessions: dict[BrowserKey, BrowserSession] = {} + self._lock = threading.Lock() + + def get(self, key: BrowserKey) -> BrowserSession | None: + with self._lock: + return self._sessions.get(key) + + def ensure( + self, + key: BrowserKey, + *, + name: str, + settings: Settings, + runner: CommandRunner | None = None, + ) -> BrowserSession: + """Return the running session for ``key``, starting one if needed.""" + run = runner or _default_runner + with self._lock: + existing = self._sessions.get(key) + if existing is not None and existing.alive(): + existing.touch() + return existing + if existing is not None: + del self._sessions[key] + + self._evict_lru(settings.browser_max_sessions) + session = self._start(name=name, settings=settings, runner=run) + self._sessions[key] = session + return session + + def _evict_lru(self, max_sessions: int) -> None: + """Free a slot by stopping the least recently used session. + + Called with the lock held, before a new session is inserted. + """ + if max_sessions <= 0: + return + while len(self._sessions) >= max_sessions: + victim_key = min(self._sessions, key=lambda key: self._sessions[key].last_seen) + session = self._sessions.pop(victim_key) + session.kill() + + def _start( + self, + *, + name: str, + settings: Settings, + runner: CommandRunner, + ) -> BrowserSession: + host = settings.browser_bind_host + env = session_env(name, settings) + runner(open_command(settings), env) + + status = _enabled_status(runner, env, settings) + if status is None: + # Streaming is on by default; enable explicitly if the session came + # up without it (for example a reused daemon). + with suppress(BrowserError): + runner(stream_enable_command(settings), env) + status = _enabled_status(runner, env, settings) + + port = status.get("port") if status is not None else None + if not isinstance(port, int) or port <= 0: + raise BrowserError("the browser stream server did not report a port") + return BrowserSession(host=host, port=port, name=name, runner=runner, settings=settings) + + def kill(self, key: BrowserKey) -> None: + with self._lock: + session = self._sessions.pop(key, None) + if session is not None: + session.kill() + + def reap_idle(self, ttl_seconds: int) -> None: + if ttl_seconds <= 0: + return + now = time.monotonic() + with self._lock: + victims = [ + key + for key, session in self._sessions.items() + if now - session.last_seen >= ttl_seconds + ] + for key in victims: + self.kill(key) + + def kill_all(self) -> None: + with self._lock: + sessions = list(self._sessions.values()) + self._sessions.clear() + for session in sessions: + session.kill() + + +manager = BrowserManager() diff --git a/docs/installation.md b/docs/installation.md index 09833d5..1485e32 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -57,6 +57,35 @@ If the binary lives outside `PATH`, point the app at it via `.env`: VSCODE_BINARY=/absolute/path/to/code-server ``` +## 1b. Install agent-browser + +The live Browser tab streams a real Chrome session driven by +[`agent-browser`](https://github.com/vercel-labs/agent-browser). Install the +CLI and download Chrome for Testing: + +```bash +npm install -g agent-browser +agent-browser install --with-deps # Linux: also installs required libraries +``` + +`--with-deps` runs the system package manager, so it needs root (use `sudo`). +`npm install -g` only needs root when npm's global prefix is system-owned (it +does not under nvm). Run `agent-browser install` as the **same user that runs +the API** so Chrome lands in that user's home; otherwise the server cannot find +it. + +Verify the install: + +```bash +agent-browser --version +``` + +If the binary lives outside `PATH`, point the app at it via `.env`: + +``` +BROWSER_BINARY=/absolute/path/to/agent-browser +``` + ## 2. Set up the API ```bash @@ -134,6 +163,40 @@ Other things to check: - `code-server` has write access to its data dir (`~/.remote-coder/vscode/cache` by default). +### `BROWSER_START_FAILED` / the Browser tab is blank + +The live Browser tab streams a real Chrome session driven by `agent-browser`. +When the binary or Chrome is missing the session cannot start, and the tab +reports **"Could not start the browser"** (error code `BROWSER_START_FAILED`). + +Install it (see step 1b): + +```bash +npm install -g agent-browser +agent-browser install --with-deps # downloads Chrome for Testing + libs +``` + +Confirm the install (step 1b): + +```bash +agent-browser --version +agent-browser stream status --json # expect {"enabled": true, ...} +``` + +Other things to check: + +- Restart the API after installing `agent-browser` so the new binary is picked + up from `PATH` (or point at it with `BROWSER_BINARY`). +- Chrome for Testing is present; re-run `agent-browser install --with-deps`. +- The API log shows `browser stream connect failed at ws://127.0.0.1:/...` + when it cannot reach the session's stream server; the port comes from + `agent-browser stream status --json`. +- To disable the feature entirely, set `BROWSER_ENABLED=false`. + +The tab and the AI agent share one browser per project, so the address bar +navigates that shared session. Use `BROWSER_ALLOWED_DOMAINS` (comma-separated) +to restrict which hosts the browser may reach. + ### `uvicorn not found` The API dependencies are not installed. Run step 2. diff --git a/tests/conftest.py b/tests/conftest.py index 73dcf17..4d6e022 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ from app.modules.auth.pow import pow_store from app.modules.auth.rate_limit import limiter from app.modules.auth.security import hash_password +from app.modules.browser import service as browser_service from app.modules.terminal import service as terminal_service from app.modules.vscode import service as vscode_service @@ -81,6 +82,7 @@ def override_get_db(): finally: terminal_service.manager.kill_all() vscode_service.manager.kill_all() + browser_service.manager.kill_all() app.dependency_overrides.clear() diff --git a/tests/test_browser.py b/tests/test_browser.py new file mode 100644 index 0000000..ca2f4a4 --- /dev/null +++ b/tests/test_browser.py @@ -0,0 +1,468 @@ +import json +import socket +import subprocess +import threading +import time +from pathlib import Path +from uuid import uuid4 + +import pytest +import uvicorn +from fastapi.testclient import TestClient +from starlette.applications import Starlette +from starlette.routing import WebSocketRoute +from starlette.websockets import WebSocket, WebSocketDisconnect + +from app.core.config import Settings +from app.modules.browser import service +from tests.conftest import LOCAL_URL, OTHER_USERNAME, _csrf, _login, assert_ws_close + + +def _register_repo_with_worktree(client: TestClient, projects_root: Path) -> tuple[int, Path]: + repo = projects_root / "wt-main" + _init_repo(repo) + _git(repo, "worktree", "add", "-b", "agent", str(projects_root / "wt-agent")) + response = client.post(LOCAL_URL, json={"path": str(repo)}, headers=_csrf(client)) + assert response.status_code == 201, response.text + return response.json()["id"], repo + + +def _git(repo: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True, text=True) + + +def _init_repo(repo: Path) -> None: + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "test@example.com") + _git(repo, "config", "user.name", "Test") + (repo / "tracked.txt").write_text("hello\n") + _git(repo, "add", "tracked.txt") + _git(repo, "commit", "-qm", "init") + + +def _browser_url(project_id: int, subpath: str = "stream") -> str: + return f"/api/v1/projects/{project_id}/browser/{subpath}" + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +class _Upstream: + """A minimal WebSocket server standing in for the agent-browser stream.""" + + def __init__(self) -> None: + self.port = _free_port() + routes = [WebSocketRoute("/", self._ws)] + config = uvicorn.Config( + Starlette(routes=routes), host="127.0.0.1", port=self.port, log_level="error" + ) + self._server = uvicorn.Server(config) + self._thread = threading.Thread(target=self._server.run, daemon=True) + + async def _ws(self, websocket: WebSocket) -> None: + await websocket.accept() + async for message in websocket.iter_text(): + await websocket.send_text(f"echo:{message}") + + def __enter__(self) -> "_Upstream": + self._thread.start() + for _ in range(200): + if self._server.started: + break + time.sleep(0.05) + else: # pragma: no cover - startup failure + raise RuntimeError("upstream server did not start") + return self + + def __exit__(self, *exc: object) -> None: + self._server.should_exit = True + self._thread.join(timeout=5) + + +class _RecordingRunner: + def __init__(self) -> None: + self.calls: list[list[str]] = [] + + def __call__(self, command: list[str], env: dict[str, str]) -> str: + self.calls.append(command) + return "" + + +def _session_for(port: int, runner: service.CommandRunner) -> service.BrowserSession: + settings = Settings() + return service.BrowserSession( + host="127.0.0.1", port=port, name="test", runner=runner, settings=settings + ) + + +class _StubManager: + def __init__(self, session: service.BrowserSession) -> None: + self.session = session + + def ensure(self, key, *, name, settings, runner=None, port=None): # noqa: ANN001 + self.session.touch() + return self.session + + def kill(self, key) -> None: # noqa: ANN001 + return None + + def kill_all(self) -> None: + return None + + +def test_browser_requires_authentication(client: TestClient, projects_root: Path) -> None: + response = client.post(_browser_url(1, "navigate"), json={"url": "https://x.com"}) + assert response.status_code == 401 + + +def test_browser_rejects_unknown_project(client: TestClient, projects_root: Path) -> None: + _login(client) + response = client.post( + _browser_url(999, "navigate"), json={"url": "https://x.com"} + ) + assert response.status_code == 404 + + + + +def test_browser_scoped_to_project_owner(client: TestClient, projects_root: Path) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + client.cookies.clear() + + _login(client, username=OTHER_USERNAME) + response = client.post( + _browser_url(project_id, "navigate"), json={"url": "https://x.com"} + ) + assert response.status_code == 404 + + +def test_browser_ws_requires_authentication(client: TestClient, projects_root: Path) -> None: + assert_ws_close(client, _browser_url(1), 4401) + + +def test_browser_ws_rejects_cross_site_origin(client: TestClient, projects_root: Path) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + + assert_ws_close( + client, + _browser_url(project_id), + 4403, + headers={"origin": "http://evil.example"}, + ) + + +def test_browser_navigate_rejects_cross_site_origin( + client: TestClient, projects_root: Path +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + response = client.post( + _browser_url(project_id, "navigate"), + json={"url": "https://example.com/"}, + headers={**_csrf(client), "origin": "http://evil.example"}, + ) + assert response.status_code == 403 + + +def test_browser_rejects_non_http_url(client: TestClient, projects_root: Path) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + response = client.post( + _browser_url(project_id, "navigate"), + json={"url": "file:///etc/passwd"}, + headers=_csrf(client), + ) + assert response.status_code == 400 + + +def test_browser_navigate_runs_command( + client: TestClient, projects_root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + runner = _RecordingRunner() + monkeypatch.setattr(service, "manager", _StubManager(_session_for(1234, runner))) + + response = client.post( + _browser_url(project_id, "navigate"), + json={"url": "https://example.com/"}, + headers=_csrf(client), + ) + + assert response.status_code == 200 + assert runner.calls == [["agent-browser", "open", "https://example.com/"]] + + +def test_browser_control_runs_action( + client: TestClient, projects_root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + runner = _RecordingRunner() + monkeypatch.setattr(service, "manager", _StubManager(_session_for(1234, runner))) + + response = client.post(_browser_url(project_id, "reload"), headers=_csrf(client)) + + assert response.status_code == 200 + assert runner.calls == [["agent-browser", "reload"]] + + +def test_browser_control_rejects_unknown_action( + client: TestClient, projects_root: Path +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + response = client.post(_browser_url(project_id, "explode"), headers=_csrf(client)) + assert response.status_code == 400 + + +def test_browser_set_viewport_runs_command( + client: TestClient, projects_root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + runner = _RecordingRunner() + monkeypatch.setattr(service, "manager", _StubManager(_session_for(1234, runner))) + + response = client.post( + _browser_url(project_id, "viewport"), + json={"width": 900, "height": 500}, + headers=_csrf(client), + ) + + assert response.status_code == 200 + assert runner.calls == [["agent-browser", "set", "viewport", "900", "500"]] + + +def test_browser_set_viewport_rejects_out_of_range( + client: TestClient, projects_root: Path +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + response = client.post( + _browser_url(project_id, "viewport"), + json={"width": 10, "height": 500}, + headers=_csrf(client), + ) + assert response.status_code == 422 + + +def test_browser_insert_runs_command( + client: TestClient, projects_root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + runner = _RecordingRunner() + monkeypatch.setattr(service, "manager", _StubManager(_session_for(1234, runner))) + + response = client.post( + _browser_url(project_id, "insert"), + json={"text": "pasted text"}, + headers=_csrf(client), + ) + + assert response.status_code == 200 + assert runner.calls == [["agent-browser", "keyboard", "inserttext", "pasted text"]] + + +def test_browser_selection_returns_text( + client: TestClient, projects_root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + + def runner(command: list[str], env: dict[str, str]) -> str: + return json.dumps({"success": True, "data": {"result": "SELECTED"}, "error": None}) + + monkeypatch.setattr(service, "manager", _StubManager(_session_for(1234, runner))) + + response = client.get(_browser_url(project_id, "selection")) + + assert response.status_code == 200 + assert response.json() == {"text": "SELECTED"} + + +def test_browser_close_session_kills( + client: TestClient, projects_root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + killed: list[tuple[int, int, str]] = [] + monkeypatch.setattr(service, "manager", _KillRecordingManager(killed)) + + response = client.delete( + f"{_browser_url(project_id, 'session')}?session=tab-1", headers=_csrf(client) + ) + + assert response.status_code == 200 + assert killed == [(1, project_id, "tab-1")] + + +class _KillRecordingManager: + def __init__(self, killed: list[tuple[int, int, str]]) -> None: + self.killed = killed + + def kill(self, key) -> None: # noqa: ANN001 + self.killed.append(key) + + def kill_all(self) -> None: + return None + + +class _FailingManager: + def ensure(self, key, *, name, settings, runner=None, port=None): # noqa: ANN001 + raise service.BrowserError("boom") + + def kill(self, key) -> None: # noqa: ANN001 + return None + + def kill_all(self) -> None: + return None + + +def test_browser_stream_reports_start_failure( + client: TestClient, projects_root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + monkeypatch.setattr(service, "manager", _FailingManager()) + + with client.websocket_connect(_browser_url(project_id)) as websocket: + message = json.loads(websocket.receive_text()) + assert message["type"] == "error" + assert message["code"] == "BROWSER_START_FAILED" + with pytest.raises(WebSocketDisconnect) as exc: + websocket.receive_text() + assert exc.value.code == 4500 + + +def test_browser_proxies_stream( + client: TestClient, projects_root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _login(client) + project_id, _ = _register_repo_with_worktree(client, projects_root) + + with _Upstream() as upstream: + session = _session_for(upstream.port, _RecordingRunner()) + monkeypatch.setattr(service, "manager", _StubManager(session)) + with client.websocket_connect(_browser_url(project_id)) as websocket: + websocket.send_text("ping") + assert websocket.receive_text() == "echo:ping" + + +def test_open_command_flags() -> None: + headless = service.open_command(Settings(browser_binary="agent-browser")) + assert headless == ["agent-browser", "open"] + + headed = service.open_command( + Settings( + browser_binary="agent-browser", + browser_headless=False, + browser_allowed_domains="example.com", + ) + ) + assert "--headed" in headed + assert "--allowed-domains" in headed + assert "example.com" in headed + + +def test_session_name() -> None: + assert service.safe_session("tab one/../x") == "tab-one-..-x" + assert service.safe_session("") == "default" + assert service.session_name(1, 2, "abc") == "u1-p2-abc" + + +def test_parse_status_extracts_json() -> None: + assert service.parse_status(None) is None + assert service.parse_status("no json here") is None + payload = service.parse_status('noise\n{"enabled": true, "port": 9223}\ntail') + assert payload == {"enabled": True, "port": 9223} + + +def _manager_runner(command: list[str], env: dict[str, str]) -> str: + return json.dumps( + { + "success": True, + "data": {"enabled": True, "port": 9223, "connected": True, "screencasting": False}, + "error": None, + } + ) + + +def test_manager_reuses_and_kills_session() -> None: + settings = Settings(browser_bind_host="127.0.0.1") + manager = service.BrowserManager() + key = (1, 1, uuid4().hex) + try: + first = manager.ensure(key, name="s1", settings=settings, runner=_manager_runner) + again = manager.ensure(key, name="s1", settings=settings, runner=_manager_runner) + assert first is again + assert first.alive() + finally: + manager.kill_all() + + assert manager.get(key) is None + + +def test_manager_evicts_least_recently_used() -> None: + settings = Settings(browser_bind_host="127.0.0.1", browser_max_sessions=1) + manager = service.BrowserManager() + first_key = (1, 1, "one") + second_key = (1, 1, "two") + try: + manager.ensure(first_key, name="one", settings=settings, runner=_manager_runner) + manager.ensure(second_key, name="two", settings=settings, runner=_manager_runner) + assert manager.get(first_key) is None + assert manager.get(second_key) is not None + finally: + manager.kill_all() + + +def test_manager_reaps_idle_session() -> None: + settings = Settings(browser_bind_host="127.0.0.1") + manager = service.BrowserManager() + key = (1, 1, uuid4().hex) + try: + session = manager.ensure(key, name="s1", settings=settings, runner=_manager_runner) + session.last_seen = time.monotonic() - 3600 + manager.reap_idle(ttl_seconds=60) + assert manager.get(key) is None + finally: + manager.kill_all() + + +def test_manager_isolates_sessions() -> None: + settings = Settings(browser_bind_host="127.0.0.1") + manager = service.BrowserManager() + try: + first = manager.ensure( + (1, 1, "a"), name="u1-p1-a", settings=settings, runner=_manager_runner + ) + second = manager.ensure( + (1, 1, "b"), name="u1-p1-b", settings=settings, runner=_manager_runner + ) + assert first is not second + assert manager.get((1, 1, "a")) is first + assert manager.get((1, 1, "b")) is second + finally: + manager.kill_all() + + +def test_manager_start_failure_raises() -> None: + def failing_runner(command: list[str], env: dict[str, str]) -> str: + raise service.BrowserError("boom") + + manager = service.BrowserManager() + with pytest.raises(service.BrowserError): + manager.ensure( + (1, 1, "x"), + name="x", + settings=Settings(browser_bind_host="127.0.0.1"), + runner=failing_runner, + ) diff --git a/web/src/components/IdeShell.tsx b/web/src/components/IdeShell.tsx index 6eaf683..c819841 100644 --- a/web/src/components/IdeShell.tsx +++ b/web/src/components/IdeShell.tsx @@ -1,6 +1,6 @@ import { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' -import { killTerminal, type Project, type User } from '../lib/api' +import { killTerminal, closeBrowserSession, type Project, type User } from '../lib/api' import type { AgentDefinition } from '../lib/agents' import { LEFT_MAX, LEFT_MIN, RIGHT_MAX, RIGHT_MIN, type LayoutState } from '../lib/layoutStore' import { @@ -11,6 +11,7 @@ import { withOpenDiff, withChatConversation, withTabTitle, + withBrowserTab, type ActiveProjectRef, type SyncedState, type WorkspaceTab, @@ -19,6 +20,7 @@ import { Sidebar } from './ide/Sidebar' import { RightPanel } from './ide/RightPanel' import { ProjectTerminal } from './ide/Terminal' import { ChatTab } from './ide/ChatTab' +import { BrowserTab } from './ide/BrowserTab' import { FileEditor } from './ide/FileEditor' import { GitDiffView } from './ide/GitDiffView' import { TopTabs } from './ide/TopTabs' @@ -148,6 +150,14 @@ export function IdeShell({ })) } + function openBrowser() { + if (!activeProject) return + update((prev) => ({ + ...prev, + workspaces: withBrowserTab(prev.workspaces, activeProject), + })) + } + function openFile(path: string, line?: number) { if (!activeProject) return update((prev) => ({ @@ -174,15 +184,25 @@ export function IdeShell({ [update], ) + // Release whatever backend resources a closing tab owns (a PTY or a live + // browser session). Best-effort: the resource may already be gone. + function releaseTabResources(projectId: number, tab: WorkspaceTab) { + if (tab.kind === 'terminal') { + void killTerminal(projectId, tab.id).catch(() => { + /* the terminal may already be gone */ + }) + } else if (tab.kind === 'browser') { + void closeBrowserSession(projectId, tab.id).catch(() => { + /* the session may already be gone */ + }) + } + } + function closeTab(id: string) { if (activeProjectId == null) return const projectId = activeProjectId const closing = workspaces[projectId]?.tabs.find((tab) => tab.id === id) - if (closing?.kind === 'terminal') { - void killTerminal(projectId, id).catch(() => { - /* the terminal may already be gone */ - }) - } + if (closing) releaseTabResources(projectId, closing) update((prev) => ({ ...prev, workspaces: withFilteredTabs(prev.workspaces, projectId, (tab) => tab.id !== id), @@ -190,13 +210,8 @@ export function IdeShell({ } function closeProjectTabs(projectId: number) { - const removed = workspaces[projectId]?.tabs ?? [] - for (const tab of removed) { - if (tab.kind === 'terminal') { - void killTerminal(projectId, tab.id).catch(() => { - /* the terminal may already be gone */ - }) - } + for (const tab of workspaces[projectId]?.tabs ?? []) { + releaseTabResources(projectId, tab) } update((prev) => { const next = { ...prev.workspaces } @@ -213,11 +228,7 @@ export function IdeShell({ function closeWorktreeTabs(projectId: number, worktree: string) { const removed = (workspaces[projectId]?.tabs ?? []).filter((tab) => tab.worktree === worktree) for (const tab of removed) { - if (tab.kind === 'terminal') { - void killTerminal(projectId, tab.id).catch(() => { - /* the terminal may already be gone */ - }) - } + releaseTabResources(projectId, tab) } update((prev) => ({ ...prev, @@ -270,6 +281,8 @@ export function IdeShell({ /> ) : tab.kind === 'vscode' && tab.worktree ? ( + ) : tab.kind === 'browser' ? ( + ) : ( activeProject && openTerminal(activeProject, agent)} onNewChat={openChat} + onNewBrowser={openBrowser} />
diff --git a/web/src/components/ide/BrowserTab.tsx b/web/src/components/ide/BrowserTab.tsx new file mode 100644 index 0000000..152de3e --- /dev/null +++ b/web/src/components/ide/BrowserTab.tsx @@ -0,0 +1,630 @@ +import { + useCallback, + useEffect, + useRef, + useState, + type FormEvent, + type KeyboardEvent as ReactKeyboardEvent, + type PointerEvent as ReactPointerEvent, + type WheelEvent as ReactWheelEvent, +} from 'react' +import { useTranslation } from 'react-i18next' +import { + browserStreamUrl, + controlBrowser, + fetchBrowserSelection, + insertBrowserText, + navigateBrowser, + setBrowserViewport, +} from '../../lib/api' +import { ArrowLeftIcon, ArrowRightIcon, GlobeIcon, MaximizeIcon, MinimizeIcon, RefreshIcon } from './icons' + +interface FrameMessage { + data: string + metadata?: { deviceWidth?: number; deviceHeight?: number } +} + +type ConnectionState = 'connecting' | 'open' | 'closed' + +const RECONNECT_BASE_MS = 500 +const RECONNECT_MAX_MS = 15000 + +function normalizeUrl(raw: string): string { + const value = raw.trim() + if (value === '') return '' + if (/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(value)) return value + if (value.startsWith('//')) return `https:${value}` + return `http://${value}` +} + +function modifierBits(event: { + altKey: boolean + ctrlKey: boolean + metaKey: boolean + shiftKey: boolean +}): number { + return ( + (event.altKey ? 1 : 0) | + (event.ctrlKey ? 2 : 0) | + (event.metaKey ? 4 : 0) | + (event.shiftKey ? 8 : 0) + ) +} + +function mouseButton(button: number): string { + if (button === 2) return 'right' + if (button === 1) return 'middle' + return 'left' +} + +export function BrowserTab({ + projectId, + sessionId, +}: { + projectId: number + sessionId: string +}) { + const { t } = useTranslation() + const canvasRef = useRef(null) + const containerRef = useRef(null) + const socketRef = useRef(null) + const inputRef = useRef(null) + const imageRef = useRef(null) + const pendingRef = useRef(null) + const loadingRef = useRef(false) + const loadNextRef = useRef<() => void>(() => {}) + const fitCanvasRef = useRef<() => void>(() => {}) + const fatalRef = useRef(false) + const moveFrameRef = useRef(null) + const movePointRef = useRef<{ x: number; y: number } | null>(null) + + const [connection, setConnection] = useState('connecting') + const [address, setAddress] = useState('') + const [error, setError] = useState(null) + const [reconnectNonce, setReconnectNonce] = useState(0) + const [isFullscreen, setIsFullscreen] = useState(false) + const [caret, setCaret] = useState<{ left: number; top: number }>({ left: 0, top: 0 }) + + const send = useCallback((payload: unknown) => { + const socket = socketRef.current + if (socket && socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify(payload)) + }, []) + + const drawFrame = useCallback((image: HTMLImageElement, frame: FrameMessage) => { + const canvas = canvasRef.current + if (!canvas) return + const width = frame.metadata?.deviceWidth ?? image.naturalWidth + const height = frame.metadata?.deviceHeight ?? image.naturalHeight + if (width > 0 && height > 0 && (canvas.width !== width || canvas.height !== height)) { + canvas.width = width + canvas.height = height + } + const context = canvas.getContext('2d') + if (!context) return + context.drawImage(image, 0, 0, canvas.width, canvas.height) + fitCanvasRef.current() + }, []) + + const loadNext = useCallback(() => { + const frame = pendingRef.current + if (!frame) return + pendingRef.current = null + loadingRef.current = true + const image = imageRef.current ?? new Image() + imageRef.current = image + image.onload = () => { + drawFrame(image, frame) + loadingRef.current = false + if (pendingRef.current) loadNextRef.current() + } + image.onerror = () => { + loadingRef.current = false + if (pendingRef.current) loadNextRef.current() + } + image.src = `data:image/jpeg;base64,${frame.data}` + }, [drawFrame]) + + useEffect(() => { + loadNextRef.current = loadNext + }, [loadNext]) + + const fitCanvas = useCallback(() => { + const canvas = canvasRef.current + const container = containerRef.current + if (!canvas || !container || canvas.width === 0 || canvas.height === 0) return + // Size the element to the contained image so the element box always equals + // the drawn frame; pointer mapping then needs no letterbox math. + const scale = Math.min( + container.clientWidth / canvas.width, + container.clientHeight / canvas.height, + ) + canvas.style.width = `${Math.round(canvas.width * scale)}px` + canvas.style.height = `${Math.round(canvas.height * scale)}px` + }, []) + + useEffect(() => { + fitCanvasRef.current = fitCanvas + }, [fitCanvas]) + + useEffect(() => { + const container = containerRef.current + if (!container) return + // Resize the remote viewport to match the pane so the page reflows to fill + // it: the canvas then renders 1:1 (no letterbox) and pointer input maps + // exactly. Debounced because each change is a remote command. + let timer: number | null = null + let lastWidth = 0 + let lastHeight = 0 + const observer = new ResizeObserver(() => { + fitCanvasRef.current() + const width = Math.round(container.clientWidth) + const height = Math.round(container.clientHeight) + if (width < 200 || height < 200) return + if (width === lastWidth && height === lastHeight) return + if (timer !== null) window.clearTimeout(timer) + timer = window.setTimeout(() => { + lastWidth = width + lastHeight = height + void setBrowserViewport(projectId, sessionId, width, height).catch(() => undefined) + }, 300) + }) + observer.observe(container) + return () => { + observer.disconnect() + if (timer !== null) window.clearTimeout(timer) + } + }, [projectId, sessionId]) + + useEffect(() => { + let cancelled = false + let socket: WebSocket | null = null + let reconnectTimer: number | null = null + let attempts = 0 + + function handleMessage(raw: string) { + let message: unknown + try { + message = JSON.parse(raw) + } catch { + return + } + if (typeof message !== 'object' || message === null) return + const data = message as Record + if (data.type === 'frame' && typeof data.data === 'string') { + setConnection('open') + setError(null) + pendingRef.current = { + data: data.data, + metadata: data.metadata as FrameMessage['metadata'], + } + if (!loadingRef.current) loadNextRef.current() + } else if (data.type === 'status') { + // The stream is up even before the first frame (blank page, no change), + // so treat status as connected instead of stalling on "connecting". + setConnection('open') + } else if (data.type === 'url' && typeof data.url === 'string') { + setAddress(data.url) + } else if (data.type === 'error') { + fatalRef.current = true + setError( + typeof data.message === 'string' && data.message ? data.message : t('apiErrors.unknown'), + ) + setConnection('closed') + } + } + + function open() { + if (cancelled) return + setConnection('connecting') + const ws = new WebSocket(browserStreamUrl(projectId, sessionId)) + socket = ws + socketRef.current = ws + + ws.onopen = () => { + attempts = 0 + setConnection('connecting') + setError(null) + } + ws.onmessage = (event) => { + if (typeof event.data === 'string') { + handleMessage(event.data) + return + } + // The stream protocol is JSON text; decode unexpected binary frames so + // a Blob never silently stalls the render loop. + const blob = event.data as Blob + void blob + .text() + .then(handleMessage) + .catch(() => undefined) + } + ws.onclose = (event) => { + if (cancelled) return + if (socketRef.current === ws) socketRef.current = null + if ( + fatalRef.current || + event.code === 4401 || + event.code === 4403 || + event.code === 4404 + ) { + setConnection('closed') + return + } + setConnection('connecting') + attempts += 1 + const delay = Math.min(RECONNECT_BASE_MS * 2 ** (attempts - 1), RECONNECT_MAX_MS) + reconnectTimer = window.setTimeout(open, delay) + } + } + + open() + return () => { + cancelled = true + if (reconnectTimer !== null) window.clearTimeout(reconnectTimer) + if (moveFrameRef.current !== null) window.cancelAnimationFrame(moveFrameRef.current) + if (socket && socketRef.current === socket) socketRef.current = null + socket?.close() + } + }, [projectId, sessionId, reconnectNonce, t]) + + useEffect(() => { + function onChange() { + setIsFullscreen(document.fullscreenElement === containerRef.current) + fitCanvasRef.current() + } + document.addEventListener('fullscreenchange', onChange) + return () => document.removeEventListener('fullscreenchange', onChange) + }, []) + + function toggleFullscreen() { + if (document.fullscreenElement) { + void document.exitFullscreen() + } else { + void containerRef.current?.requestFullscreen() + } + } + + function targetPoint(event: { clientX: number; clientY: number }) { + const canvas = canvasRef.current + if (!canvas || canvas.width === 0 || canvas.height === 0) return { x: 0, y: 0 } + const rect = canvas.getBoundingClientRect() + if (rect.width === 0 || rect.height === 0) return { x: 0, y: 0 } + return { + x: Math.round(((event.clientX - rect.left) / rect.width) * canvas.width), + y: Math.round(((event.clientY - rect.top) / rect.height) * canvas.height), + } + } + + function onPointerMove(event: ReactPointerEvent) { + const { x, y } = targetPoint(event) + // Coalesce moves to one event per animation frame so a fast pointer does + // not flood the stream with hundreds of messages per second. + movePointRef.current = { x, y } + if (moveFrameRef.current !== null) return + moveFrameRef.current = window.requestAnimationFrame(() => { + moveFrameRef.current = null + const point = movePointRef.current + if (point) send({ type: 'input_mouse', eventType: 'mouseMoved', x: point.x, y: point.y }) + }) + } + + function onPointerDown(event: ReactPointerEvent) { + event.preventDefault() + const { x, y } = targetPoint(event) + const canvas = canvasRef.current + const rect = canvas?.getBoundingClientRect() + const containerRect = containerRef.current?.getBoundingClientRect() + // Park the hidden input at the click point so IME candidates appear there, + // then focus it: it is the sink for typed/pasted/composed text. + if (rect && containerRect && canvas) { + setCaret({ + left: rect.left - containerRect.left + (x / canvas.width) * rect.width, + top: rect.top - containerRect.top + (y / canvas.height) * rect.height, + }) + } + inputRef.current?.focus() + const button = mouseButton(event.button) + // Move first so the server sees the cursor position before the press. + send({ type: 'input_mouse', eventType: 'mouseMoved', x, y }) + send({ type: 'input_mouse', eventType: 'mousePressed', x, y, button, clickCount: 1 }) + } + + function onPointerUp(event: ReactPointerEvent) { + const { x, y } = targetPoint(event) + send({ + type: 'input_mouse', + eventType: 'mouseReleased', + x, + y, + button: mouseButton(event.button), + }) + } + + function onWheel(event: ReactWheelEvent) { + if (event.ctrlKey || event.metaKey) { + // Ctrl/Cmd + wheel zooms, like a real browser. + event.preventDefault() + const zoomIn = event.deltaY < 0 + const key = zoomIn ? '+' : '-' + const code = zoomIn ? 'Equal' : 'Minus' + const virtualKeyCode = zoomIn ? 187 : 189 + for (const eventType of ['keyDown', 'keyUp'] as const) { + send({ + type: 'input_keyboard', + eventType, + key, + code, + modifiers: 2, + windowsVirtualKeyCode: virtualKeyCode, + nativeVirtualKeyCode: virtualKeyCode, + }) + } + return + } + const { x, y } = targetPoint(event) + send({ + type: 'input_mouse', + eventType: 'mouseWheel', + x, + y, + deltaX: event.deltaX, + deltaY: event.deltaY, + }) + } + + function onKeyDown(event: ReactKeyboardEvent) { + const native = event.nativeEvent + if (native.isComposing || event.keyCode === 229) return + if (event.ctrlKey && event.altKey) { + event.currentTarget.blur() + return + } + const accel = event.ctrlKey || event.metaKey + if (accel && !event.altKey && (event.key === 'v' || event.key === 'V')) { + // Let the browser perform the paste into the hidden input; onInput then + // forwards the pasted text. This avoids clipboard read permission. + return + } + if (event.key.length === 1 && !accel && !event.altKey) { + // Printable text flows through the hidden input's onInput so IME and + // typematic input share one path. + return + } + event.preventDefault() + if (accel && !event.altKey && 'cCxX'.includes(event.key)) { + // Read the selection before the remote acts, so a cut still reaches the + // user's clipboard; then forward the shortcut for the remote's own copy. + void copyOrCut(event.key === 'x' || event.key === 'X') + return + } + // Non-text keys (Backspace, Delete, arrows, Enter, ...) need the virtual + // key code; without it the remote browser ignores them. Enter also needs + // its carriage-return text so textareas/forms react. + const virtualKeyCode = event.keyCode + send({ + type: 'input_keyboard', + eventType: 'keyDown', + key: event.key, + code: event.code, + modifiers: modifierBits(event), + windowsVirtualKeyCode: virtualKeyCode, + nativeVirtualKeyCode: virtualKeyCode, + ...(event.key === 'Enter' ? { text: '\r' } : {}), + }) + } + + function onInput(event: React.FormEvent) { + const element = event.currentTarget + const native = event.nativeEvent as InputEvent + if (native.isComposing) return + const value = element.value + if (value === '') return + element.value = '' + if (value.length === 1) { + send({ type: 'input_keyboard', eventType: 'char', text: value }) + } else { + void insertBrowserText(projectId, sessionId, value).catch(() => undefined) + } + } + + function onCompositionEnd(event: React.CompositionEvent) { + // Guarantees an IME commit is forwarded even if the trailing `input` event + // never arrives; whichever runs first clears the value so the other no-ops. + const element = event.currentTarget + const value = element.value + if (value === '') return + element.value = '' + void insertBrowserText(projectId, sessionId, value).catch(() => undefined) + } + + function onKeyUp(event: ReactKeyboardEvent) { + if (event.nativeEvent.isComposing) return + if (event.key.length === 1 && !event.ctrlKey && !event.altKey && !event.metaKey) return + const virtualKeyCode = event.keyCode + send({ + type: 'input_keyboard', + eventType: 'keyUp', + key: event.key, + code: event.code, + modifiers: modifierBits(event), + windowsVirtualKeyCode: virtualKeyCode, + nativeVirtualKeyCode: virtualKeyCode, + }) + } + + async function submitAddress(event: FormEvent) { + event.preventDefault() + const next = normalizeUrl(address) + if (next === '') return + try { + await navigateBrowser(projectId, sessionId, next) + inputRef.current?.focus() + } catch (err) { + setError(err instanceof Error ? err.message : t('apiErrors.unknown')) + } + } + + async function control(action: 'back' | 'forward' | 'reload') { + try { + await controlBrowser(projectId, sessionId, action) + } catch (err) { + setError(err instanceof Error ? err.message : t('apiErrors.unknown')) + } + } + + async function copyOrCut(cut: boolean) { + try { + const text = await fetchBrowserSelection(projectId, sessionId) + if (text) await navigator.clipboard.writeText(text) + } catch { + /* clipboard access can be denied; the remote still copies below */ + } + const virtualKeyCode = cut ? 88 : 67 + for (const eventType of ['keyDown', 'keyUp'] as const) { + send({ + type: 'input_keyboard', + eventType, + key: cut ? 'x' : 'c', + code: cut ? 'KeyX' : 'KeyC', + modifiers: 2, + windowsVirtualKeyCode: virtualKeyCode, + nativeVirtualKeyCode: virtualKeyCode, + }) + } + } + + const toolbarButton = + 'shrink-0 rounded-md p-1.5 text-[var(--muted)] transition hover:bg-[var(--hover)] hover:text-[var(--fg-strong)] disabled:opacity-40 disabled:hover:bg-transparent' + + return ( +
+
+ + + + +
+ + setAddress(event.target.value)} + placeholder={t('ide.browserPlaceholder')} + aria-label={t('ide.browserAddress')} + spellCheck={false} + className="min-w-0 flex-1 bg-transparent font-mono text-[12px] text-[var(--fg-2)] outline-none placeholder:text-[var(--muted-3)]" + /> +
+ +
+ + {error && ( +

+ {error} +

+ )} + +
+ event.preventDefault()} + className="block cursor-default outline-none" + /> +