diff --git a/fixos/cli/_cleanup_docker_volumes.py b/fixos/cli/_cleanup_docker_volumes.py new file mode 100644 index 0000000..a48cede --- /dev/null +++ b/fixos/cli/_cleanup_docker_volumes.py @@ -0,0 +1,202 @@ +"""Selective removal of unused (dangling) Docker volumes. + +Volumes hold real data, so nothing here is bulk: candidates are listed with +name, Compose project and age, the user picks them, and each one is checked +again right before ``docker volume rm``. ``--yes`` never selects volumes whose +name looks like stored data (databases, git mirrors, backups, ...). +""" + +from __future__ import annotations + +import json +import re +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +import click + +from fixos.cli import _cleanup_space as space +from fixos.cli._cleanup_utils import _format_bytes + +# Name tokens that usually mean irreplaceable data rather than a cache. +_DATA_TOKENS = re.compile( + r"(?:^|[_.-])(mirror|git|gitea|onedev|repo|repos|postgres|postgresql|pg|" + r"mysql|mariadb|mongo|mongodb|redis|db|database|backup|backups|minio|s3|" + r"vault|secrets?|keycloak|grafana|prometheus|influx|elastic|opensearch|" + r"nextcloud|uploads|media|webui|openwebui|open-webui|n8n|storage|certs|" + r"letsencrypt|wordpress|mattermost|synapse|chat-history)(?:$|[_.-])", + re.IGNORECASE, +) +_INSPECT_BATCH = 100 + + +def _docker_lines(args: list[str], timeout: int = 30) -> list[str] | None: + try: + result = space._run_cmd(["docker", *args], timeout=timeout) + except (OSError, subprocess.TimeoutExpired): + return None + if result.returncode != 0: + return None + return [line for line in result.stdout.splitlines() if line.strip()] + + +def _is_unused(name: str) -> bool: + """Re-check right before removal: still dangling and no container (even stopped).""" + dangling = _docker_lines(["volume", "ls", "-q", "-f", "dangling=true", "-f", f"name=^{name}$"]) + containers = _docker_lines(["ps", "-aq", "--filter", f"volume={name}"]) + return dangling is not None and name in dangling and containers == [] + + +def _age_days(created_at: str, now: datetime) -> float | None: + try: + created = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + except ValueError: + return None + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + return (now - created).total_seconds() / 86400 + + +def _volume_sizes(mountpoints: list[str]) -> dict[str, int]: + """Sizes via ``du``; volumes are root-owned, so fall back to ``sudo -n``. + + ``sudo -n`` never prompts: without passwordless sudo the size stays unknown. + """ + sizes = space._dir_sizes([Path(m) for m in mountpoints]) + missing = [m for m in mountpoints if m not in sizes] + if missing: + try: + result = space._run_cmd(["sudo", "-n", "du", "-sbl0", "--", *missing], timeout=300) + for record in result.stdout.split("\0"): + size, sep, path = record.partition("\t") + if sep and size.isdigit(): + sizes[path] = int(size) + except (OSError, subprocess.TimeoutExpired): + pass + return sizes + + +def _unused_volumes( + days: float | None, match: list[str], now: datetime | None = None +) -> list[dict] | None: + names = _docker_lines(["volume", "ls", "-q", "-f", "dangling=true"]) + if names is None: + return None + now = now or datetime.now(timezone.utc) + records: list[dict] = [] + for start in range(0, len(names), _INSPECT_BATCH): + raw = _docker_lines(["volume", "inspect", *names[start : start + _INSPECT_BATCH]], timeout=60) + if raw is None: + return None + records.extend(json.loads("\n".join(raw))) + candidates = [] + for record in records: + name = record.get("Name", "") + if match and not any(token.lower() in name.lower() for token in match): + continue + age = _age_days(record.get("CreatedAt", ""), now) + if days is not None and (age is None or age < days): + continue + labels = record.get("Labels") or {} + project = labels.get("com.docker.compose.project") or "(bez projektu)" + protected = bool(_DATA_TOKENS.search(name)) + candidates.append( + { + "name": name, + "description": name, + "project": project, + "age_days": age, + "mountpoint": record.get("Mountpoint", ""), + "protected": protected, + "size_bytes": 0, + "size_known": False, + } + ) + sizes = _volume_sizes([c["mountpoint"] for c in candidates if c["mountpoint"]]) + for candidate in candidates: + if candidate["mountpoint"] in sizes: + candidate["size_bytes"] = sizes[candidate["mountpoint"]] + candidate["size_known"] = True + age = candidate["age_days"] + size = _format_bytes(candidate["size_bytes"]) if candidate["size_known"] else "rozmiar nieznany" + candidate["note"] = ( + f"{candidate['project']}, {age:.0f} dni" if age is not None else candidate["project"] + ) + (" — wygląda na DANE, tylko ręczny wybór" if candidate["protected"] else "") + candidate["size_label"] = size + candidates.sort(key=lambda c: (c["size_known"], c["size_bytes"]), reverse=True) + return candidates + + +def _cleanup_docker_volumes( + json_output: bool, + dry_run: bool, + list_only: bool, + yes: bool, + days: float | None = None, + match: list[str] | None = None, +) -> None: + """List unused Docker volumes and remove the selected ones.""" + candidates = _unused_volumes(days, match or []) + if json_output: + space._echo_json( + { + "error": None if candidates is not None else "docker volume ls failed", + "candidates": [ + {k: c[k] for k in ("name", "project", "age_days", "size_bytes", "size_known", "protected")} + for c in candidates or [] + ], + } + ) + return + if candidates is None: + click.echo(click.style("Błąd: nie udało się odczytać wolumenów Dockera.", fg="red")) + return + if not candidates: + click.echo(click.style("Brak nieużywanych wolumenów Dockera dla podanych filtrów.", fg="green")) + return + known = sum(c["size_bytes"] for c in candidates if c["size_known"]) + click.echo( + f"\n{click.style('DOCKER — nieużywane wolumeny (dane!):', fg='magenta', bold=True)} " + f"{len(candidates)}, znany rozmiar {_format_bytes(known)}" + ) + for index, item in enumerate(candidates, 1): + color = "red" if item["protected"] else "yellow" + click.echo( + f" [{index:3d}] {click.style(item['name'], fg=color)}: {item['size_label']} — {item['note']}" + ) + if list_only: + return + if yes: + selected = [i for i, c in enumerate(candidates) if not c["protected"]] + skipped = len(candidates) - len(selected) + if skipped: + click.echo(f" --yes pomija {skipped} wolumenów wyglądających na dane; wybierz je ręcznie.") + else: + selected = space._choose(candidates, "Numery wolumenów do usunięcia (np. 1,2, all)") + if not selected: + click.echo("Pominięto.") + return + if dry_run: + space._show_dry_run_commands([["docker", "volume", "rm", candidates[i]["name"]] for i in selected]) + return + if not yes and not space._confirm_remove(): + click.echo("Anulowano.") + return + freed = 0 + for index in selected: + name = candidates[index]["name"] + if not _is_unused(name): + click.echo(f" ⏭️ {name}: jest już używany albo nie istnieje — pominięto") + continue + try: + result = space._run_cmd(["docker", "volume", "rm", name], timeout=120) + except (OSError, subprocess.TimeoutExpired) as exc: + click.echo(click.style(f" ❌ {name}: {exc}", fg="red")) + continue + if result.returncode == 0: + freed += candidates[index]["size_bytes"] + click.echo(click.style(f" ✅ {name}", fg="green")) + else: + click.echo(click.style(f" ❌ {name}: {result.stderr.strip()[:120]}", fg="red")) + click.echo(f"Usunięto wolumeny o znanym rozmiarze łącznie {_format_bytes(freed)}.") diff --git a/fixos/cli/cleanup_cmd.py b/fixos/cli/cleanup_cmd.py index a485033..de5dfcf 100644 --- a/fixos/cli/cleanup_cmd.py +++ b/fixos/cli/cleanup_cmd.py @@ -14,6 +14,8 @@ import click +from fixos.cli._cleanup_docker_volumes import _cleanup_docker_volumes + # Re-export public symbols used by fixos.cli (backward-compat) from fixos.cli._cleanup_flatpak import _cleanup_flatpak_detailed from fixos.cli._cleanup_space import ( @@ -1611,7 +1613,7 @@ def _execute_yes_cleanup(plan: dict, scanner, dry_run: bool = False) -> None: "Wyczyść konkretną usługę " "(docker, docker-all, docker-old, docker-containers, docker-networks, " "docker-stale-services, orphaned-projects, ollama-old, snap-old, " - "journal, user-cache, jetbrains, libvirt, gitive, docker-buildcache, tmp, venvs-old, " + "journal, user-cache, jetbrains, libvirt, gitive, docker-buildcache, docker-volumes, tmp, venvs-old, " "npm, ...)" ), ) @@ -1719,6 +1721,23 @@ def _execute_yes_cleanup(plan: dict, scanner, dry_run: bool = False) -> None: f"(domyślnie {DEFAULT_OLLAMA_OLD_UNUSED_DAYS}); pomija modele aktualnie uruchomione" ), ) +@click.option( + "--docker-volumes", + "docker_volumes", + is_flag=True, + default=False, + help=( + "Wybierz i usuń nieużywane wolumeny Dockera (nazwa, projekt Compose, wiek, " + "rozmiar przez du/sudo -n); filtry --days i --match; --yes pomija " + "wolumeny wyglądające na dane" + ), +) +@click.option( + "--match", + "match", + default=None, + help="Dla --docker-volumes: fragmenty nazw rozdzielone przecinkami, np. ollama,huggingface", +) @click.option( "--snap-old", "snap_old", @@ -1893,6 +1912,8 @@ def cleanup_services( list_orphan_pins, process_hours, ollama_old, + docker_volumes, + match, snap_old, journal_cleanup, user_cache, @@ -1951,6 +1972,8 @@ def cleanup_services( fixos cleanup --unpin-orphan-project /absolute/missing/project # usuwa ochronę, ale nie zmienia Dockera ani procesów fixos cleanup --ollama-old --days 90 --dry-run + fixos cleanup --docker-volumes --list # nieużywane wolumeny: nazwa, projekt, wiek, rozmiar + fixos cleanup --docker-volumes --match ollama,huggingface --days 90 # modele Ollama niezmieniane od 90+ dni fixos cleanup -c ollama-old fixos cleanup --snap-old # wyłączone rewizje snapów @@ -2004,6 +2027,7 @@ def cleanup_services( list_orphan_pins, ollama_old, snap_old, + docker_volumes, journal_cleanup, user_cache, jetbrains, @@ -2053,6 +2077,7 @@ def cleanup_services( ollama_old, docker_buildcache, snap_old, + docker_volumes, journal_cleanup, user_cache, jetbrains, @@ -2106,6 +2131,17 @@ def cleanup_services( "libvirt": (libvirt, _cleanup_libvirt), "gitive": (gitive, _cleanup_gitive), "docker-buildcache": (docker_buildcache, _cleanup_docker_buildcache), + "docker-volumes": ( + docker_volumes, + lambda j, d, lst, y: _cleanup_docker_volumes( + j, + d, + lst, + y, + days=effective_days, + match=[t.strip() for t in (match or "").split(",") if t.strip()], + ), + ), "tmp": ( tmp_cleanup, lambda j, d, lst, y: _cleanup_tmp( @@ -2168,7 +2204,7 @@ def cleanup_services( click.echo( click.style( "Akcje odzyskiwania miejsca (snap/journal/cache/JetBrains/" - "libvirt/gitive/docker-buildcache/tmp/trash/thumbnails) uruchom jako osobną, pojedynczą akcję.", + "libvirt/gitive/docker-buildcache/docker-volumes/tmp/trash/thumbnails) uruchom jako osobną, pojedynczą akcję.", fg="red", ) ) diff --git a/fixos/diagnostics/_storage_container_mixin.py b/fixos/diagnostics/_storage_container_mixin.py index d451ce2..e837f92 100644 --- a/fixos/diagnostics/_storage_container_mixin.py +++ b/fixos/diagnostics/_storage_container_mixin.py @@ -3,15 +3,16 @@ from __future__ import annotations import os +from typing import ClassVar -from fixos.diagnostics.storage_analyzer import StorageItem from fixos.constants import MIN_DOCKER_DANGLING_MB, MIN_PODMAN_MB +from fixos.diagnostics.storage_analyzer import StorageItem class _ContainerAnalyzerMixin: """Mixin providing container _analyze_* methods for StorageAnalyzer.""" - _DOCKER_DF_SECTIONS = { + _DOCKER_DF_SECTIONS: ClassVar[dict[str, str]] = { "Images space usage:": "images", "Containers space usage:": "containers", "Local Volumes space usage:": "volumes", @@ -119,11 +120,14 @@ def _add_docker_total_item(self, total_images: int, dangling_images: int) -> Non size_bytes=total_docker, category="containers", risk="medium", - cleanup_command="docker system prune -a --volumes", + # Never a bulk volume prune: volumes are data and are + # chosen one by one with `fixos cleanup --docker-volumes`. + cleanup_command="docker builder prune --force --filter until=168h", description=( f"Docker łącznie: {StorageItem._format_size(total_docker)}. " f"Obrazy: {total_images}, dangling: {dangling_images}. " - "Uwaga: usunie wszystkie nieużywane zasoby." + "Czyści cache buildów starszy niż 7 dni; wolumeny wybierz " + "ręcznie przez `fixos cleanup --docker-volumes`." ), ) ) diff --git a/project/ticket-081/README.md b/project/ticket-081/README.md new file mode 100644 index 0000000..9e45a28 --- /dev/null +++ b/project/ticket-081/README.md @@ -0,0 +1,42 @@ +# Ticket 081: fixos cleanup --docker-volumes: select and remove unused Docker volumes safely + +- **ID**: ticket-081 +- **Owner**: claude +- **Status**: IN_PROGRESS +- **Workflow state**: EDIT +- **Created**: 2026-10-13 + +## Goal and scope + +On 2026-10-13 the host had 397 unused Docker volumes (89 GB). fixos only +counted them; removing the 18 model volumes (55.5 GB) needed manual +`docker volume rm`. The only volume path in fixos was `cleanup --full` +"Docker Total" → `docker system prune -a --volumes`, which deletes every +unused volume at once, including data such as git mirrors. + +`fixos cleanup --docker-volumes` (also `-c docker-volumes`): + +- lists unused volumes with name, Compose project, age and size (`du`, then + non-interactive `sudo -n du`; otherwise "rozmiar nieznany"), largest first; +- `--match ollama,huggingface` and `--days N` filters; `--list`, `--dry-run`, + `--json`; +- removes only selected volumes, re-checking each is still dangling and not + referenced by any container right before `docker volume rm`; +- `--yes` never selects names that look like data (mirror, git, db, postgres, + backup, Open WebUI, n8n, storage, ...); those need an explicit choice. A live + dry-run showed `--yes` would otherwise have removed three Open WebUI volumes. + +`--full` "Docker Total" now runs the bounded build-cache prune and points to +`--docker-volumes` for volumes. + +SESSION_EXECUTION_AUTHORIZATION: user answered "tak" to adding this command to fixos. + +## Acceptance criteria + +- [x] AC-01: tests cover listing/sorting, filters, `--yes` protection, removal-time re-check, dry-run/list and the `--full` command. +- [x] AC-02: full suite, ruff on changed files and `./project/governance-check.sh` pass. + +## Tracking boundary + +This directory contains the minimal reviewed intent. Optional participant prose +and raw command logs are not required delivery output. diff --git a/project/ticket-081/intent.json b/project/ticket-081/intent.json new file mode 100644 index 0000000..601ff26 --- /dev/null +++ b/project/ticket-081/intent.json @@ -0,0 +1,88 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-081", + "summary": "fixos cleanup --docker-volumes: select and remove unused Docker volumes safely", + "workstream": "application", + "classification": { + "kind": "FEATURE", + "priority": "P2", + "origin": "requested" + }, + "allowedPaths": [ + "project/ticket-081/**", + "TODO.md", + "project/TICKETS.md", + "fixos/cli/_cleanup_docker_volumes.py", + "fixos/cli/cleanup_cmd.py", + "fixos/diagnostics/_storage_container_mixin.py", + "tests/unit/test_cleanup_docker_volumes.py" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [ + "python" + ], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null, + "delivery": { + "acceptedBaseSha": "d0d37038727598ab0234d06de40952c3114e9cd2", + "targetBranch": "main", + "outcome": "fixos cleanup --docker-volumes lists unused Docker volumes with name, Compose project, age and size (du, then sudo -n du), filters by --match/--days, removes only selected volumes after re-checking they are still unused, and --yes never selects data-looking volumes; --full no longer runs docker system prune -a --volumes.", + "nonGoals": [ + "No automatic removal from the default cleanup plan.", + "No volume size via docker system df." + ], + "complexity": "M", + "estimatedMinutes": 90, + "budgets": { + "maxImplementationFiles": 4, + "maxAffectedComponents": 1, + "maxPublicInterfaceChanges": 0, + "maxRuntimeDependencies": 0 + }, + "architecture": { + "status": "accepted", + "decision": "New space action module following the gitive/tmp pattern (list, choose, dry-run, confirm), wired as --docker-volumes/-c docker-volumes with --match; full analysis uses the bounded builder prune instead of a bulk volume prune.", + "components": [ + { + "name": "docker-volumes", + "paths": [ + "fixos/cli/_cleanup_docker_volumes.py", + "fixos/cli/cleanup_cmd.py", + "fixos/diagnostics/_storage_container_mixin.py", + "tests/unit/test_cleanup_docker_volumes.py" + ] + } + ], + "responsibilityChanges": false, + "interfaceChanges": [], + "dataChanges": [], + "ui": { + "impact": "none", + "states": [], + "evidence": [] + }, + "rollback": "Revert the commit." + }, + "runtimeDependencies": [], + "validation": [ + { + "criterion": "AC-01", + "commands": [ + "pytest -q tests/unit/test_cleanup_docker_volumes.py" + ], + "evidence": "project/ticket-081/README.md" + }, + { + "criterion": "AC-02", + "commands": [ + "pytest -q", + "./project/governance-check.sh" + ], + "evidence": "project/ticket-081/README.md" + } + ] + } +} diff --git a/tests/unit/test_cleanup_docker_volumes.py b/tests/unit/test_cleanup_docker_volumes.py new file mode 100644 index 0000000..da57b57 --- /dev/null +++ b/tests/unit/test_cleanup_docker_volumes.py @@ -0,0 +1,143 @@ +"""fixos cleanup --docker-volumes: listing, selection safety and re-check.""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from types import SimpleNamespace + +import click +from click.testing import CliRunner + +from fixos.cli import _cleanup_docker_volumes as volumes +from fixos.cli import _cleanup_space as space +from fixos.diagnostics.storage_analyzer import ( + StorageAnalyzer, # loads the mixins in order +) + +NOW = datetime(2026, 10, 13, tzinfo=timezone.utc) +INSPECT = [ + {"Name": "llmass_huggingface-cache", "CreatedAt": "2025-10-06T10:00:00Z", + "Labels": {"com.docker.compose.project": "llmass"}, "Mountpoint": "/v/hf"}, + {"Name": "ifuri-onedev_mirror-data", "CreatedAt": "2026-07-20T10:00:00Z", + "Labels": {"com.docker.compose.project": "ifuri-onedev"}, "Mountpoint": "/v/mirror"}, + {"Name": "chat_ollama_data", "CreatedAt": "2026-10-10T10:00:00Z", + "Labels": None, "Mountpoint": "/v/chat"}, +] + + +class FakeDocker: + """Answers the docker/du/sudo commands the module issues.""" + + def __init__(self, dangling=None, in_use=()): + self.dangling = list(dangling or [v["Name"] for v in INSPECT]) + self.in_use = set(in_use) + self.calls: list[list[str]] = [] + + def __call__(self, argv, timeout=60): + self.calls.append(argv) + out, code = "", 0 + if argv[:5] == ["docker", "volume", "ls", "-q", "-f"]: + names = self.dangling + if len(argv) > 6: + wanted = argv[-1].split("=", 1)[1].strip("^$") + names = [n for n in names if n == wanted] + out = "\n".join(names) + elif argv[:3] == ["docker", "volume", "inspect"]: + out = json.dumps([v for v in INSPECT if v["Name"] in argv[3:]]) + elif argv[:3] == ["docker", "ps", "-aq"]: + name = argv[-1].split("=", 1)[1] + out = "c1" if name in self.in_use else "" + elif argv[:2] == ["du", "-sbl0"] or argv[:3] == ["sudo", "-n", "du"]: + sizes = {"/v/hf": 23_500_000_000, "/v/mirror": 7_400_000_000} + out = "".join(f"{sizes[p]}\t{p}\0" for p in argv if p in sizes) + elif argv[:3] == ["docker", "volume", "rm"]: + self.dangling.remove(argv[3]) + return SimpleNamespace(returncode=code, stdout=out, stderr="") + + +def _invoke(fn, *args, input_text="", **kwargs): + @click.command() + def command(): + fn(*args, **kwargs) + + return CliRunner().invoke(command, input=input_text) + + +def test_lists_name_project_age_and_sorts_by_size(monkeypatch): + monkeypatch.setattr(space, "_run_cmd", FakeDocker()) + + found = volumes._unused_volumes(None, [], now=NOW) + + assert [c["name"] for c in found] == [ + "llmass_huggingface-cache", "ifuri-onedev_mirror-data", "chat_ollama_data", + ] + assert found[0]["project"] == "llmass" and round(found[0]["age_days"]) == 372 + assert found[2]["size_known"] is False + assert [c["protected"] for c in found] == [False, True, False] + + +def test_match_and_days_filter(monkeypatch): + monkeypatch.setattr(space, "_run_cmd", FakeDocker()) + + found = volumes._unused_volumes(30, ["ollama", "huggingface"], now=NOW) + + assert [c["name"] for c in found] == ["llmass_huggingface-cache"] + + +def test_yes_never_removes_data_looking_volumes(monkeypatch): + docker = FakeDocker() + monkeypatch.setattr(space, "_run_cmd", docker) + + result = _invoke(volumes._cleanup_docker_volumes, False, False, False, True) + + removed = [c[3] for c in docker.calls if c[:3] == ["docker", "volume", "rm"]] + assert result.exit_code == 0 + assert "ifuri-onedev_mirror-data" not in removed + assert set(removed) == {"llmass_huggingface-cache", "chat_ollama_data"} + + +def test_volume_in_use_at_removal_time_is_skipped(monkeypatch): + docker = FakeDocker(in_use={"chat_ollama_data"}) + monkeypatch.setattr(space, "_run_cmd", docker) + + _invoke(volumes._cleanup_docker_volumes, False, False, False, False, input_text="3\ny\n") + + assert not any(c[:3] == ["docker", "volume", "rm"] for c in docker.calls) + + +def test_dry_run_and_list_do_not_remove(monkeypatch): + docker = FakeDocker() + monkeypatch.setattr(space, "_run_cmd", docker) + + _invoke(volumes._cleanup_docker_volumes, False, True, False, True) + _invoke(volumes._cleanup_docker_volumes, False, False, True, False) + + assert not any(c[:3] == ["docker", "volume", "rm"] for c in docker.calls) + + +def test_full_analysis_never_prunes_volumes(monkeypatch): + from fixos.diagnostics._storage_container_mixin import _ContainerAnalyzerMixin + + assert issubclass(StorageAnalyzer, _ContainerAnalyzerMixin) + + class Analyzer(_ContainerAnalyzerMixin): + def __init__(self): + self.items = [] + + def _get_dir_size(self, path): + return 300 * 1024**3 + + analyzer = Analyzer() + analyzer._add_docker_total_item(10, 2) + + command = analyzer.items[0].cleanup_command + assert "--volumes" not in command and "system prune" not in command + assert command == "docker builder prune --force --filter until=168h" + + +def test_app_data_volumes_are_protected(): + for name in ("mcp_openwebui-data", "proxy_open-webui-data", "n8n_storage", "mcp_git-repo-storage"): + assert volumes._DATA_TOKENS.search(name), name + for name in ("llmass_huggingface-cache", "chat_ollama_data", "if-uri_trivy-cache"): + assert not volumes._DATA_TOKENS.search(name), name