From ba01db6adffe3955c66814fd29672c453b935308 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Tue, 13 Oct 2026 06:17:55 +0200 Subject: [PATCH 1/2] perf(ticket-214): optimize directory pruning and combined regex pattern matching in file filter Co-authored-by: Koru Agent --- VERSION | 2 +- code2llm/__init__.py | 2 +- code2llm/core/analyzer.py | 6 ++++- code2llm/core/file_filter.py | 52 +++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- tests/test_edge_cases.py | 33 +++++++++++++++++++++++ 6 files changed, 87 insertions(+), 10 deletions(-) diff --git a/VERSION b/VERSION index a9ad2ac..bcfc69a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.181 +0.5.182 diff --git a/code2llm/__init__.py b/code2llm/__init__.py index 5fc109b..630d773 100644 --- a/code2llm/__init__.py +++ b/code2llm/__init__.py @@ -8,7 +8,7 @@ and entity resolution with multilingual support. """ -__version__ = "0.5.181" +__version__ = "0.5.182" __author__ = "STTS Project" # Core analysis components (lightweight, always needed) diff --git a/code2llm/core/analyzer.py b/code2llm/core/analyzer.py index 62fd141..9986219 100644 --- a/code2llm/core/analyzer.py +++ b/code2llm/core/analyzer.py @@ -320,12 +320,16 @@ def _collect_files(self, project_path: Path) -> List[Tuple[str, str]]: project_name = project_path.name # Fast path: native Rust directory traversal + skip_dirs = list( + _SKIP_DIR_NAMES + | getattr(self.file_filter, "_simple_dir_excludes", set()) + ) native_files = native_walk_project_files( project_str, extensions=list(ALL_EXTENSIONS), filenames=list(ALL_FILENAMES), filename_prefixes=list(LANGUAGE_FILENAME_PREFIXES), - skip_dirs=list(_SKIP_DIR_NAMES), + skip_dirs=skip_dirs, respect_gitignore=getattr(self.config.filters, "gitignore_enabled", True), ) if native_files is not None: diff --git a/code2llm/core/file_filter.py b/code2llm/core/file_filter.py index e835f8e..21d7134 100644 --- a/code2llm/core/file_filter.py +++ b/code2llm/core/file_filter.py @@ -66,6 +66,15 @@ ".serverless", ".netlify", ".vercel", + # Additional heavy directories and vendor bundles + "c2004", + "c2004-git-backup", + "workshop", + "managed_components", + "vendor", + ".subactor", + ".worktrees", + ".planfile", } ) | ARCHIVE_DIR_NAMES @@ -83,13 +92,31 @@ def __init__(self, config: FilterConfig, project_path: Path = None): # Split patterns into fast substring checks vs pre-compiled regex self._simple_excludes = [] self._regex_excludes = [] + self._simple_dir_excludes: set[str] = set() + wildcard_patterns = [] for p in config.exclude_patterns: p_lower = p.lower() + p_dir = p_lower.removesuffix("/**").removesuffix("/*").rstrip("/") + if p_dir and not any(c in p_dir for c in ("*", "?", "[")): + self._simple_dir_excludes.add(p_dir) + if "/" in p_dir: + self._simple_dir_excludes.add(p_dir.split("/")[-1]) if any(c in p_lower for c in ("*", "?", "[")): self._regex_excludes.append(re.compile(fnmatch.translate(p_lower))) + wildcard_patterns.append(p_lower) else: self._simple_excludes.append(p_lower) + if wildcard_patterns: + try: + self._combined_regex_excludes = re.compile( + "|".join(f"(?:{fnmatch.translate(p)})" for p in wildcard_patterns) + ) + except Exception: + self._combined_regex_excludes = None + else: + self._combined_regex_excludes = None + self._simple_includes = [] self._regex_includes = [] for p in config.include_patterns: @@ -107,7 +134,11 @@ def __init__(self, config: FilterConfig, project_path: Path = None): def should_skip_dir(self, dirname: str) -> bool: """Fast O(1) check: skip this directory entirely during tree walk?""" lower = dirname.lower() - return lower.startswith(".") or lower in _SKIP_DIR_NAMES + return ( + lower.startswith(".") + or lower in _SKIP_DIR_NAMES + or lower in self._simple_dir_excludes + ) def _passes_gitignore(self, file_path: str) -> bool: """Check if file passes gitignore patterns (True = pass, False = excluded).""" @@ -124,14 +155,23 @@ def _passes_excludes( if pattern in path_lower or (rel_lower and pattern in rel_lower): return False - # Pre-compiled wildcard excludes - for regex in self._regex_excludes: + # Fast combined regex check + if self._combined_regex_excludes is not None: if ( - regex.match(path_lower) - or regex.match(basename_lower) - or (rel_lower and regex.match(rel_lower)) + self._combined_regex_excludes.match(path_lower) + or self._combined_regex_excludes.match(basename_lower) + or (rel_lower and self._combined_regex_excludes.match(rel_lower)) ): return False + else: + # Pre-compiled wildcard excludes + for regex in self._regex_excludes: + if ( + regex.match(path_lower) + or regex.match(basename_lower) + or (rel_lower and regex.match(rel_lower)) + ): + return False return True diff --git a/pyproject.toml b/pyproject.toml index 7aaa88c..d762a0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "code2llm" -version = "0.5.181" +version = "0.5.182" description = "High-performance Python code flow analysis with optimized TOON format - CFG, DFG, call graphs, and intelligent code queries" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 635e66e..fda465b 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -238,6 +238,39 @@ def test_skip_properties(self): assert filter_obj.should_skip_function(3, is_property=True) + def test_should_skip_dir(self): + """Test directory skipping for built-in and custom exclude patterns.""" + config = FilterConfig(exclude_patterns=["custom_heavy/**", "workshop/*", "vendor_bundle/"]) + filter_obj = FastFileFilter(config) + + # Built-in heavy dirs + assert filter_obj.should_skip_dir("c2004") + assert filter_obj.should_skip_dir("c2004-git-backup") + assert filter_obj.should_skip_dir("node_modules") + assert filter_obj.should_skip_dir(".git") + assert filter_obj.should_skip_dir(".subactor") + + # Custom exclude pattern extracted dirs + assert filter_obj.should_skip_dir("custom_heavy") + assert filter_obj.should_skip_dir("workshop") + assert filter_obj.should_skip_dir("vendor_bundle") + + # Normal dirs should not be skipped + assert not filter_obj.should_skip_dir("src") + assert not filter_obj.should_skip_dir("app") + assert not filter_obj.should_skip_dir("lib") + assert not filter_obj.should_skip_dir("my_module") + + def test_combined_regex_excludes(self): + """Test combined regex exclude filter matching.""" + config = FilterConfig(exclude_patterns=["*.tmp", "*_backup.*", "legacy/**"]) + filter_obj = FastFileFilter(config) + + assert not filter_obj.should_process("temp/test.tmp") + assert not filter_obj.should_process("code_backup.py") + assert not filter_obj.should_process("legacy/old.py") + assert filter_obj.should_process("app/main.py") + class TestNLPEdgeCases: """Test NLP pipeline edge cases.""" From e43e0075c3ca5ef26575e3010d0e18d0c47aa0a9 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Tue, 13 Oct 2026 06:19:02 +0200 Subject: [PATCH 2/2] chore(ticket-214): sync uv.lock for version 0.5.182 Co-authored-by: Koru Agent --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 6e97e51..13caf46 100644 --- a/uv.lock +++ b/uv.lock @@ -649,7 +649,7 @@ source = { editable = "packages/code2graph" } [[package]] name = "code2llm" -version = "0.5.181" +version = "0.5.182" source = { editable = "." } dependencies = [ { name = "astroid" },