perf(lsh): support datasketch 2.0 native MinHash and batch Rayon signatures (ticket-016)

packages/redup-fast-hash/src/lib.rs

@@ -73,6 +73,23 @@ fn py_compute_minhash_signature(features: Vec<String>, num_perm: usize) -> Vec<u
     lsh::compute_minhash_signature(&features, num_perm)
 }
 
+/// Batch compute MinHash signatures in parallel with Rayon, releasing GIL.
+#[pyfunction]
+#[pyo3(signature = (items, num_perm = 128))]
+fn batch_minhash_signature(
+    py: Python<'_>,
+    items: Vec<Vec<String>>,
+    num_perm: usize,
+) -> Vec<Vec<u64>> {
+    use rayon::prelude::*;
+    py.allow_threads(|| {
+        items
+            .par_iter()
+            .map(|feats| lsh::compute_minhash_signature(feats, num_perm))
+            .collect()
+    })
+}
+
 /// Python wrapper for MinHash Jaccard similarity.
 #[pyfunction]
 #[pyo3(signature = (a, b))]
@@ -89,6 +106,7 @@ fn redup_fast_hash(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(batch_simhash, m)?)?;
     m.add_function(wrap_pyfunction!(py_find_simhash_candidates, m)?)?;
     m.add_function(wrap_pyfunction!(py_compute_minhash_signature, m)?)?;
+    m.add_function(wrap_pyfunction!(batch_minhash_signature, m)?)?;
     m.add_function(wrap_pyfunction!(py_minhash_jaccard, m)?)?;
 
     // Add aliases matching standard function names

project/TICKETS.md

@@ -18,4 +18,5 @@ New identities are reserved exclusively by project/new-ticket.sh.
 | **ticket-012** | [`README.md`](./ticket-012/README.md) | - | - | - | - | - |
 | **ticket-013** | [`README.md`](./ticket-013/README.md) | - | - | - | - | - |
 | **ticket-014** | [`README.md`](./ticket-014/README.md) | - | - | - | - | - |
+| **ticket-015** | [`README.md`](./ticket-015/README.md) | - | - | - | - | - |
 <!-- AUTO:TICKET_INDEX:END -->

project/ticket-016/README.md

@@ -0,0 +1,24 @@
+# Ticket 016: Native Rust MinHash scheme compatibility and batch Rayon signatures
+
+- **ID**: ticket-016
+- **Owner**: agent:antigravity
+- **Status**: IN_PROGRESS
+- **Workflow state**: EDIT
+- **Created**: 2026-10-13
+
+## Goal and scope
+
+Resolve Datasketch >=2.0 scheme compatibility for native Rust MinHash signature instantiation, introduce multi-threaded batch MinHash signatures via Rayon in `redup_fast_hash`, vectorize LSH block indexing in `LSHIndex.add_blocks`, and eliminate redundant pairwise similarity re-computations in `find_fuzzy_groups`.
+
+## Acceptance criteria
+
+- [x] AC-01: Support datasketch >=2.0 scheme specification (`scheme='legacy'`) and fallback assignment for native Rust MinHash signature instantiation.
+- [x] AC-02: Implement `batch_minhash_signature` in `redup-fast-hash` with Rayon parallelization and GIL release.
+- [x] AC-03: Implement batch indexing in `LSHIndex.add_blocks` for high-throughput parallel signature generation.
+- [x] AC-04: Cache pairwise similarity in `find_fuzzy_groups` to eliminate duplicate `sequence_similarity` re-computation.
+- [x] AC-05: Pass test suite and governance checks.
+
+## Tracking boundary
+
+This directory contains the minimal reviewed intent. Optional participant prose
+and raw command logs are not required delivery output.

project/ticket-016/intent.json

@@ -0,0 +1,26 @@
+{
+  "schema": "new-project.intent/v3",
+  "ticket": "ticket-016",
+  "summary": "Native Rust MinHash scheme compatibility and batch Rayon signatures",
+  "workstream": "application",
+  "classification": {
+    "kind": "SERVICE",
+    "priority": "P2",
+    "origin": "health"
+  },
+  "allowedPaths": [
+    "project/ticket-016/**",
+    "TODO.md",
+    "project/TICKETS.md",
+    "packages/**",
+    "src/**",
+    "tests/**"
+  ],
+  "forbiddenPaths": [
+    "project/ticket-*/user-*.md"
+  ],
+  "stacks": [],
+  "dependsOn": [],
+  "conflictsWith": [],
+  "integrationTicket": null
+}

src/redup/core/lsh_matcher.py

@@ -30,6 +30,22 @@
 )
 
 
+def _make_minhash_from_signature(signature: Any, num_perm: int) -> MinHash | None:
+    """Instantiate a datasketch MinHash from raw permutation hashvalues safely."""
+    if MinHash is None or _np is None:
+        return None
+    arr = _np.asarray(signature, dtype=_np.uint64)
+    try:
+        return MinHash(num_perm=num_perm, hashvalues=arr, scheme="legacy")
+    except (TypeError, ValueError):
+        try:
+            m = MinHash(num_perm=num_perm)
+            m.hashvalues = arr
+            return m
+        except Exception:
+            return None
+
+
 def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None:
     """Create MinHash from normalized text."""
     if MinHash is None:
@@ -47,7 +63,9 @@ def _create_minhash(text: str, num_perm: int = 128) -> MinHash | None:
     ):
         try:
             signature = _native.compute_minhash_signature(list(dict.fromkeys(tokens)), num_perm)
-            return MinHash(num_perm=num_perm, hashvalues=_np.asarray(signature, dtype=_np.uint64))
+            mh = _make_minhash_from_signature(signature, num_perm)
+            if mh is not None:
+                return mh
         except Exception:
             pass  # Fall through to the pure-datasketch path below.
 
@@ -87,6 +105,36 @@ def add(self, block: CodeBlock) -> None:
             minhash = _create_simple_minhash(block.text, self.num_perm)
             self.minhashes.append(minhash)
 
+    def add_blocks(self, blocks: list[CodeBlock]) -> None:
+        """Add multiple code blocks to the LSH index in a vectorized/parallel batch."""
+        if not blocks:
+            return
+
+        if (
+            _native is not None
+            and _np is not None
+            and hasattr(_native, "batch_minhash_signature")
+            and MinHash is not None
+        ):
+            try:
+                feature_lists = [
+                    list(dict.fromkeys(_normalize_text(b.text).split()))
+                    for b in blocks
+                ]
+                signatures = _native.batch_minhash_signature(feature_lists, self.num_perm)
+                for block, sig in zip(blocks, signatures):
+                    self.blocks.append(block)
+                    minhash = _make_minhash_from_signature(sig, self.num_perm)
+                    if minhash and self.lsh:
+                        self.lsh.insert(str(len(self.blocks) - 1), minhash)
+                    self.minhashes.append(minhash)
+                return
+            except Exception:
+                pass
+
+        for block in blocks:
+            self.add(block)
+
     def find_near_duplicates(self, block: CodeBlock) -> list[tuple[CodeBlock, float]]:
         """Find near-duplicates of the given block."""
         if MinHash is None or not self.minhashes:
@@ -175,11 +223,8 @@ def build_lsh_index(
 ) -> LSHIndex:
     """Build LSH index from code blocks."""
     index = LSHIndex(threshold=threshold)
-
-    for block in blocks:
-        if block.line_count >= min_lines:
-            index.add(block)
-
+    eligible = [b for b in blocks if b.line_count >= min_lines]
+    index.add_blocks(eligible)
     return index
 
 

src/redup/core/pipeline/duplicate_finder.py

@@ -108,6 +108,7 @@ def find_fuzzy_groups(
             continue
 
         cluster = [anchor]
+        cluster_sims: list[float] = []
         for other_index in sorted(candidate_indices.get(index, ())):
             other = candidates[other_index]
             other_key = (other.file, other.line_start)
@@ -123,6 +124,7 @@ def find_fuzzy_groups(
             if similarity < min_similarity:
                 continue
             cluster.append(other)
+            cluster_sims.append(similarity)
 
         if len(cluster) < 2:
             continue
@@ -138,9 +140,7 @@ def find_fuzzy_groups(
             )
             for block in cluster
         ]
-        avg_similarity = sum(
-            sequence_similarity(cluster[0].text, block.text) for block in cluster[1:]
-        ) / (len(cluster) - 1)
+        avg_similarity = sum(cluster_sims) / len(cluster_sims)
 
         groups.append(
             DuplicateGroup(

tests/test_lsh_matcher.py

@@ -99,3 +99,30 @@ def test_native_minhash_matches_signature_and_is_order_invariant():
     )
     assert list(first.hashvalues) == list(expected)
     assert first.jaccard(reordered) == 1.0
+
+
+def test_batch_minhash_signature():
+    pytest.importorskip("datasketch")
+    native = pytest.importorskip("redup_fast_hash")
+    assert hasattr(native, "batch_minhash_signature")
+    items = [
+        ["alpha", "beta"],
+        ["gamma", "delta"],
+    ]
+    sigs = native.batch_minhash_signature(items, 128)
+    assert len(sigs) == 2
+    assert sigs[0] == native.compute_minhash_signature(items[0], 128)
+    assert sigs[1] == native.compute_minhash_signature(items[1], 128)
+
+
+def test_index_add_blocks_batch(backend):
+    blocks = [
+        block("b1.py", "alpha beta gamma delta epsilon"),
+        block("b2.py", "alpha beta gamma delta epsilon"),
+        block("b3.py", "one two three four five"),
+    ]
+    index = lsh_matcher.LSHIndex(threshold=0.8)
+    index.add_blocks(blocks)
+    assert len(index.blocks) == 3
+    matches = index.find_near_duplicates(block("query.py", "alpha beta gamma delta epsilon"))
+    assert len(matches) == 2