diff --git a/Cargo.toml b/Cargo.toml
index 50a0c9a9..71dc1d87 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,6 +45,10 @@ test = true
name = "ocr_benchmark"
required-features = ["std"]
+[[example]]
+name = "bf16_rne_exhaustive"
+required-features = ["std"]
+
[[example]]
name = "splat3d_flex"
required-features = ["splat3d"]
diff --git a/README-DE.md b/README-DE.md
index 43cb7e7a..f30ca59e 100644
--- a/README-DE.md
+++ b/README-DE.md
@@ -1,6 +1,8 @@
# ndarray — HPC-Erweiterung fuer Rust
-*Fork von [rust-ndarray/ndarray](https://github.com/rust-ndarray/ndarray) mit 55 HPC-Modulen, 880 Tests, und SIMD-Kernels von Intel AMX bis Raspberry Pi NEON. Laeuft auf stabilem Rust 1.94 ohne Nightly-Features.*
+*Fork von [rust-ndarray/ndarray](https://github.com/rust-ndarray/ndarray) mit 100 HPC-Modulen, 2,534 bestandenen Bibliothekstests und SIMD-Kernels von Intel AMX bis Raspberry Pi NEON. Laeuft auf stabilem Rust 1.98.1 ohne Nightly-Features.*
+
+Zaehlungen bei Commit `f2c1aea`: `pub mod`-Eintraege in `src/hpc/mod.rs`; `cargo test --lib` (2,534 bestanden, 32 ignoriert). Wie jede Zahl auf dieser Seite ermittelt wurde: [Belege](#belege-fuer-die-zahlen-auf-dieser-seite).
[English Version](README.md) | [Kompletter Feature-Vergleich (146 Module)](COMPARISON.md)
@@ -8,169 +10,152 @@
## Worum geht es
-Das Upstream-ndarray ist eine solide Bibliothek fuer n-dimensionale Arrays in Rust. Was es nicht liefert: hardwarenahe SIMD-Beschleunigung, BLAS ohne externe C-Bibliotheken, und Unterstuetzung fuer Datentypen wie f16 oder BF16, die Rust auf stabilem Toolchain schlicht nicht anbietet.
+Das Upstream-ndarray ist eine solide Bibliothek fuer n-dimensionale Arrays in Rust. Was es nicht liefert: hardwarenahe SIMD-Beschleunigung, BLAS ohne externe C-Bibliotheken und Unterstuetzung fuer Datentypen wie f16 oder BF16, die Rust auf einem stabilen Toolchain schlicht nicht anbietet.
-Dieser Fork schliesst diese Luecken. Die Erweiterung umfasst 80.000 Zeilen Code in 179 neuen Dateien — von Goto-GEMM-Mikrokernels ueber ARM-NEON-Stufenerkennung bis zu einem Codec-Stack, der Cosine-Aehnlichkeit als Integer-Tabellen-Lookup implementiert.
+Dieser Fork schliesst diese Luecken. Die Erweiterung umfasst rund 205,000 Zeilen Rust in 424 Dateien, die es upstream nicht gibt — von Goto-GEMM-Mikrokernels ueber ARM-NEON-Stufenerkennung bis zu einem Codec-Stack, der Cosine-Aehnlichkeit als Integer-Tabellen-Lookup implementiert.
-Das Ergebnis laesst sich an einer Zahl festmachen: **611 Millionen Aehnlichkeitsvergleiche pro Sekunde** auf einer Consumer-CPU, ohne Fliesskomma-Arithmetik, ohne GPU.
+Der Kerntrick in einer Zahl: eine Palette-Aehnlichkeit ist **ein Tabellenzugriff — etwa 0.84 ns, ~1.19 Milliarden Lookups pro Sekunde auf einem Kern** eines 2.8-GHz-Cascade-Lake-Xeon, ohne Fliesskomma-Arithmetik und ohne GPU (gemessen; siehe [Belege](#belege-fuer-die-zahlen-auf-dieser-seite)).
---
## Die zentrale Idee: Cosine-Aehnlichkeit ohne Fliesskomma
-Vektorsuche in Datenbanken wie LanceDB oder FAISS berechnet fuer jeden Kandidaten ein Skalarprodukt: `dot(a,b) / (|a| * |b|)`. Bei 768 Dimensionen sind das 1.536 Fliesskomma-Operationen und 6 KB Speicherbandbreite pro Vergleich.
-
-Dieser Fork geht einen anderen Weg. Vektoren werden offline auf 256 Archetypes quantisiert. Die paarweisen Distanzen zwischen allen Archetypes sind in einer 256x256-Tabelle (64 KB) vorberechnet. Zur Laufzeit reduziert sich eine Cosine-Abfrage auf einen einzigen Byte-Lesevorgang aus dem L1-Cache.
+Vektorsuche in Datenbanken wie LanceDB oder FAISS berechnet fuer jeden Kandidaten ein Skalarprodukt: `dot(a,b) / (|a| * |b|)`. Bei 768 Dimensionen sind das 1,536 Fliesskomma-Operationen und 3 KB Kandidatendaten pro Vergleich.
-### Messwerte nach Hardware
+Dieser Fork geht einen anderen Weg. Vektoren werden offline auf 256 Archetypes quantisiert. Die paarweisen Distanzen zwischen allen Archetypes sind in einer 256x256-Tabelle vorberechnet (`DistanceMatrix`, u16-Eintraege, 128 KB). Zur Laufzeit reduziert sich eine Cosine-Abfrage auf einen einzigen Tabellenzugriff.
-| System | Durchsatz | Latenz | Leistung |
-|--------|-----------|--------|----------|
-| Intel Xeon w9 (Sapphire Rapids) | ~3.200 Mio/s | ~0,3 ns | 350 W |
-| Intel i7-11700K (11. Generation) | 2.400 Mio/s | 0,4 ns | 65 W |
-| Raspberry Pi 4 (Cortex-A72) | ~400 Mio/s | ~2,5 ns | 5 W |
-| Raspberry Pi Zero 2W (Cortex-A53) | ~80 Mio/s | ~12 ns | 2 W |
+### Gemessen
-### Einordnung gegenueber GPU und FAISS
+| Operation | Host | Ergebnis |
+|-----------|------|----------|
+| `DistanceMatrix::distance`, zufaellige Paare | Xeon @ 2.8 GHz (Cascade-Lake-Klasse, AVX-512 + VNNI), 1 Thread | 0.84 ns, ~1.19 G Lookups/s |
+| `Base17::l1`, 20,000 Kandidaten | derselbe | je 3.04 ns, gesamt 60.7 µs |
-| System | Methode | Durchsatz | Hardware | Leistung |
-|--------|---------|-----------|----------|----------|
-| Dieser Fork (i7-11700K) | Palette u8 Lookup | 2.400 Mio/s | CPU | 65 W |
-| FAISS GPU (IVF-PQ) | CUDA quantisiert | 200-500 Mio/s | RTX 3060 | 170 W |
-| FAISS GPU (cuVS) | CUDA optimiert | 1.000-2.000 Mio/s | H100 80 GB | 700 W |
-| FAISS CPU (Flat) | AVX2 FP32 Dot | ~50 Mio/s | i7 | 65 W |
-| FAISS CPU (IVF-PQ) | AVX2 quantisiert | 100-200 Mio/s | i7 | 65 W |
-
-> **Zur Methodik:** Alle Zahlen sind pro vollstaendiger Query — ein Vektor rein, ein Aehnlichkeitswert raus. Beide Ansaetze erfordern einmalige Offline-Vorbereitung. Der Unterschied: ein Palette-Lookup ist ein u8-Lesevorgang (0 FLOPs), FAISS PQ dekodiert 8 Subspaces (~16 Ops), FAISS Flat berechnet ein 768-dimensionales Skalarprodukt (~1.536 FLOPs). Der Approximationsfehler beim Foveal-Tier (1/40 Sigma) betraegt 0,4% — geringer als die 5-10% bei typischen PQ-Konfigurationen.
+Fruehere Fassungen dieser Seite nannten Raten pro Plattform (Sapphire Rapids ~3.2 G/s, i7-11700K 2.4 G/s, Raspberry Pi 4 ~400 M/s, Pi Zero 2W ~80 M/s) und einen Vergleich mit FAISS CPU/GPU und cuVS. Fuer diese Zahlen gibt es in diesem Repository keinen Benchmark, und die FAISS/GPU-Zahlen wurden hier nicht gemessen; sie werden daher nicht mehr als Ergebnisse angefuehrt. Ein fairer FAISS-Vergleich braeuchte dieselben Daten, dasselbe Recall-Ziel und dieselbe Hardware.
---
-## Dreistufige Cascade: Wie die Suche tatsaechlich ablaeuft
+## Dreistufige Kaskade: Wie die Suche tatsaechlich funktioniert
-Die Palette-Tabelle allein erklaert noch nicht, wie eine Million Vektoren in zwei Millisekunden durchsucht werden. Dafuer sorgt eine dreistufige Cascade, bei der jede Stufe eine mathematisch gesicherte untere Schranke der naechsten darstellt. Keine Stufe kann einen relevanten Treffer verlieren.
+Die Palette-Tabelle allein erklaert nicht, wie eine Million Vektoren schnell durchsucht wird. Das leistet eine dreistufige Kaskade, in der jede Stufe Kandidaten fuer die naechste aussortiert. Ob eine Stufe ein relevantes Ergebnis verlieren kann, haengt von der verwendeten Schranke ab; diese Garantie ist in diesem Repository noch nicht getestet.
### Stufe 1: Hamming-Sweep ueber bitgepackte Fingerprints
-Jeder Vektor wird als 256-Bit-Fingerprint gespeichert (32 Bytes). Der Vergleich zweier Fingerprints ist eine XOR-Operation gefolgt von einem Hardware-Popcount:
+Jeder Vektor wird als bitgepackter Fingerprint gespeichert. Die folgende Kaskade geht von 32-Byte-Fingerprints (256 Bit) aus; zu beachten: der crate-eigene Typ `Fingerprint<256>` hat 256 *Woerter* — 2,048 Bytes. Der Vergleich zweier Fingerprints ist ein XOR gefolgt von einem Popcount:
-- **AVX-512 VPOPCNTDQ**: Zwei Fingerprints in einem Takt
-- **NEON vcntq_u8**: Pro-Byte-Popcount, nativ auf jedem ARM-Prozessor
+- **AVX-512 VPOPCNTDQ**: nativer 64-Bit-Lane-Popcount, wo verfuegbar; sonst ein VPSHUFB-Lookup + VPSADBW (AVX-512 BW / AVX2)
+- **NEON vcntq_u8**: Popcount pro Byte, nativ auf jedem ARM-Prozessor
-Ein Scan ueber eine Million Fingerprints dauert etwa 2 Millisekunden und eliminiert 97-99% der Kandidaten. Die Hamming-Distanz ist eine beweisbare untere Schranke der Cosine-Distanz — es gibt keine False Negatives.
+Gemessen mit `bitwise::hamming_batch_raw` auf einem Kern des Cascade-Lake-Hosts (ohne VPOPCNTDQ): eine Anfrage gegen eine Million 32-Byte-Fingerprints dauert **15.5 ms**; gegen 2,048-Byte-Zeilen vom Typ `Fingerprint<256>` kostet es 277 ns pro Zeile (~7.4 GB/s). Die Aussortierungsrate haengt von den Daten und dem Schwellwert ab; sie ist hier nicht gemessen.
-### Stufe 2: Base17 L1-Distanz
+### Stufe 2: Base17-L1-Distanz
-Die verbleibenden ~20.000 Kandidaten werden mit 17-dimensionalen i16-Vektoren (34 Bytes) verfeinert. Das passt in einen einzigen AVX-512-Load oder zwei NEON-Loads. Kosten: ~3 Nanosekunden pro Vergleich. Uebrig bleiben ~200 Kandidaten.
+Die verbleibenden ~20,000 Kandidaten werden mit 17-dimensionalen i16-Vektoren (34 Bytes) verfeinert. Gemessene Kosten: 3.04 ns pro Vergleich (60.7 µs fuer 20,000). Etwa 200 Kandidaten ueberleben.
### Stufe 3: Palette-Lookup
-Die ~200 Finalisten werden ueber die vorberechnete 256x256-Tabelle bewertet. Ein Lesevorgang pro Kandidat, 0,4 Nanosekunden.
+Die ~200 Finalisten werden ueber die vorberechnete 256x256-Tabelle bewertet. Ein Zugriff pro Kandidat, 0.84 ns gemessen.
-### Gesamtbilanz fuer eine Million Vektoren
+### Ende-zu-Ende: Eine Million Vektoren bis Top-K
-| Stufe | Eingang | Ausgang | Dauer | Bandbreite |
-|-------|---------|---------|-------|------------|
-| Hamming-Sweep | 1.000.000 | ~20.000 | ~2 ms | 32 MB |
-| Base17 L1 | 20.000 | ~200 | ~60 us | 680 KB |
-| Palette-Lookup | 200 | Top-K | ~0,08 us | 200 B |
-| **Gesamt** | | | **~2,1 ms** | **~33 MB** |
+| Stufe | Eingang | Ausgang | Dauer | Beleg |
+|-------|---------|---------|-------|-------|
+| Hamming-Sweep (32 B) | 1,000,000 | datenabhaengig | 15.5 ms | gemessen, 1 Kern, ohne VPOPCNTDQ |
+| Base17 L1 | 20,000 | ~200 | 60.7 µs | gemessen |
+| Palette-Lookup | 200 | Top-K | ~0.17 µs | 200 × 0.84 ns, abgeleitet |
-FAISS CPU Flat benoetigt fuer dieselbe Aufgabe ~20 ms und liest ~6 GB. Die Cascade ist zehnmal schneller bei zweihundertmal weniger Speicherbandbreite.
+Bei 32-Byte-Zeilen laeuft der Sweep mit 2.1 GB/s; der Overhead pro Zeile, nicht die Speicherbandbreite, ist also die Grenze (2,048-Byte-Zeilen erreichen 7.4 GB/s); Multi-Core-Skalierung ist hier nicht gemessen. Ein Ende-zu-Ende-Vergleich mit FAISS Flat wurde in diesem Repository nicht durchgefuehrt.
-### Integration in LanceDB
+### Integration mit Lance
-In einem Lance-Dataset ersetzt der Cascade-Sweep die FP32-Distanzberechnung von `lance-linalg`. Der Scan liest die bitgepackte Fingerprint-Spalte, fuehrt den Hardware-Popcount-Sweep durch, und holt vollstaendige Vektoren nur fuer die wenigen Ueberlebenden.
+Die Kaskade ist ein Substratpfad, kein Lance-Index. In [lance-graph](https://github.com/AdaWorldAPI/lance-graph) wird die Hamming-Distanz von Bitvektoren als DataFusion-UDF `hamming_distance` bereitgestellt (sie ruft `bitwise::hamming_distance_raw` auf). Sie ist **nicht** in die ANN-Suche von Lance eingebunden, die weiterhin `lance-linalg`-Distanzen verwendet und fuer eine Hamming-Metrik einen Fehler zurueckgibt. Nichts hier ersetzt `lance-linalg` innerhalb eines Lance-Scans.
---
-## Was Upstream liefert und was dieser Fork ergaenzt
+## Was Upstream bietet und was dieser Fork hinzufuegt
### SIMD-Abdeckung
-Das Upstream-ndarray delegiert Matrixmultiplikation an das externe Crate `matrixmultiply`, das AVX2 nutzen kann. Eigene SIMD-Typen oder Hardware-Erkennung gibt es nicht. Auf ARM faellt Upstream auf skalaren Code zurueck.
+Upstream-ndarray delegiert die Matrixmultiplikation an den externen Crate `matrixmultiply`, der AVX2 nutzen kann. Es hat keine eigenen SIMD-Typen und keine Hardwareerkennung. Auf ARM faellt Upstream auf Skalarcode zurueck.
+
+Dieser Fork implementiert eine eigene SIMD-Schicht: 27 portable Vektor-/Maskentypen, zur Compile-Zeit ausgewaehlt (AVX-512, AVX2, NEON, WASM SIMD128, skalar oder Nightly-`core::simd`), dazu zur Laufzeit dispatchte Kernels ueber 7 Stufen (`amx_int8 > avx512vnni > avx512f > avxvnni > avx2_fma > neon > scalar`). Jede Stufe ist an das Instruktions-Feature gebunden, das ihr Kernel braucht; die Stufe `avxvnni` (VEX `VPDPBUSD`) ist an AVX-VNNI gebunden. Diese Stufe konnte auf dem Messhost nicht ausgefuehrt werden, der AVX-512 VNNI, aber nicht AVX-VNNI hat, und ihr Kernel wird nur anhand seiner emittierten Instruktionskodierung geprueft.
-Dieser Fork implementiert eine vollstaendige SIMD-Schicht mit Laufzeiterkennung:
+Was die Schicht bringt, ist pro Operation gegen eine benannte Baseline gemessen, auf einem Kern des Cascade-Lake-Hosts (Median aus 15 Laeufen, 1 M Elemente). Jedes Zeitpaar nennt zuerst den Fork-Wert, danach den Baseline-Wert:
-| Befehlssatz | Upstream | Dieser Fork | Beschleunigung |
-|-------------|----------|-------------|----------------|
-| AVX-512 (16 x f32) | Skalar | Native __m512-Typen | ~8x |
-| AVX-512 VNNI (int8) | Skalar | 64 MACs/Instruktion | ~32x |
-| AVX-512 VPOPCNTDQ | Skalar | Nativer 512-Bit-Popcount | ~16x |
-| AMX (256 MACs) | Nicht vorhanden | Inline-ASM auf stabilem Rust | ~128x |
-| AVX2 + FMA (8 x f32) | Extern (matrixmultiply) | Goto-GEMM + Dispatch | ~4x |
-| NEON (4 x f32) | Skalar | 3-stufig: A53/A72/A76 | ~4x |
-| NEON dotprod (ARMv8.2) | Nicht vorhanden | vdotq_s32 (Pi 5) | ~16x |
+| Operation | Baseline | Fork | Verhaeltnis |
+|-----------|----------|------|-------------|
+| u8-Vergleich → Bitmaske (`simd::eq_u8_to_mask`) | einfache Rust-Schleife | 0.029 vs 0.088 ns/elem | 3.1× |
+| f32 → BF16 RNE (`f32_to_bf16_batch_rne`) | skalar pro Element | 0.196 vs 1.62 ns/elem | 8.3× |
+| maskierte i32-Summe (`masked_sum_i32`, 50% Dichte) | einfache Bit-Test-Schleife | 0.53 vs 0.79 ns/elem | 1.5× |
+| f32-Summe (`F32x16` + `reduce_sum`) | sequentielles `iter().sum()` | 0.128 vs 1.26 ns/elem | 9.8× |
+| int8-GEMM u8×i8→i32 256³ (`gemm_u8_i8`, VNNI) | `int8_gemm_i32` (skalar) | 22.5 vs 5.9 GMAC/s | 3.8× |
+| AMX-INT8-GEMM 2048³ | skalar | 169.7 GMAC/s | 600× (Emerald Rapids, [`AMX_GOTCHAS.md`](.claude/AMX_GOTCHAS.md)) |
-Die Erkennung erfolgt einmalig beim ersten Zugriff ueber `LazyLock` — ein CPUID-Aufruf, danach nur noch ein Pointer-Deref pro Funktionsaufruf (0,3 ns statt 1-3 ns bei wiederholter Feature-Abfrage).
+Wo einfaches Rust bereits autovektorisiert, erreicht der Polyfill dasselbe, ohne es zu uebertreffen: Fused Multiply-Add, gechunkte f32-Summen, 64-Bit-Popcount und `popcount(a&b&c)` liegen alle innerhalb von ±20% der einfachen Schleife (LLVM emittiert denselben VPSHUFB-Popcount und VPTERNLOGQ). Die Instruktionsbreite (16 f32-Lanes, 64 VNNI-MACs) ist eine Obergrenze, kein Speedup.
+
+Die Erkennung erfolgt einmalig ueber `LazyLock`. Auf diesem Host kostet ein wiederholtes `is_x86_feature_detected!` ~0.34 ns und eine `simd_caps()`-Kopie ~0.62 ns; der Gewinn des Einfrierens des Dispatch ist also vorhersagbarer Dispatch und eine Entscheidung pro Prozess, keine grosse Ersparnis pro Aufruf.
### GEMM-Leistung
-| Matrixgroesse | Upstream | Dieser Fork | NumPy (OpenBLAS) | GPU (RTX 3060) |
-|--------------|----------|-------------|------------------|----------------|
-| 512 x 512 | ~20 GFLOPS | 47 GFLOPS | ~45 GFLOPS | ~1.200 GFLOPS |
-| 1024 x 1024 | ~13 GFLOPS | 139 GFLOPS | ~120 GFLOPS | ~3.500 GFLOPS |
-| 2048 x 2048 | ~13 GFLOPS | ~150 GFLOPS | ~140 GFLOPS | ~5.000 GFLOPS |
+Gemessen auf einem Kern des Cascade-Lake-Hosts (beste von 3–7 Laeufen, `matrixmultiply`-Threading aus):
+
+| Matrixgroesse | `Array::::dot` | `Array::::dot` | `simd::gemm_f64_tiled_fma` |
+|---------------|--------------------|--------------------|---------------------------|
+| 512 × 512 | 70.8 GFLOPS | 34.3 GFLOPS | 9.7 GFLOPS |
+| 1024 × 1024 | 70.1 GFLOPS | 34.3 GFLOPS | 9.1 GFLOPS |
+| 2048 × 2048 | 65.0 GFLOPS | 32.7 GFLOPS | — |
-Upstream trifft bei 1024 x 1024 auf ein Cache-Problem: kein Tiling, kein Threading, kein Microkernel. Der Fork nutzt den Goto-Algorithmus mit Cache-Blocking (L1/L2/L3) und erreicht 10,5-fachen Durchsatz — auf dem Niveau von NumPys jahrzehntealtem OpenBLAS.
+`Array::dot()` ruft `matrixmultiply::sgemm`/`dgemm` auf (`src/linalg/impl_linalg.rs:503,522`) — dieselbe Engine, die Upstream nutzt; diese Spalten sind also kein Vergleich Fork gegen Upstream. Das forkeigene `gemm_f64_tiled_fma` (festes `TILE=64`, `F64x8`-Akkumulation) ist bei f64 derzeit ~3.6× langsamer als `matrixmultiply`. Eine fruehere Tabelle auf dieser Seite (Fork 47/139/~150 GFLOPS gegen Upstream 13–20, dazu NumPy- und RTX-3060-Spalten) hatte keinen Benchmark im Repository und wurde zurueckgezogen.
+
+`simd_ops::array_chunks` durchlaeuft einen Slice als nicht ueberlappende `&[T; N]`-Fenster; `array_windows` ist das ueberlappende Gegenstueck (ein Stable-Rust-Aequivalent des Nightly-`slice::array_windows::()`). Beide legen die Fenstergroesse an der Aufrufstelle fest, sodass sie direkt in `F32x16::from_array` / `F64x8::from_array` einfliesst, und beide sparen die Bounds-Pruefung pro Element, die eine dynamisch indizierte Schleife zahlt. Aktuelle In-Crate-Aufrufstellen: `hpc::blake3` (64-Byte-Block-Chunking) und `heel_f64x8::cosine_f32_to_f64_simd`, beide ueber `array_chunks`; `array_windows`, `array_windows_checked` und `array_chunks_checked` sind exportiert, haben aber noch keinen produktiven Aufrufer im Crate. Sie sind das Traversierungs-Primitiv, auf dem die handgeschriebenen BLAS-Graph-/bgz17-Kernels aufbauen, wo das Const-Generic-Fenster nahe an eine Cranelift-JIT-kompilierte innere Schleife herankam, ohne fuer einen JIT zu zahlen — siehe die Moduldokumentation in `src/simd_ops.rs`.
### Datentypen jenseits von f32/f64
| Typ | Upstream | Dieser Fork | Methode |
|-----|----------|-------------|---------|
-| f16 (IEEE 754) | Nicht vorhanden | Vorhanden | u16 als Traeger + F16C-Hardware (x86) / FCVTL via Inline-ASM (ARM) |
-| BF16 (bfloat16) | Nicht vorhanden | Vorhanden | Hardware-Instruktionen + RNE-Emulation (bit-exakt mit VCVTNEPS2BF16) |
-| i8/u8 (quantisiert) | Nicht vorhanden | Vorhanden | VNNI-Dot, Hamming, Popcount |
-| i16 (Base17) | Nicht vorhanden | Vorhanden | L1-Distanz mit SIMD-Widen/Narrow |
+| f16 (IEEE 754) | Nicht verfuegbar | Verfuegbar | u16-Traeger + F16C-Hardware (x86) / FCVTL ueber Inline-Asm (ARM) |
+| BF16 (bfloat16) | Nicht verfuegbar | Verfuegbar | Hardware-Instruktionen + RNE-Emulation (bitgenau mit VCVTNEPS2BF16) |
+| i8/u8 (quantisiert) | Nicht verfuegbar | Verfuegbar | VNNI-Dot, Hamming, Popcount |
+| i16 (Base17) | Nicht verfuegbar | Verfuegbar | L1-Distanz mit SIMD-Widen/Narrow |
-Rusts `f16`-Typ ist Nightly-only (Issue #116909). Der Fork nutzt denselben Trick wie bei AMX: `u16` als Traegertyp, Hardware-Instruktionen ueber stabile `#[target_feature]`-Attribute oder Inline-Assembler. Das Ergebnis ist IEEE-754-konforme Konvertierung mit Hardware-Geschwindigkeit auf stabilem Rust.
+Rusts `f16`-Typ ist nur auf Nightly verfuegbar (Issue #116909). Der Fork nutzt denselben Ansatz wie bei AMX: `u16` als Traeger, Hardware-Instruktionen ueber stabile `#[target_feature]`-Attribute oder Inline-Assembler. Das Ergebnis ist IEEE-754-konforme Konvertierung mit Hardware-Geschwindigkeit auf stabilem Rust.
---
## Sieben Dinge, die sonst niemand auf stabilem Rust macht
-**1. Vollstaendiger std::simd-Polyfill.** Die portable SIMD-API von Rust ist seit Jahren Nightly-only. Dieser Fork implementiert dieselbe Typoberflaeche — F32x16, F64x8, U8x64, Masken, Reduktionen, Vergleiche — mit stabilen core::arch-Intrinsics. Wenn std::simd stabilisiert wird, aendert sich eine use-Zeile.
+**1. Ein std::simd-foermiger Polyfill auf Stable.** Rusts portable SIMD-API ist seit Jahren nur auf Nightly verfuegbar. Dieser Fork implementiert eine `std::simd`-artige Typoberflaeche — 27 Typen einschliesslich F32x16, F64x8, U8x64, Masken, Reduktionen und Vergleiche — auf stabilem `core::arch`, mit einem Nightly-`core::simd`-Backend hinter dem Feature `nightly-simd` und bitgenauen Paritaets-Crates in der CI (`simd-masking-parity`, ausgefuehrt unter AVX-512, AVX2, NEON via qemu und wasm). Es ist nicht die vollstaendige `std::simd`-API, und eine Methode, die auf einem Backend vorhanden ist, ist nicht auf allen garantiert; das Paritaetsprogramm prueft genau das, und die `F64x8`-Vergleiche fehlten auf AVX2 und NEON, bis sie dort hinzugefuegt wurden.
-**2. f16 ohne Nightly.** Carrier-Typ u16 plus Hardware-Instruktionen: F16C (VCVTPH2PS/VCVTPS2PH) auf x86, FCVTL/FCVTN via asm!() auf ARM. Drei Praezisionsstufen: Plain f16 (10 Bit Mantisse), Scaled-f16 (bereichsoptimiert, 1,5x praeziser), Double-f16 (hi+lo-Paar, ~20 Bit effektiv).
+**2. f16 ohne Nightly.** Traegertyp u16 plus Hardware-Instruktionen: F16C (VCVTPH2PS/VCVTPS2PH) auf x86, FCVTL/FCVTN ueber asm!() auf ARM. Drei Genauigkeitsstufen: einfaches f16 (10-Bit-Mantisse), scaled-f16 (bereichsoptimiert, 1.5x genauer), double-f16 (Hi+Lo-Paar, ~20 Bit effektiv).
-**3. AMX auf stabilem Rust.** Intels Advanced Matrix Extensions (TDPBUSD: 16x16 Tile, 256 MACs pro Instruktion) sind als Rust-Intrinsics Nightly-only (Issue #126622). Der Fork emittiert die Instruktionen direkt als asm!(".byte ...") — verifiziert auf Rust 1.94 mit Kernel 6.18+.
+**3. AMX auf stabilem Rust.** Intels Advanced Matrix Extensions (TDPBUSD: eine 16×16-Kachel von Ausgaben ueber K=64 Bytes, 16,384 MACs pro Instruktion) sind als Rust-Intrinsics nur auf Nightly verfuegbar (Issue #126622). Der Fork emittiert sie ueber `asm!` — alle vier INT8-Formen (`tdpb{ss,su,us,uu}d`), BF16, FP16 und FP8 — und maß 169.7 GMAC/s single-threaded bei INT8 2048³ (Emerald Rapids, Kernel 6.18.5, [`AMX_GOTCHAS.md`](.claude/AMX_GOTCHAS.md)).
-**4. Gestufte ARM-NEON-Unterstuetzung.** Drei Stufen mit Laufzeiterkennung: A53-Baseline (Pi Zero 2W, Pi 3 — eine NEON-Pipeline), A72-Fast (Pi 4, Orange Pi 4 — zwei Pipelines, 2x-Unrolling), A76-DotProd (Pi 5, Orange Pi 5 — vdotq_s32, natives fp16). big.LITTLE-Systeme (RK3399, RK3588) werden korrekt behandelt.
+**4. Gestufte ARM-NEON-Erkennung.** Drei Stufen mit Laufzeiterkennung (auf aarch64 werden die portablen NEON-Typen verwendet; die dotprod-/BF16-Kernel-Stubs und die `simd_dispatch`-Tabelle leiten weiterhin an skalare Wrapper): A53-Basis (Pi Zero 2W, Pi 3 — einzelne NEON-Pipeline), A72 schnell (Pi 4, Orange Pi 4 — duale Pipeline, 2x Unrolling), A76 dotprod (Pi 5, Orange Pi 5 — vdotq_s32, natives fp16). big.LITTLE-Systeme (RK3399, RK3588) werden korrekt behandelt.
-**5. Eingefrorener Dispatch mit 0,3 ns pro Aufruf.** Ueblicher SIMD-Code prueft pro Aufruf: `if is_x86_feature_detected!("avx512f") { ... }` — ein atomarer Load plus Branch. Dieser Fork erkennt einmal und friert eine Funktionszeiger-Tabelle ein (LazyLock, Copy-Struct). Danach: ein indirekter Call, kein Atomic, kein Branch-Prediction-Miss.
+**5. Eingefrorener Dispatch.** Der Fork erkennt CPU-Features einmal und friert eine Funktionszeiger-Tabelle ein (`LazyLock`), sodass jeder spaetere Aufruf denselben indirekten Pfad nimmt. Die Ersparnis pro Aufruf ist auf aktuellen CPUs klein — ein gecachtes `is_x86_feature_detected!` kostet hier bereits ~0.34 ns —; der Wert liegt in einer Entscheidung pro Prozess und einer einzigen Stelle, die die gewaehlte Stufe benennt.
-**6. BF16-Konvertierung bit-exakt mit Hardware.** Die Funktion f32_to_bf16_batch_rne() implementiert den IEEE-754-RNE-Algorithmus mit reinen AVX-512-F-Instruktionen und stimmt Bit-fuer-Bit mit Intels VCVTNEPS2BF16 ueberein. Verifiziert gegen Hardware-Ausgabe auf ueber einer Million Eingaben, einschliesslich Subnormalen, Unendlich, NaN und Halfway-Ties.
+**6. BF16-Konvertierung bitgenau mit der Hardware.** Die Funktion f32_to_bf16_batch_rne() implementiert den IEEE-754-RNE-Algorithmus mit reinen AVX-512-F-Instruktionen und stimmt bitweise mit Intels VCVTNEPS2BF16 ueberein. Geprueft gegen die skalare RNE-Referenz und ein unabhaengiges f64-Orakel ueber **alle 4,294,967,296 f32-Bitmuster: 0 Abweichungen** (`cargo run --release --example bf16_rne_exhaustive`, 11.5 s auf 4 Threads des Cascade-Lake-Hosts). Der Vergleich ist exakte u16-Gleichheit, NaN-Vorzeichen und Payload-Bits zaehlen also mit. Ein Unit-Test vergleicht zusaetzlich mit der Hardware-Instruktion `VCVTNEPS2BF16` auf Hosts mit AVX-512-BF16; der Messhost hat keinen, dieser Vergleich wurde hier also nicht ausgefuehrt.
-**7. Kognitiver Codec-Stack.** Ueber klassische Numerik hinaus implementiert der Fork eine vollstaendige Encoding-Pipeline: Fingerprint<256> (VSA, SIMD-Hamming), Base17 (17-dimensionale i16-Vektoren), CAM-PQ (Produkt-Quantisierung mit kompilierten Distanztabellen), Palette-Semiring (256x256-Distanzmatrizen fuer O(1)-Lookups), bgz7/bgz17 (komprimiertes Modellgewichts-Format: 201 GB BF16-Safetensors -> 685 MB bgz7).
+**7. Kognitiver Codec-Stack.** Ueber klassische Numerik hinaus implementiert der Fork eine vollstaendige Kodierungs-Pipeline: Fingerprint<256> (VSA, SIMD-Hamming), Base17 (17-dimensionale i16-Vektoren), CAM-PQ (Produktquantisierung mit kompilierten Distanztabellen), Palette-Semiring (256x256-Distanzmatrizen fuer O(1)-Lookups), bgz7/bgz17 (komprimiertes Modellgewichtsformat; eine Konvertierung von 201 GB BF16 → 685 MB wurde fuer die Release-Artefakte in lance-graph berichtet, in diesem Repository nicht reproduziert).
---
-## Codebook-Inferenz: Token-Generierung ohne GPU
-
-Neben Vektorsuche nutzt der Fork denselben Tabellenansatz fuer LLM-Inferenz. Statt Matrixmultiplikation (`y = W*x`) wird ein vorberechnetes Codebook indiziert (`y = codebook[index[x]]`) — O(1) pro Token.
+## Codebook-Inferenz: Token-Erzeugung ohne GPU
-| Hardware | Befehlssatz | Tokens/s | Latenz (50 Tokens) | Leistung |
-|----------|-------------|----------|---------------------|----------|
-| Sapphire Rapids | AMX | 380.000 | 0,13 ms | 250 W |
-| Xeon (AVX-512 VNNI) | VNNI | 10.000-50.000 | 1-5 ms | 150 W |
-| Raspberry Pi 5 | NEON + dotprod | 2.000-5.000 | 10-25 ms | 5 W |
-| Raspberry Pi 4 | NEON (dual) | 500-2.000 | 25-100 ms | 5 W |
-
-Bei 5 Watt generiert ein Pi 4 eine 50-Token-Antwort fuer einen Sprachassistenten in unter 100 Millisekunden.
+Ueber die Vektorsuche hinaus nutzt der Fork denselben Tabellenansatz fuer LLM-Inferenz. Statt Matrixmultiplikation (`y = W*x`) wird ein vorberechnetes Codebook indiziert (`y = codebook[index[x]]`) — O(1) pro Token. Eine hier zuvor gezeigte Tokens-pro-Sekunde-Tabelle (AMX 380,000 tok/s bis Pi 4 500–2,000 tok/s) hatte keinen Benchmark im Repository und wurde zurueckgezogen, bis sie reproduziert ist.
---
-## f16-Gewichtstranskodierung
+## f16-Gewichts-Transcodierung
-Getestet mit einem 15-Millionen-Parameter-Modell (Groessenordnung Piper TTS):
+Gemessen auf einem Kern des Cascade-Lake-Hosts (F16C), 15 Millionen gaussverteilte Gewichte (σ = 0.02):
| Format | Groesse | Maximaler Fehler | RMSE | Durchsatz |
-|--------|---------|-----------------|------|-----------|
+|--------|---------|------------------|------|-----------|
| f32 (Original) | 60 MB | — | — | — |
-| f16 (IEEE 754) | 30 MB | 7,3 x 10^-6 | 2,5 x 10^-6 | 94 Mio Params/s |
-| Scaled-f16 | 30 MB | 4,9 x 10^-6 | 2,1 x 10^-6 | 91 Mio Params/s |
-| Double-f16 | 60 MB | 5,7 x 10^-8 | 1,8 x 10^-8 | 42 Mio Params/s |
+| f16 (`cast_f32_to_f16_batch`) | 30 MB | 3.1 × 10⁻⁵ | 4.2 × 10⁻⁶ | 1,805 M Params/s |
-Mit AVX2-F16C-Hardware: ~500 Millionen Parameter pro Sekunde (8 Konvertierungen pro Taktzyklus).
+Der Fehler haengt von der Gewichtsverteilung ab; scaled-f16 und double-f16 sind fuer engere Fehlergrenzen verfuegbar. Eine fruehere Tabelle (94/91/42 M Params/s) hatte keinen Benchmark im Repository.
---
@@ -181,40 +166,80 @@ use ndarray::Array2;
use ndarray::hpc::simd_caps::simd_caps;
let a = Array2::::ones((1024, 1024));
-let c = a.dot(&a); // AVX-512 / AVX2 / NEON — automatisch
+let c = a.dot(&a); // matrixmultiply, as upstream
let caps = simd_caps();
-if caps.avx512f { println!("AVX-512 aktiv"); }
-if caps.neon { println!("ARM-Profil: {}", caps.arm_profile().name()); }
+if caps.avx512f { println!("AVX-512 active"); }
+if caps.neon { println!("ARM profile: {}", caps.arm_profile().name()); }
```
```bash
-# Automatische SIMD-Erkennung
+# Portable / distribution build — x86-64-v3 (AVX2) baseline, runs on any
+# Haswell-or-later x86_64. Pass the config EXPLICITLY: since 2026-09-16 the
+# default is `target-cpu=native`, which tunes the artifact to the BUILD host
+# and is not safe to ship (`.cargo/config-native.toml` says so in as many
+# words). Runtime `simd_caps()` detection cannot rescue a binary whose
+# baseline codegen already emits host-only instructions.
+cargo --config .cargo/config-v3.toml build --release
+
+# Build for THIS machine (dev / benchmarking). Fastest here, portable nowhere.
cargo build --release
-# Cross-Kompilierung fuer Raspberry Pi 4
+# Cross-compile for Raspberry Pi 4
cargo build --release --target aarch64-unknown-linux-gnu
-# Maximale Leistung auf AVX-512-Server
-RUSTFLAGS="-C target-cpu=x86-64-v4" cargo build --release
+# Maximum performance on AVX-512 server
+cargo --config .cargo/config-v4.toml build --release
-# 880 HPC-Tests ausfuehren
-cargo test
+# Library tests (2,534 at f2c1aea)
+cargo test --lib
```
-## Voraussetzungen
+## Anforderungen
-- Rust 1.94 stable (kein Nightly, keine instabilen Features)
+- Rust 1.98.1 stable (festgelegt in `rust-toolchain.toml`; kein Nightly, keine instabilen Features)
- Optional: gcc-aarch64-linux-gnu fuer Pi-Cross-Kompilierung
-- Optional: Intel MKL oder OpenBLAS (Feature-gated)
+- Optional: Intel MKL oder OpenBLAS (Feature-gesteuert)
+
+### Transitive Abhaengigkeiten des Features `std`
+
+**Keine fuer Hashing.** BLAKE3 ist im Crate enthalten.
+
+Die kognitiven Substratmodule unter `hpc/` — `plane`, `seal`,
+`merkle_tree`, `vsa`, `spo_bundle`, `crystal_encoder`, `compression_curves`,
+`deepnsm` — nutzen `hpc::blake3` fuer Integritaets-Hashing und XOF-Expansion. Das
+ist eine portable, reine Rust-Transkription der BLAKE3-Referenzimplementierung,
+die in diesem Crate ausgeliefert wird: kein SIMD, kein `unsafe`, kein C und kein
+Build-Skript.
+
+Fruehere Fassungen zogen hier den externen Crate **`blake3`** ein, zunaechst
+hinter `hpc-extras` (was wiederkehrende "missing blake3"-Build-Fehler fuer
+Konsumenten wie `burn-ndarray` verursachte, die
+`default-features = false, features = ["std"]` waehlen), dann an `std` gebunden.
+**Beides ist entfernt.** `blake3` und seine transitiven `constant_time_eq`,
+`arrayref` und `arrayvec` erscheinen in keiner Feature-Kombination mehr im
+Abhaengigkeitsgraphen, die Falle kann sich also nicht wiederholen.
+
+Konsumenten, die mit `default-features = false` bauen (kein `std`, z. B. das
+nostd-Target `thumbv6m-none-eabi`), ueberspringen das Modul `hpc` und damit den
+BLAKE3-Code; das nostd-Linken bleibt unberuehrt.
+
+## Belege fuer die Zahlen auf dieser Seite
+
+| Zahl | Beleg |
+|------|-------|
+| 100 HPC-Module, 2,534 Lib-Tests, ~205k hinzugefuegte Zeilen / 424 Dateien | gezaehlt bei `f2c1aea` (`src/hpc/mod.rs`; `cargo test --lib`; Pfad-Diff gegen rust-ndarray `bd3ade9`) |
+| 0.84 ns Palette-Lookup, 3.04 ns Base17 L1, 15.5 ms 1-M-Sweep, SIMD-Verhaeltnisse, GEMM, f16 | gemessen auf einem Kern eines Xeon @ 2.8 GHz (Family 6 Model 85, AVX-512 F/BW/VL/DQ/CD + VNNI; kein AMX, kein AVX-512-BF16, kein VPOPCNTDQ), Rust 1.98.1, `target-cpu=native`, Median aus 15 Laeufen, sofern nicht anders angegeben |
+| BF16 RNE, alle 2³² Eingaben, 0 Abweichungen | `examples/bf16_rne_exhaustive.rs`: jedes u32-Bitmuster in 4 zusammenhaengenden Bereichen, Batches von 65,536 Eingaben durch den AVX-512F-Pfad; exakter u16-Vergleich gegen `f32_to_bf16_scalar_rne` und ein unabhaengiges f64-Nearest-Value-Orakel (quiet-forced NaN, DAZ, Ties to Even); reihenfolgeunabhaengige Ausgabe-Pruefsumme `0x5cd3eaa07f7f8080` (gleich fuer 2 und 4 Threads); 11.5 s, Rust 1.98.1. Ein absichtlich kaputtes Orakel (Ties weg von Null) meldet 32,512 Abweichungen, die Pruefung kann also fehlschlagen |
+| AMX 169.7 GMAC/s, 600× skalar | gemessen auf Emerald Rapids, [`AMX_GOTCHAS.md`](.claude/AMX_GOTCHAS.md) |
## Oekosystem
Dieser Fork ist das Hardware-Fundament einer groesseren Architektur:
-| Repository | Aufgabe |
-|------------|---------|
-| [lance-graph](https://github.com/AdaWorldAPI/lance-graph) | Graph-Query-Engine, Cypher-Parser, Codec-Stack |
+| Repository | Zweck |
+|------------|-------|
+| [lance-graph](https://github.com/AdaWorldAPI/lance-graph) | Cypher/SQL-Engine auf DataFusion, die spaltenorientierte Abfrageoberflaeche Quack, Codec-Stack. Verantwortet Graph-, Abfrage- und Ende-zu-Ende-Benchmarks; dieses Repository verantwortet Kernel- und Mikrobenchmark-Zahlen |
| [home-automation-rs](https://github.com/AdaWorldAPI/home-automation-rs) | Smart Home mit Sprach-KI, MCP-Server, MQTT |
## Lizenz
diff --git a/README.md b/README.md
index 3e6aeb4e..7e47cd37 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
# ndarray — HPC Expansion for Rust
-*Fork of [rust-ndarray/ndarray](https://github.com/rust-ndarray/ndarray) with 55 HPC modules, 880 tests, and SIMD kernels from Intel AMX to Raspberry Pi NEON. Runs on stable Rust 1.94 without nightly features.*
+*Fork of [rust-ndarray/ndarray](https://github.com/rust-ndarray/ndarray) with 100 HPC modules, 2,534 passing library tests, and SIMD kernels from Intel AMX to Raspberry Pi NEON. Runs on stable Rust 1.98.1 without nightly features.*
+
+Counts at commit `f2c1aea`: `pub mod` entries in `src/hpc/mod.rs`; `cargo test --lib` (2,534 passed, 32 ignored). How every number on this page was obtained: [Evidence](#evidence-for-the-numbers-on-this-page).
[Deutsche Version](README-DE.md) | [Full Feature Comparison (146 modules)](COMPARISON.md)
@@ -10,76 +12,63 @@
The upstream ndarray is a solid library for n-dimensional arrays in Rust. What it does not provide: hardware-aware SIMD acceleration, BLAS without external C libraries, and support for data types like f16 or BF16 that Rust simply does not offer on a stable toolchain.
-This fork closes those gaps. The expansion comprises 80,000 lines of code in 179 new files — from Goto-GEMM microkernels to ARM NEON tier detection to a codec stack that implements cosine similarity as an integer table lookup.
+This fork closes those gaps. The expansion comprises about 205,000 lines of Rust in 424 files that do not exist upstream — from Goto-GEMM microkernels to ARM NEON tier detection to a codec stack that implements cosine similarity as an integer table lookup.
-The result can be captured in a single number: **611 million similarity comparisons per second** on a consumer CPU, without floating-point arithmetic, without a GPU.
+The core trick in one number: a palette similarity is **one table read — about 0.84 ns, ~1.19 billion lookups per second on one core** of a 2.8 GHz Cascade Lake Xeon, with no floating-point arithmetic and no GPU (measured; see [Evidence](#evidence-for-the-numbers-on-this-page)).
---
## The Core Idea: Cosine Similarity Without Floating Point
-Vector search in databases like LanceDB or FAISS computes a dot product for every candidate: `dot(a,b) / (|a| * |b|)`. At 768 dimensions, that is 1,536 floating-point operations and 6 KB of memory bandwidth per comparison.
-
-This fork takes a different approach. Vectors are quantized offline to 256 archetypes. The pairwise distances between all archetypes are precomputed into a 256x256 table (64 KB). At query time, a cosine lookup reduces to a single byte read from L1 cache.
-
-### Measurements by Hardware
+Vector search in databases like LanceDB or FAISS computes a dot product for every candidate: `dot(a,b) / (|a| * |b|)`. At 768 dimensions, that is 1,536 floating-point operations and 3 KB of candidate data per comparison.
-| System | Throughput | Latency | Power |
-|--------|-----------|---------|-------|
-| Intel Xeon w9 (Sapphire Rapids) | ~3,200M/s | ~0.3 ns | 350W |
-| Intel i7-11700K (11th generation) | 2,400M/s | 0.4 ns | 65W |
-| Raspberry Pi 4 (Cortex-A72) | ~400M/s | ~2.5 ns | 5W |
-| Raspberry Pi Zero 2W (Cortex-A53) | ~80M/s | ~12 ns | 2W |
+This fork takes a different approach. Vectors are quantized offline to 256 archetypes. The pairwise distances between all archetypes are precomputed into a 256x256 table (`DistanceMatrix`, u16 entries, 128 KB). At query time, a cosine lookup reduces to a single table read.
-### In Context: GPU and FAISS
+### Measured
-| System | Method | Throughput | Hardware | Power |
-|--------|--------|-----------|----------|-------|
-| This fork (i7-11700K) | Palette u8 lookup | 2,400M/s | CPU | 65W |
-| FAISS GPU (IVF-PQ) | CUDA quantized | 200-500M/s | RTX 3060 | 170W |
-| FAISS GPU (cuVS) | CUDA optimized | 1,000-2,000M/s | H100 80GB | 700W |
-| FAISS CPU (Flat) | AVX2 FP32 dot | ~50M/s | i7 | 65W |
-| FAISS CPU (IVF-PQ) | AVX2 quantized | 100-200M/s | i7 | 65W |
+| Operation | Host | Result |
+|-----------|------|--------|
+| `DistanceMatrix::distance`, random pairs | Xeon @ 2.8 GHz (Cascade Lake class, AVX-512 + VNNI), 1 thread | 0.84 ns, ~1.19 G lookups/s |
+| `Base17::l1`, 20,000 candidates | same | 3.04 ns each, 60.7 µs total |
-> **On methodology:** All figures are per complete query — one vector in, one similarity score out. Both approaches require one-time offline preparation. The difference: a palette lookup is a u8 memory read (0 FLOPs); FAISS PQ decodes 8 subspaces (~16 ops); FAISS Flat computes a full 768-dimensional dot product (~1,536 FLOPs). The approximation error at the Foveal tier (1/40 sigma) is 0.4% — lower than the typical 5-10% of PQ configurations.
+Earlier revisions of this page listed per-platform rates (Sapphire Rapids ~3.2 G/s, i7-11700K 2.4 G/s, Raspberry Pi 4 ~400 M/s, Pi Zero 2W ~80 M/s) and a comparison with FAISS CPU/GPU and cuVS. Those figures have no benchmark in this repository and the FAISS/GPU numbers were not measured here, so they are no longer quoted as results. A like-for-like FAISS comparison would need the same data, recall target and hardware.
---
## Three-Level Cascade: How the Search Actually Works
-The palette table alone does not explain how a million vectors are searched in two milliseconds. That is the job of a three-level cascade where each level is a mathematically guaranteed lower bound of the next. No level can lose a relevant result.
+The palette table alone does not explain how a million vectors are searched quickly. That is the job of a three-level cascade in which each level prunes candidates for the next. Whether a level can lose a relevant result depends on the bound it uses; that guarantee is not yet tested in this repository.
### Level 1: Hamming Sweep over Bitpacked Fingerprints
-Each vector is stored as a 256-bit fingerprint (32 bytes). Comparing two fingerprints is an XOR followed by a hardware popcount:
+Each vector is stored as a bitpacked fingerprint. The cascade below assumes 32-byte (256-bit) fingerprints; note that the crate's own `Fingerprint<256>` type is 256 *words* — 2,048 bytes. Comparing two fingerprints is an XOR followed by a popcount:
-- **AVX-512 VPOPCNTDQ**: Two fingerprints in a single cycle
+- **AVX-512 VPOPCNTDQ**: native 64-bit lane popcount where available; otherwise a VPSHUFB lookup + VPSADBW (AVX-512 BW / AVX2)
- **NEON vcntq_u8**: Per-byte popcount, native on every ARM processor
-A sweep over one million fingerprints takes about 2 milliseconds and eliminates 97-99% of candidates. The Hamming distance is a provable lower bound of cosine distance — there are no false negatives.
+Measured with `bitwise::hamming_batch_raw` on one core of the Cascade Lake host (no VPOPCNTDQ): one query against one million 32-byte fingerprints takes **15.5 ms**; against 2,048-byte `Fingerprint<256>` rows it costs 277 ns per row (~7.4 GB/s). The elimination rate depends on the data and the threshold; it is not measured here.
### Level 2: Base17 L1 Distance
-The remaining ~20,000 candidates are refined with 17-dimensional i16 vectors (34 bytes). This fits in a single AVX-512 load or two NEON loads. Cost: ~3 nanoseconds per comparison. About 200 candidates survive.
+The remaining ~20,000 candidates are refined with 17-dimensional i16 vectors (34 bytes). Measured cost: 3.04 ns per comparison (60.7 µs for 20,000). About 200 candidates survive.
### Level 3: Palette Lookup
-The ~200 finalists are scored via the precomputed 256x256 table. One read per candidate, 0.4 nanoseconds.
+The ~200 finalists are scored via the precomputed 256x256 table. One read per candidate, 0.84 ns measured.
### End-to-End: One Million Vectors to Top-K
-| Level | In | Out | Duration | Bandwidth |
-|-------|-----|-----|----------|-----------|
-| Hamming sweep | 1,000,000 | ~20,000 | ~2 ms | 32 MB |
-| Base17 L1 | 20,000 | ~200 | ~60 us | 680 KB |
-| Palette lookup | 200 | Top-K | ~0.08 us | 200 B |
-| **Total** | | | **~2.1 ms** | **~33 MB** |
+| Level | In | Out | Duration | Evidence |
+|-------|-----|-----|----------|----------|
+| Hamming sweep (32 B) | 1,000,000 | data-dependent | 15.5 ms | measured, 1 core, no VPOPCNTDQ |
+| Base17 L1 | 20,000 | ~200 | 60.7 µs | measured |
+| Palette lookup | 200 | Top-K | ~0.17 µs | 200 × 0.84 ns, derived |
-FAISS CPU Flat on the same task: ~20 ms reading ~6 GB. The cascade is ten times faster at two hundred times less bandwidth.
+At 32-byte rows the sweep runs at 2.1 GB/s, so per-row overhead, not memory bandwidth, is the limit (2,048-byte rows reach 7.4 GB/s); multi-core scaling is not measured here. An end-to-end comparison with FAISS Flat has not been run in this repository.
-### Integration with LanceDB
+### Integration with Lance
-In a Lance dataset, the cascade sweep replaces FP32 distance computation from `lance-linalg`. The scan reads the bitpacked fingerprint column, runs the hardware popcount sweep, and fetches full vectors only for the few survivors.
+The cascade is a substrate path, not a Lance index. In [lance-graph](https://github.com/AdaWorldAPI/lance-graph), bit-vector Hamming distance is exposed as the DataFusion UDF `hamming_distance` (calling `bitwise::hamming_distance_raw`). It is **not** wired into Lance's ANN search, which still uses `lance-linalg` distances and returns an error for a Hamming metric. Nothing here replaces `lance-linalg` inside a Lance scan.
---
@@ -89,36 +78,34 @@ In a Lance dataset, the cascade sweep replaces FP32 distance computation from `l
Upstream ndarray delegates matrix multiplication to the external `matrixmultiply` crate, which can use AVX2. It has no own SIMD types or hardware detection. On ARM, upstream falls back to scalar code.
-This fork implements a complete SIMD layer with runtime detection:
+This fork implements its own SIMD layer: 27 portable vector/mask types selected at compile time (AVX-512, AVX2, NEON, WASM SIMD128, scalar, or nightly `core::simd`), plus runtime-dispatched kernels across 7 tiers (`amx_int8 > avx512vnni > avx512f > avxvnni > avx2_fma > neon > scalar`). Each tier is gated on the instruction feature its kernel needs; the `avxvnni` tier (VEX `VPDPBUSD`) is gated on AVX-VNNI. That tier could not be executed on the measuring host, which has AVX-512 VNNI but not AVX-VNNI, and its kernel is checked by its emitted instruction encoding only.
-| ISA | Upstream | This Fork | Speedup |
-|-----|----------|-----------|---------|
-| AVX-512 (16 x f32) | Scalar | Native __m512 types | ~8x |
-| AVX-512 VNNI (int8) | Scalar | 64 MACs/instruction | ~32x |
-| AVX-512 VPOPCNTDQ | Scalar | Native 512-bit popcount | ~16x |
-| AMX (256 MACs) | Not available | Inline asm on stable Rust | ~128x |
-| AVX2 + FMA (8 x f32) | External (matrixmultiply) | Goto-GEMM + dispatch | ~4x |
-| NEON (4 x f32) | Scalar | 3-tier: A53/A72/A76 | ~4x |
-| NEON dotprod (ARMv8.2) | Not available | vdotq_s32 (Pi 5) | ~16x |
+What the layer buys is measured per operation, against a named baseline, on one core of the Cascade Lake host (median of 15 runs, 1 M elements). Each timing pair lists the fork first and the baseline second:
-Detection happens once on first access via `LazyLock` — a single CPUID call, then only a pointer dereference per function call (0.3 ns instead of 1-3 ns for repeated feature queries).
+| Operation | Baseline | Fork | Ratio |
+|-----------|----------|------|-------|
+| u8 compare → bitmask (`simd::eq_u8_to_mask`) | plain Rust loop | 0.029 vs 0.088 ns/elem | 3.1× |
+| f32 → BF16 RNE (`f32_to_bf16_batch_rne`) | scalar per element | 0.196 vs 1.62 ns/elem | 8.3× |
+| masked i32 sum (`masked_sum_i32`, 50% density) | plain bit-test loop | 0.53 vs 0.79 ns/elem | 1.5× |
+| f32 sum (`F32x16` + `reduce_sum`) | sequential `iter().sum()` | 0.128 vs 1.26 ns/elem | 9.8× |
+| int8 GEMM u8×i8→i32 256³ (`gemm_u8_i8`, VNNI) | `int8_gemm_i32` (scalar) | 22.5 vs 5.9 GMAC/s | 3.8× |
+| AMX INT8 GEMM 2048³ | scalar | 169.7 GMAC/s | 600× (Emerald Rapids, [`AMX_GOTCHAS.md`](.claude/AMX_GOTCHAS.md)) |
+
+Where plain Rust already autovectorizes, the polyfill matches it rather than beating it: fused multiply-add, chunked f32 sums, 64-bit popcount and `popcount(a&b&c)` all land within ±20% of the plain loop (LLVM emits the same VPSHUFB popcount and VPTERNLOGQ). Instruction width (16 f32 lanes, 64 VNNI MACs) is a ceiling, not a speedup.
+
+Detection happens once via `LazyLock`. On this host a repeated `is_x86_feature_detected!` costs ~0.34 ns and a `simd_caps()` copy ~0.62 ns, so the gain of freezing dispatch is predictable dispatch and one decision per process, not a large per-call saving.
### GEMM Performance
-| Matrix Size | Upstream | This Fork | NumPy (OpenBLAS) | GPU (RTX 3060) |
-|-------------|----------|-----------|------------------|----------------|
-| 512 x 512 | ~20 GFLOPS | 47 GFLOPS | ~45 GFLOPS | ~1,200 GFLOPS |
-| 1024 x 1024 | ~13 GFLOPS | 139 GFLOPS | ~120 GFLOPS | ~3,500 GFLOPS |
-| 2048 x 2048 | ~13 GFLOPS | ~150 GFLOPS | ~140 GFLOPS | ~5,000 GFLOPS |
-
-> **Provenance of this table is unverified.** The numbers predate the current
-> tree and the benchmark that produced them is not in the repository, so the
-> API and element type they measured cannot be identified. Do not cite them as
-> a fork-vs-upstream result until they are reproduced. What the code does say:
->
-> - `Array::dot()` calls `matrixmultiply::sgemm`/`dgemm` (`src/linalg/impl_linalg.rs:503,522`) — for f32 this is the **same engine** `backend::native::gemm_f32` uses (`src/backend/native.rs:220`), so on that path there is no fork-vs-upstream engine difference to attribute a speedup to.
-> - `matrixmultiply` implements Goto-style cache blocking with microkernels, so "upstream has no tiling/microkernel" is false.
-> - The one genuinely fork-local GEMM kernel is `simd::gemm_f64_tiled` (`simd_ops.rs:947`) — fixed `TILE=64`, `F64x8` register accumulation, reached via `backend::native::gemm_f64` / `BlasLevel3::blas_gemm`, **not** via `Array::dot()`.
+Measured on one core of the Cascade Lake host (best of 3–7 runs, `matrixmultiply` threading off):
+
+| Matrix size | `Array::::dot` | `Array::::dot` | `simd::gemm_f64_tiled_fma` |
+|-------------|--------------------|--------------------|---------------------------|
+| 512 × 512 | 70.8 GFLOPS | 34.3 GFLOPS | 9.7 GFLOPS |
+| 1024 × 1024 | 70.1 GFLOPS | 34.3 GFLOPS | 9.1 GFLOPS |
+| 2048 × 2048 | 65.0 GFLOPS | 32.7 GFLOPS | — |
+
+`Array::dot()` calls `matrixmultiply::sgemm`/`dgemm` (`src/linalg/impl_linalg.rs:503,522`) — the same engine upstream uses, so these columns are not a fork-vs-upstream comparison. The fork-local `gemm_f64_tiled_fma` (fixed `TILE=64`, `F64x8` accumulation) is currently ~3.6× slower than `matrixmultiply` on f64. An earlier table on this page (fork 47/139/~150 GFLOPS vs upstream 13–20, plus NumPy and RTX 3060 columns) had no benchmark in the repository and has been withdrawn.
`simd_ops::array_chunks` walks a slice as non-overlapping `&[T; N]` windows; `array_windows` is the overlapping counterpart (a stable-Rust equivalent of nightly `slice::array_windows::()`). Both pin the window size at the call site so it feeds `F32x16::from_array` / `F64x8::from_array` directly, and both drop the per-element bounds check a dynamically-indexed loop pays. Current in-crate call sites: `hpc::blake3` (64-byte block chunking) and `heel_f64x8::cosine_f32_to_f64_simd`, both via `array_chunks`; `array_windows`, `array_windows_checked`, and `array_chunks_checked` are exported but have no in-crate production caller yet. They are the traversal primitive the hand-rolled BLAS-graph/bgz17 kernels are built on, where the const-generic window landed close to a Cranelift-JIT'd inner loop without paying for a JIT — see `src/simd_ops.rs` module docs.
@@ -137,49 +124,38 @@ Rust's `f16` type is nightly-only (issue #116909). The fork uses the same approa
## Seven Things Nobody Else Does on Stable Rust
-**1. Complete std::simd polyfill.** Rust's portable SIMD API has been nightly-only for years. This fork implements the same type surface — F32x16, F64x8, U8x64, masks, reductions, comparisons — using stable core::arch intrinsics. When std::simd stabilizes, one `use` line changes.
+**1. A std::simd-shaped polyfill on stable.** Rust's portable SIMD API has been nightly-only for years. This fork implements a `std::simd`-style type surface — 27 types including F32x16, F64x8, U8x64, masks, reductions and comparisons — on stable `core::arch`, with a nightly `core::simd` backend behind the `nightly-simd` feature and bit-exact parity crates in CI (`simd-masking-parity`, run under AVX-512, AVX2, NEON via qemu and wasm). It is not the complete `std::simd` API, and a method present on one backend is not guaranteed on all; the parity program is what checks that, and the `F64x8` comparisons were missing on AVX2 and NEON until they were added to it.
**2. f16 without nightly.** Carrier type u16 plus hardware instructions: F16C (VCVTPH2PS/VCVTPS2PH) on x86, FCVTL/FCVTN via asm!() on ARM. Three precision levels: plain f16 (10-bit mantissa), scaled-f16 (range-optimized, 1.5x more precise), double-f16 (hi+lo pair, ~20-bit effective).
-**3. AMX on stable Rust.** Intel's Advanced Matrix Extensions (TDPBUSD: 16x16 tile, 256 MACs per instruction) are nightly-only as Rust intrinsics (issue #126622). The fork emits instructions directly as asm!(".byte ...") — verified working on Rust 1.94 with kernel 6.18+.
+**3. AMX on stable Rust.** Intel's Advanced Matrix Extensions (TDPBUSD: a 16×16 tile of outputs over K=64 bytes, 16,384 MACs per instruction) are nightly-only as Rust intrinsics (issue #126622). The fork emits them through `asm!` — all four INT8 forms (`tdpb{ss,su,us,uu}d`), BF16, FP16 and FP8 — and measured 169.7 GMAC/s single-threaded on INT8 2048³ (Emerald Rapids, kernel 6.18.5, [`AMX_GOTCHAS.md`](.claude/AMX_GOTCHAS.md)).
-**4. Tiered ARM NEON.** Three tiers with runtime detection: A53 baseline (Pi Zero 2W, Pi 3 — single NEON pipeline), A72 fast (Pi 4, Orange Pi 4 — dual pipeline, 2x unrolling), A76 dotprod (Pi 5, Orange Pi 5 — vdotq_s32, native fp16). big.LITTLE systems (RK3399, RK3588) handled correctly.
+**4. Tiered ARM NEON detection.** Three tiers with runtime detection (the portable NEON types are used on aarch64; the dotprod/BF16 kernel stubs and the `simd_dispatch` table still route to scalar wrappers): A53 baseline (Pi Zero 2W, Pi 3 — single NEON pipeline), A72 fast (Pi 4, Orange Pi 4 — dual pipeline, 2x unrolling), A76 dotprod (Pi 5, Orange Pi 5 — vdotq_s32, native fp16). big.LITTLE systems (RK3399, RK3588) handled correctly.
-**5. Frozen dispatch at 0.3 ns per call.** Typical SIMD code checks every call: `if is_x86_feature_detected!("avx512f") { ... }` — an atomic load plus branch. This fork detects once and freezes a function pointer table (LazyLock, Copy struct). After that: one indirect call, no atomic, no branch prediction miss.
+**5. Frozen dispatch.** The fork detects CPU features once and freezes a function-pointer table (`LazyLock`), so every later call takes the same indirect path. The per-call saving is small on current CPUs — a cached `is_x86_feature_detected!` already costs ~0.34 ns here — the value is one decision per process and a single place that names the selected tier.
-**6. BF16 conversion bit-exact with hardware.** The function f32_to_bf16_batch_rne() implements the IEEE 754 RNE algorithm using pure AVX-512-F instructions, matching Intel's VCVTNEPS2BF16 bit-for-bit. Verified against hardware output on over one million inputs, including subnormals, infinity, NaN, and halfway ties.
+**6. BF16 conversion bit-exact with hardware.** The function f32_to_bf16_batch_rne() implements the IEEE 754 RNE algorithm using pure AVX-512-F instructions, matching Intel's VCVTNEPS2BF16 bit-for-bit. Checked against the scalar RNE reference and an independent f64 oracle on **all 4,294,967,296 f32 bit patterns: 0 mismatches** (`cargo run --release --example bf16_rne_exhaustive`, 11.5 s on 4 threads of the Cascade Lake host). The comparison is exact u16 equality, so NaN sign and payload bits count. A unit test additionally compares against the hardware `VCVTNEPS2BF16` on hosts with AVX-512-BF16; the measuring host has none, so that comparison was not run here.
-**7. Cognitive codec stack.** Beyond classical numerics, the fork implements a complete encoding pipeline: Fingerprint<256> (VSA, SIMD Hamming), Base17 (17-dimensional i16 vectors), CAM-PQ (product quantization with compiled distance tables), palette semiring (256x256 distance matrices for O(1) lookups), bgz7/bgz17 (compressed model weight format: 201 GB BF16 safetensors to 685 MB bgz7).
+**7. Cognitive codec stack.** Beyond classical numerics, the fork implements a complete encoding pipeline: Fingerprint<256> (VSA, SIMD Hamming), Base17 (17-dimensional i16 vectors), CAM-PQ (product quantization with compiled distance tables), palette semiring (256x256 distance matrices for O(1) lookups), bgz7/bgz17 (compressed model weight format; a 201 GB BF16 → 685 MB conversion was reported for the release artifacts in lance-graph, not reproduced in this repository).
---
## Codebook Inference: Token Generation Without GPU
-Beyond vector search, the fork uses the same table approach for LLM inference. Instead of matrix multiplication (`y = W*x`), a precomputed codebook is indexed (`y = codebook[index[x]]`) — O(1) per token.
-
-| Hardware | ISA | Tokens/s | Latency (50 tokens) | Power |
-|----------|-----|----------|---------------------|-------|
-| Sapphire Rapids | AMX | 380,000 | 0.13 ms | 250W |
-| Xeon (AVX-512 VNNI) | VNNI | 10,000-50,000 | 1-5 ms | 150W |
-| Raspberry Pi 5 | NEON + dotprod | 2,000-5,000 | 10-25 ms | 5W |
-| Raspberry Pi 4 | NEON (dual) | 500-2,000 | 25-100 ms | 5W |
-
-At 5 watts, a Pi 4 generates a 50-token voice assistant response in under 100 milliseconds.
+Beyond vector search, the fork uses the same table approach for LLM inference. Instead of matrix multiplication (`y = W*x`), a precomputed codebook is indexed (`y = codebook[index[x]]`) — O(1) per token. A tokens-per-second table previously shown here (AMX 380,000 tok/s down to Pi 4 500–2,000 tok/s) had no benchmark in the repository and has been withdrawn until it is reproduced.
---
## f16 Weight Transcoding
-Tested with a 15 million parameter model (Piper TTS scale):
+Measured on one core of the Cascade Lake host (F16C), 15 million Gaussian weights (σ = 0.02):
-| Format | Size | Maximum Error | RMSE | Throughput |
+| Format | Size | Maximum error | RMSE | Throughput |
|--------|------|---------------|------|------------|
| f32 (original) | 60 MB | — | — | — |
-| f16 (IEEE 754) | 30 MB | 7.3 x 10^-6 | 2.5 x 10^-6 | 94M params/s |
-| Scaled-f16 | 30 MB | 4.9 x 10^-6 | 2.1 x 10^-6 | 91M params/s |
-| Double-f16 | 60 MB | 5.7 x 10^-8 | 1.8 x 10^-8 | 42M params/s |
+| f16 (`cast_f32_to_f16_batch`) | 30 MB | 3.1 × 10⁻⁵ | 4.2 × 10⁻⁶ | 1,805 M params/s |
-With AVX2 F16C hardware: ~500 million parameters per second (8 conversions per clock cycle).
+Error depends on the weight distribution; scaled-f16 and double-f16 are available for tighter error. An earlier table (94/91/42 M params/s) had no benchmark in the repository.
---
@@ -190,7 +166,7 @@ use ndarray::Array2;
use ndarray::hpc::simd_caps::simd_caps;
let a = Array2::::ones((1024, 1024));
-let c = a.dot(&a); // AVX-512 / AVX2 / NEON — automatic
+let c = a.dot(&a); // matrixmultiply, as upstream
let caps = simd_caps();
if caps.avx512f { println!("AVX-512 active"); }
@@ -215,13 +191,13 @@ cargo build --release --target aarch64-unknown-linux-gnu
# Maximum performance on AVX-512 server
cargo --config .cargo/config-v4.toml build --release
-# Run 880 HPC tests
-cargo test
+# Library tests (2,534 at f2c1aea)
+cargo test --lib
```
## Requirements
-- Rust 1.94 stable (no nightly, no unstable features)
+- Rust 1.98.1 stable (pinned in `rust-toolchain.toml`; no nightly, no unstable features)
- Optional: gcc-aarch64-linux-gnu for Pi cross-compilation
- Optional: Intel MKL or OpenBLAS (feature-gated)
@@ -248,13 +224,22 @@ Consumers building `default-features = false` (no `std`, e.g. the
`thumbv6m-none-eabi` nostd target) skip the `hpc` module and the BLAKE3
code with it, so the nostd link is unaffected.
+## Evidence for the numbers on this page
+
+| Number | Evidence |
+|--------|----------|
+| 100 HPC modules, 2,534 lib tests, ~205k added lines / 424 files | counted at `f2c1aea` (`src/hpc/mod.rs`; `cargo test --lib`; path diff against rust-ndarray `bd3ade9`) |
+| 0.84 ns palette lookup, 3.04 ns Base17 L1, 15.5 ms 1 M sweep, SIMD ratios, GEMM, f16 | measured on one core of a Xeon @ 2.8 GHz (family 6 model 85, AVX-512 F/BW/VL/DQ/CD + VNNI; no AMX, no AVX-512-BF16, no VPOPCNTDQ), Rust 1.98.1, `target-cpu=native`, median of 15 runs unless stated |
+| BF16 RNE, all 2³² inputs, 0 mismatches | `examples/bf16_rne_exhaustive.rs`: every u32 bit pattern in 4 contiguous ranges, 65,536-input batches through the AVX-512F path; exact u16 comparison against `f32_to_bf16_scalar_rne` and an independent f64 nearest-value oracle (quiet-forced NaN, DAZ, ties to even); order-independent output checksum `0x5cd3eaa07f7f8080` (same for 2 and 4 threads); 11.5 s, Rust 1.98.1. A deliberately broken oracle (ties away from zero) reports 32,512 mismatches, so the check can fail |
+| AMX 169.7 GMAC/s, 600× scalar | measured on Emerald Rapids, [`AMX_GOTCHAS.md`](.claude/AMX_GOTCHAS.md) |
+
## Ecosystem
This fork is the hardware foundation for a larger architecture:
| Repository | Purpose |
|------------|---------|
-| [lance-graph](https://github.com/AdaWorldAPI/lance-graph) | Graph query engine, Cypher parser, codec stack |
+| [lance-graph](https://github.com/AdaWorldAPI/lance-graph) | Cypher/SQL engine on DataFusion, the Quack columnar query surface, codec stack. Owns graph, query and end-to-end benchmarks; this repository owns kernel and microbenchmark numbers |
| [home-automation-rs](https://github.com/AdaWorldAPI/home-automation-rs) | Smart home with voice AI, MCP server, MQTT |
## License
diff --git a/crates/burn/src/ops/matmul.rs b/crates/burn/src/ops/matmul.rs
index b3ae7339..a4717eef 100644
--- a/crates/burn/src/ops/matmul.rs
+++ b/crates/burn/src/ops/matmul.rs
@@ -375,9 +375,9 @@ pub fn build_distance_table_vnni(centroids_u8: &[u8], k: usize, dim: usize) -> V
// Tier 2: avx512vnni VPDPBUSD zmm (512-bit) 64 MACs/instr Cascade Lake+, Zen 4+
// Stable detection: is_x86_feature_detected!("avx512vnni")
//
- // Tier 1: avxvnniint8 VPDPBSSD ymm (256-bit) ~32 MACs/instr Sierra Forest+, Arrow Lake+
- // VNNI2: signed×signed dot product. Stable detection on Rust 1.94.
- // TODO: implement ymm-width kernel when hardware available.
+ // Tier 1: avxvnni VEX VPDPBUSD ymm (256-bit) ~32 MACs/instr Alder Lake+, Sierra Forest
+ // u8×i8 dot product (`simd_amx::vnni2_dot_u8_i8`). NOT avxvnniint8,
+ // which is VPDPBSSD/VPDPBUUD and absent on Alder Lake.
//
// Tier 0: Scalar loop 1 MAC/iter any CPU
//
@@ -389,8 +389,8 @@ pub fn build_distance_table_vnni(centroids_u8: &[u8], k: usize, dim: usize) -> V
3 // AMX present — use avx512vnni as bridge
} else if is_x86_feature_detected!("avx512vnni") {
2 // AVX-512 VNNI: 64 MACs/instr
- } else if is_x86_feature_detected!("avxvnniint8") {
- 1 // VNNI2: signed i8×i8 (ymm, ~32 MACs) — TODO: needs ymm kernel
+ } else if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("avxvnni") {
+ 1 // AVX-VNNI: VEX VPDPBUSD ymm, u8×i8 (~32 MACs)
} else {
0
}
@@ -408,10 +408,10 @@ pub fn build_distance_table_vnni(centroids_u8: &[u8], k: usize, dim: usize) -> V
#[cfg(not(target_arch = "x86_64"))]
ndarray::simd_amx::vnni_dot_u8_i8_scalar(a, b)
},
- // Tier 1: avxvnniint8 — ymm-width VPDPBUSD (32 MACs/instr)
- // For NUC 14 i9-185H (Arrow Lake) and similar non-AVX-512 CPUs
+ // Tier 1: avxvnni — ymm-width VEX VPDPBUSD (32 MACs/instr)
+ // For Alder Lake and later non-AVX-512 client CPUs
1 => |a, b| {
- // SAFETY: avxvnniint8 confirmed via is_x86_feature_detected above
+ // SAFETY: avx2 + avxvnni confirmed via is_x86_feature_detected above
#[cfg(target_arch = "x86_64")]
unsafe { ndarray::simd_amx::vnni2_dot_u8_i8(a, b) }
#[cfg(not(target_arch = "x86_64"))]
diff --git a/crates/simd-masking-parity/src/lib.rs b/crates/simd-masking-parity/src/lib.rs
index 96ee9614..dba9f5d8 100644
--- a/crates/simd-masking-parity/src/lib.rs
+++ b/crates/simd-masking-parity/src/lib.rs
@@ -27,7 +27,8 @@
//! the index-addressed permutation/scatter family); `0xD3x` the
//! index-addressed `masked_group_sum_i32_via` (two-hop zero-fallback); `0xD4x`
//! `eq_u32_via_to_mask` (the same index lane, packed as a predicate rather than
-//! folded into a sum). `main.rs` (native / qemu) and
+//! folded into a sum); `0xExx` the `F64x8` lane compares (all six relations
+//! over every pair of 16 IEEE edge values: NaN, ±0, ±inf, subnormal). `main.rs` (native / qemu) and
//! `selfcheck()` (the wasm cdylib export, driven by `run.mjs`) both call
//! [`run`].
@@ -43,11 +44,11 @@ use ndarray::simd::{
masked_strided_group_sum, masked_sum_i32, masked_sum_wrapping_add_i32, ne_i32_to_mask, ne_i32_to_mask_under,
ne_u32_to_mask, ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask, ternary_match_strided_to_mask,
ternary_match_u32_to_mask, ternary_match_u32_to_mask_under, ternary_match_u64_to_mask,
- ternary_match_u64_to_mask_under, ternlog, I32x16, KeyRunCarry, MortonDir, U32x16, U64x8,
+ ternary_match_u64_to_mask_under, ternlog, F64x8, I32x16, KeyRunCarry, MortonDir, U32x16, U64x8,
};
/// Number of check groups [`run`] executes (for the log line only).
-pub const CHECKS: usize = 13;
+pub const CHECKS: usize = 14;
/// The wasm export: identical to [`run`], `extern "C"` so `run.mjs` can call it.
#[no_mangle]
@@ -61,6 +62,7 @@ pub fn run() -> u32 {
check_ternlog_all_tables, check_u64x8_algebra, check_i32x16_compare, check_predicates_to_mask,
check_mask_algebra, check_care_match, check_masked_reductions, check_blend, check_morton_shift,
check_predicates_under, check_set_range, check_unsigned_compare_to_mask, check_gather_scatter_group,
+ check_f64x8_compare,
];
for g in groups {
if let Err(code) = g() {
@@ -393,6 +395,94 @@ fn check_i32x16_compare() -> Result<(), u32> {
Ok(())
}
+// ── 0xExx: F64x8 lane compares — every pair of IEEE edge values ─────────────
+//
+// All six relations against plain scalar `f64` operators. The edge set covers
+// NaN (positive, negative, payload-carrying), ±0, ±inf, the extremes and a
+// subnormal. Expected semantics are IEEE-754 / Rust's own: `==`,`<`,`>`,`<=`,
+// `>=` are ORDERED (false if either operand is NaN), `!=` is UNORDERED (true
+// if either is NaN), and `+0.0 == -0.0`. The mask is read back through
+// `select`, the only accessor every realization of `F64Mask8` shares.
+
+fn f64_mask_bits(m: impl FnOnce(F64x8, F64x8) -> F64x8) -> u8 {
+ let lanes = m(F64x8::splat(1.0), F64x8::splat(0.0)).to_array();
+ let mut bits = 0u8;
+ for (i, &v) in lanes.iter().enumerate() {
+ if v == 1.0 {
+ bits |= 1 << i;
+ }
+ }
+ bits
+}
+
+fn check_f64x8_compare() -> Result<(), u32> {
+ let edges: [f64; 16] = [
+ f64::NAN,
+ -f64::NAN,
+ f64::from_bits(0x7FF0_0000_0000_0001), // signalling-pattern NaN with payload
+ 0.0,
+ -0.0,
+ f64::INFINITY,
+ f64::NEG_INFINITY,
+ f64::MAX,
+ f64::MIN,
+ f64::MIN_POSITIVE,
+ f64::from_bits(1), // smallest subnormal
+ 1.0,
+ -1.0,
+ 1.0 + f64::EPSILON,
+ 0.5,
+ 1.0,
+ ];
+ let mut pairs = 0u32;
+ for chunk in 0..(16 * 16 / 8) {
+ let mut a_arr = [0.0f64; 8];
+ let mut b_arr = [0.0f64; 8];
+ for lane in 0..8 {
+ let k = chunk * 8 + lane;
+ a_arr[lane] = edges[k / 16];
+ b_arr[lane] = edges[k % 16];
+ }
+ let (a, b) = (F64x8::from_array(a_arr), F64x8::from_array(b_arr));
+ let expect = |rel: fn(f64, f64) -> bool| -> u8 {
+ let mut bits = 0u8;
+ for lane in 0..8 {
+ if rel(a_arr[lane], b_arr[lane]) {
+ bits |= 1 << lane;
+ }
+ }
+ bits
+ };
+ let got = [
+ f64_mask_bits(|t, f| a.simd_eq(b).select(t, f)),
+ f64_mask_bits(|t, f| a.simd_ne(b).select(t, f)),
+ f64_mask_bits(|t, f| a.simd_lt(b).select(t, f)),
+ f64_mask_bits(|t, f| a.simd_le(b).select(t, f)),
+ f64_mask_bits(|t, f| a.simd_gt(b).select(t, f)),
+ f64_mask_bits(|t, f| a.simd_ge(b).select(t, f)),
+ ];
+ let want = [
+ expect(|x, y| x == y),
+ expect(|x, y| x != y),
+ expect(|x, y| x < y),
+ expect(|x, y| x <= y),
+ expect(|x, y| x > y),
+ expect(|x, y| x >= y),
+ ];
+ for rel in 0..6 {
+ if got[rel] != want[rel] {
+ return Err(0xE00 | rel as u32);
+ }
+ }
+ pairs += 8;
+ }
+ // Anti-vacuity: every one of the 16 x 16 edge pairs was checked.
+ if pairs != 256 {
+ return Err(0xE0F);
+ }
+ Ok(())
+}
+
// ── 0x5xx: predicate → mask, every tail shape, full-overwrite + zero tail ────
fn check_predicates_to_mask() -> Result<(), u32> {
diff --git a/examples/bf16_rne_exhaustive.rs b/examples/bf16_rne_exhaustive.rs
new file mode 100644
index 00000000..be8980ab
--- /dev/null
+++ b/examples/bf16_rne_exhaustive.rs
@@ -0,0 +1,139 @@
+//! Exhaustive F32 -> BF16 round-to-nearest-even parity: ALL 2^32 f32 bit patterns.
+//!
+//! Reproduces the README claim "4,294,967,296 inputs, 0 mismatches". For every
+//! `u32` bit pattern `b`, `simd::f32_to_bf16_batch_rne` is compared bit-for-bit
+//! (u16 equality, so NaN payloads and signs count) against
+//!
+//! 1. `simd::f32_to_bf16_scalar_rne`, the repo's scalar reference, and
+//! 2. an INDEPENDENT oracle defined here: pick whichever of the two adjacent
+//! BF16 values is nearer in f64, ties to the even mantissa; overflow past
+//! BF16::MAX rounds to infinity at the 2^128 midpoint.
+//!
+//! The intended semantics are Intel SDM `VCVTNEPS2BF16`:
+//! - NaN keeps its sign and top 7 payload bits, and the quiet bit is forced;
+//! - subnormal input flushes to a signed zero (DAZ);
+//! - infinities and zeros pass through.
+//!
+//! The oracle encodes exactly that and nothing else.
+//!
+//! Method: the 2^32 space is split into `threads` equal contiguous ranges. Each
+//! range is enumerated in chunks of 65,536 inputs; each chunk is converted in
+//! one batch call. An order-independent checksum of every converted output
+//! (Σ out(b)·(b|1) mod 2^64) is printed so the work cannot be optimized away
+//! and runs with different thread counts can be compared.
+//!
+//! Run (x86_64; the batch takes the AVX-512F path when the host has it):
+//! ```text
+//! cargo run --release --example bf16_rne_exhaustive [threads]
+//! ```
+
+#[cfg(target_arch = "x86_64")]
+fn oracle_bf16(bits: u32) -> u16 {
+ let exp = bits & 0x7F80_0000;
+ let mant = bits & 0x007F_FFFF;
+ if exp == 0x7F80_0000 && mant != 0 {
+ return ((bits >> 16) as u16) | 0x0040; // NaN: keep sign + top payload, force quiet
+ }
+ if exp == 0 {
+ return ((bits >> 16) as u16) & 0x8000; // zero or subnormal -> signed zero
+ }
+ if exp == 0x7F80_0000 {
+ return (bits >> 16) as u16; // infinity
+ }
+ let x = f32::from_bits(bits) as f64;
+ let lo = (bits >> 16) as u16; // truncation: toward zero
+ let hi = lo + 1; // next magnitude up, same sign
+ let f_lo = f32::from_bits((lo as u32) << 16) as f64;
+ let hi_bits = (hi as u32) << 16;
+ let d_lo = (x - f_lo).abs();
+ let d_hi = if (hi_bits & 0x7F80_0000) == 0x7F80_0000 {
+ // `hi` is infinity: the rounding midpoint to infinity is 2^128 - 2^119.
+ 2f64.powi(128) - x.abs()
+ } else {
+ (f32::from_bits(hi_bits) as f64 - x).abs()
+ };
+ if d_lo < d_hi || (d_lo == d_hi && lo & 1 == 0) {
+ lo
+ } else {
+ hi
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn main() {
+ use ndarray::simd::{f32_to_bf16_batch_rne, f32_to_bf16_scalar_rne};
+ use std::time::Instant;
+
+ let threads: u64 = std::env::args()
+ .nth(1)
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(4);
+ assert!(threads.is_power_of_two() && threads <= 64, "threads must be a power of two <= 64");
+ let path = if is_x86_feature_detected!("avx512f") {
+ "AVX-512F"
+ } else {
+ "scalar fallback"
+ };
+ println!("bf16_rne_exhaustive: batch path = {path}, threads = {threads}, chunk = 65536");
+
+ let t0 = Instant::now();
+ let span = (1u64 << 32) / threads;
+ let results: Vec<(u64, u64, u64, u64, Option)> = std::thread::scope(|s| {
+ let handles: Vec<_> = (0..threads)
+ .map(|t| {
+ s.spawn(move || {
+ const CHUNK: usize = 1 << 16;
+ let mut inp = vec![0f32; CHUNK];
+ let mut out = vec![0u16; CHUNK];
+ let (mut n, mut vs_scalar, mut vs_oracle, mut sum) = (0u64, 0u64, 0u64, 0u64);
+ let mut first = None;
+ let mut base = t * span;
+ while base < (t + 1) * span {
+ for (i, x) in inp.iter_mut().enumerate() {
+ *x = f32::from_bits((base + i as u64) as u32);
+ }
+ f32_to_bf16_batch_rne(&inp, &mut out);
+ for i in 0..CHUNK {
+ let b = (base + i as u64) as u32;
+ let got = out[i];
+ // Order-independent, so the value is the same for any thread count.
+ sum = sum.wrapping_add((got as u64).wrapping_mul(b as u64 | 1));
+ if got != f32_to_bf16_scalar_rne(inp[i]) {
+ vs_scalar += 1;
+ }
+ if got != oracle_bf16(b) {
+ vs_oracle += 1;
+ first.get_or_insert(b);
+ }
+ }
+ n += CHUNK as u64;
+ base += CHUNK as u64;
+ }
+ (n, vs_scalar, vs_oracle, sum, first)
+ })
+ })
+ .collect();
+ handles.into_iter().map(|h| h.join().unwrap()).collect()
+ });
+ let n: u64 = results.iter().map(|r| r.0).sum();
+ let vs_scalar: u64 = results.iter().map(|r| r.1).sum();
+ let vs_oracle: u64 = results.iter().map(|r| r.2).sum();
+ let checksum = results.iter().fold(0u64, |a, r| a.wrapping_add(r.3));
+ println!("inputs checked : {n}");
+ println!("mismatch vs scalar : {vs_scalar}");
+ match results.iter().find_map(|r| r.4) {
+ Some(b) => println!("mismatch vs oracle : {vs_oracle} (first input 0x{b:08x})"),
+ None => println!("mismatch vs oracle : {vs_oracle}"),
+ }
+ println!("output checksum : 0x{checksum:016x}");
+ println!("elapsed : {:.1} s", t0.elapsed().as_secs_f64());
+ assert_eq!(n, 1u64 << 32, "did not cover the full input space");
+ if vs_scalar != 0 || vs_oracle != 0 {
+ std::process::exit(1);
+ }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+fn main() {
+ eprintln!("bf16_rne_exhaustive: x86_64 only (the batch path under test is AVX-512F)");
+}
diff --git a/src/simd_amx.rs b/src/simd_amx.rs
index d537e389..1dca8829 100644
--- a/src/simd_amx.rs
+++ b/src/simd_amx.rs
@@ -23,7 +23,7 @@
//! Troubleshooting playbook: `.claude/AMX_GOTCHAS.md` Agent: `amx-savant`
//!
//! Dispatch tiers (MACs/instruction): AMX 16 384 → avx512vnni 64 →
-//! avxvnniint8 32 → scalar 1.
+//! avxvnni 32 → scalar 1.
// ═══════════════════════════════════════════════════════════════════════════
// Detection (stable — just CPUID, no AMX instructions)
@@ -375,10 +375,17 @@ pub unsafe fn vnni_matvec(table: &[u8], energy_i8: &[i8], result: &mut [i32], n:
}
}
-/// AVX-VNNI (ymm, 256-bit) dot product: 32 MACs per VPDPBUSD instruction.
-/// For CPUs with avxvnniint8 but NOT avx512vnni (Arrow Lake, NUC 14 i9-185H, etc.)
+/// AVX-VNNI (ymm, 256-bit) dot product: 32 MACs per VEX-encoded VPDPBUSD.
+/// For CPUs with AVX-VNNI but NOT avx512vnni (Alder Lake and later client
+/// parts, Sierra Forest).
+///
+/// # Safety
+/// Caller must have verified `avx2` and `avxvnni` at runtime. The kernel uses
+/// `_mm256_dpbusd_avx_epi32` (VEX). It previously called
+/// `_mm256_dpbusd_epi32`, which is the AVX-512VNNI+VL (EVEX) form and raises
+/// #UD on exactly the non-AVX-512 hosts this tier exists for.
#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avxvnniint8")]
+#[target_feature(enable = "avx2,avxvnni")]
pub unsafe fn vnni2_dot_u8_i8(row: &[u8], energy: &[i8]) -> i32 {
use core::arch::x86_64::*;
let n = row.len().min(energy.len());
@@ -390,7 +397,7 @@ pub unsafe fn vnni2_dot_u8_i8(row: &[u8], energy: &[i8]) -> i32 {
let a = _mm256_loadu_si256(row[off..].as_ptr() as *const __m256i);
let b = _mm256_loadu_si256(energy[off..].as_ptr() as *const __m256i);
// VPDPBUSD ymm: 8 lanes × 4 u8×i8 products = 32 MACs
- acc = _mm256_dpbusd_epi32(acc, a, b);
+ acc = _mm256_dpbusd_avx_epi32(acc, a, b);
}
// Horizontal sum of 8 i32 lanes
@@ -411,7 +418,7 @@ pub unsafe fn vnni2_dot_u8_i8(row: &[u8], energy: &[i8]) -> i32 {
/// VNNI2 MatVec for the entire distance table × energy vector (ymm path).
#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avxvnniint8")]
+#[target_feature(enable = "avx2,avxvnni")]
pub unsafe fn vnni2_matvec(table: &[u8], energy_i8: &[i8], result: &mut [i32], n: usize) {
for i in 0..n {
let row = &table[i * n..(i + 1) * n];
@@ -437,21 +444,23 @@ pub fn vnni_matvec_scalar(table: &[u8], energy_i8: &[i8], result: &mut [i32], n:
}
}
-/// Runtime-dispatched VNNI MatVec: avx512vnni → avxvnniint8 → scalar i32.
+/// Runtime-dispatched VNNI MatVec: avx512vnni → avxvnni → scalar i32.
///
/// Three tiers, checked in order (first match wins):
/// avx512vnni — 64 MACs/instr (zmm, Cascade Lake+, Zen 4+)
-/// avxvnniint8 — 32 MACs/instr (ymm, Arrow Lake, NUC 14 i9-185H)
+/// avxvnni — 32 MACs/instr (ymm, Alder Lake+, Sierra Forest)
/// scalar i32 — only for non-x86 or testing
///
-/// IMPORTANT: avxvnniint8 (VNNI2, 256-bit) is NEVER reached when
+/// IMPORTANT: avxvnni (VEX 256-bit) is NEVER reached when
/// avx512vnni (VNNI512) is present. This is correct:
/// - CPUs with avx512vnni always have 512-bit VPDPBUSD (faster)
-/// - avxvnniint8 exists ONLY for CPUs that dropped AVX-512
-/// but added 256-bit VNNI (Arrow Lake, Meteor Lake U-series)
+/// - the avxvnni tier exists for CPUs WITHOUT AVX-512 that added
+/// 256-bit VEX VNNI (Alder Lake and later client parts)
/// - The two instructions have DIFFERENT encodings:
/// avx512vnni: EVEX-encoded VPDPBUSD zmm (512-bit)
-/// avxvnniint8: VEX-encoded VPDPBUSD ymm (256-bit)
+/// avxvnni: VEX-encoded VPDPBUSD ymm (256-bit)
+/// (`avxvnniint8` is a DIFFERENT feature: VPDPBSSD/VPDPBUUD, signed×signed
+/// and unsigned×unsigned — not what this u8×i8 kernel needs)
/// - Running EVEX VPDPBUSD on a VEX-only CPU = SIGILL
/// - Running VEX VPDPBUSD on an EVEX CPU = works but wastes half the width
///
@@ -467,7 +476,7 @@ pub fn matvec_dispatch(table: &[u8], energy_i8: &[i8], result: &mut [i32], n: us
}
return;
}
- if is_x86_feature_detected!("avxvnniint8") {
+ if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("avxvnni") {
unsafe {
vnni2_matvec(table, energy_i8, result, n);
}
diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs
index 602aa475..1364b722 100644
--- a/src/simd_avx2.rs
+++ b/src/simd_avx2.rs
@@ -982,6 +982,58 @@ impl F64x8 {
}
F64Mask8(bits)
}
+ /// Lane-wise `==` — ordered: false if either lane is NaN; `+0.0 == -0.0` (AVX-512 `_CMP_EQ_OQ`).
+ #[inline(always)]
+ pub fn simd_eq(self, other: Self) -> F64Mask8 {
+ let a = self.to_array();
+ let b = other.to_array();
+ let mut bits: u8 = 0;
+ for i in 0..8 {
+ if a[i] == b[i] {
+ bits |= 1 << i;
+ }
+ }
+ F64Mask8(bits)
+ }
+ /// Lane-wise `!=` — unordered: true if either lane is NaN (AVX-512 `_CMP_NEQ_UQ`).
+ #[inline(always)]
+ pub fn simd_ne(self, other: Self) -> F64Mask8 {
+ let a = self.to_array();
+ let b = other.to_array();
+ let mut bits: u8 = 0;
+ for i in 0..8 {
+ if a[i] != b[i] {
+ bits |= 1 << i;
+ }
+ }
+ F64Mask8(bits)
+ }
+ /// Lane-wise `<` — ordered: false if either lane is NaN (AVX-512 `_CMP_LT_OS`).
+ #[inline(always)]
+ pub fn simd_lt(self, other: Self) -> F64Mask8 {
+ let a = self.to_array();
+ let b = other.to_array();
+ let mut bits: u8 = 0;
+ for i in 0..8 {
+ if a[i] < b[i] {
+ bits |= 1 << i;
+ }
+ }
+ F64Mask8(bits)
+ }
+ /// Lane-wise `>` — ordered: false if either lane is NaN (AVX-512 `other.simd_lt(self)`).
+ #[inline(always)]
+ pub fn simd_gt(self, other: Self) -> F64Mask8 {
+ let a = self.to_array();
+ let b = other.to_array();
+ let mut bits: u8 = 0;
+ for i in 0..8 {
+ if a[i] > b[i] {
+ bits |= 1 << i;
+ }
+ }
+ F64Mask8(bits)
+ }
#[inline(always)]
pub fn to_bits(self) -> U64x8 {
let a = self.to_array();
diff --git a/src/simd_caps.rs b/src/simd_caps.rs
index 179b4ac0..c69ba749 100644
--- a/src/simd_caps.rs
+++ b/src/simd_caps.rs
@@ -71,6 +71,12 @@ pub struct SimdCaps {
/// (`is_x86_feature_detected!("avxvnniint8")`).
/// Present on Arrow Lake, Lunar Lake, NUC 14 (Meteor Lake-H).
pub avxvnniint8: bool,
+ /// AVX-VNNI: VEX-encoded 256-bit `VPDPBUSD`/`VPDPWSSD` (u8×i8 / i16×i16
+ /// dot product) without AVX-512 (`is_x86_feature_detected!("avxvnni")`,
+ /// CPUID.07H.1H:EAX bit 4). Present on Alder Lake and later client parts,
+ /// Sierra Forest, Zen 5. This — not `avxvnniint8` — is what the 256-bit
+ /// `u8×i8` kernel (`simd_amx::vnni2_dot_u8_i8`) requires.
+ pub avxvnni: bool,
/// AVX-512 FP16 arithmetic (CPUID.07H.0H:EDX bit 23). Native
/// `__m512h` operations (`_mm512_*_ph`). Present on Sapphire Rapids,
/// Granite Rapids, Zen 4+. Bit is exposed for downstream substrate
@@ -142,6 +148,7 @@ impl SimdCaps {
amx_bf16: false,
avx512bf16: false,
avxvnniint8: false,
+ avxvnni: false,
avx512fp16: false,
avx512vp2intersect: false,
amx_fp16: false,
@@ -193,6 +200,7 @@ impl SimdCaps {
amx_bf16,
avx512bf16: is_x86_feature_detected!("avx512bf16"),
avxvnniint8: is_x86_feature_detected!("avxvnniint8"),
+ avxvnni: is_x86_feature_detected!("avxvnni"),
avx512fp16,
avx512vp2intersect,
amx_fp16,
@@ -228,6 +236,7 @@ impl SimdCaps {
amx_bf16: false,
avx512bf16: false,
avxvnniint8: false,
+ avxvnni: false,
avx512fp16: false,
avx512vp2intersect: false,
amx_fp16: false,
@@ -260,6 +269,7 @@ impl SimdCaps {
amx_bf16: false,
avx512bf16: false,
avxvnniint8: false,
+ avxvnni: false,
avx512fp16: false,
avx512vp2intersect: false,
amx_fp16: false,
@@ -317,6 +327,12 @@ impl SimdCaps {
self.avxvnniint8
}
+ /// AVX-VNNI (VEX 256-bit `VPDPBUSD`) available.
+ #[inline(always)]
+ pub fn has_avxvnni(self) -> bool {
+ self.avxvnni
+ }
+
/// True if AVX-512 FP16 (`__m512h`) is available. Distinguishes
/// SapphireRapids-class silicon (and Zen 4+) from the CascadeLake /
/// IceLakeSp / SkylakeX baseline that lacks native `__m512h` math.
@@ -606,6 +622,7 @@ mod tests {
amx_bf16: false,
avx512bf16: false,
avxvnniint8: false,
+ avxvnni: false,
avx512fp16: false,
avx512vp2intersect: false,
amx_fp16: true,
diff --git a/src/simd_neon.rs b/src/simd_neon.rs
index 05e09127..b95b4e75 100644
--- a/src/simd_neon.rs
+++ b/src/simd_neon.rs
@@ -1021,6 +1021,58 @@ pub mod aarch64_simd {
}
F64Mask8(bits)
}
+ /// Lane-wise `==` — ordered: false if either lane is NaN; `+0.0 == -0.0` (AVX-512 `_CMP_EQ_OQ`).
+ #[inline(always)]
+ pub fn simd_eq(self, other: Self) -> F64Mask8 {
+ let a = self.to_array();
+ let b = other.to_array();
+ let mut bits: u8 = 0;
+ for i in 0..8 {
+ if a[i] == b[i] {
+ bits |= 1 << i;
+ }
+ }
+ F64Mask8(bits)
+ }
+ /// Lane-wise `!=` — unordered: true if either lane is NaN (AVX-512 `_CMP_NEQ_UQ`).
+ #[inline(always)]
+ pub fn simd_ne(self, other: Self) -> F64Mask8 {
+ let a = self.to_array();
+ let b = other.to_array();
+ let mut bits: u8 = 0;
+ for i in 0..8 {
+ if a[i] != b[i] {
+ bits |= 1 << i;
+ }
+ }
+ F64Mask8(bits)
+ }
+ /// Lane-wise `<` — ordered: false if either lane is NaN (AVX-512 `_CMP_LT_OS`).
+ #[inline(always)]
+ pub fn simd_lt(self, other: Self) -> F64Mask8 {
+ let a = self.to_array();
+ let b = other.to_array();
+ let mut bits: u8 = 0;
+ for i in 0..8 {
+ if a[i] < b[i] {
+ bits |= 1 << i;
+ }
+ }
+ F64Mask8(bits)
+ }
+ /// Lane-wise `>` — ordered: false if either lane is NaN (AVX-512 `other.simd_lt(self)`).
+ #[inline(always)]
+ pub fn simd_gt(self, other: Self) -> F64Mask8 {
+ let a = self.to_array();
+ let b = other.to_array();
+ let mut bits: u8 = 0;
+ for i in 0..8 {
+ if a[i] > b[i] {
+ bits |= 1 << i;
+ }
+ }
+ F64Mask8(bits)
+ }
#[inline(always)]
pub fn to_bits(self) -> U64x8 {
diff --git a/src/simd_runtime/cpu_ops.rs b/src/simd_runtime/cpu_ops.rs
index c567f662..725ac296 100644
--- a/src/simd_runtime/cpu_ops.rs
+++ b/src/simd_runtime/cpu_ops.rs
@@ -176,8 +176,29 @@ static CPU_OPS_SCALAR: CpuOps = CpuOps {
/// [`simd_caps`]: crate::hpc::simd_caps::simd_caps
pub fn cpu_ops() -> &'static CpuOps {
static SELECTED: LazyLock<&'static CpuOps> = LazyLock::new(|| {
- let _caps = crate::hpc::simd_caps::simd_caps();
+ let caps = crate::hpc::simd_caps::simd_caps();
+ #[cfg(target_arch = "x86_64")]
+ let amx_os_ok = caps.amx_int8 && crate::simd_amx::amx_available();
+ #[cfg(not(target_arch = "x86_64"))]
+ let amx_os_ok = false;
+ select_cpu_ops(caps, amx_os_ok)
+ });
+ *SELECTED
+}
+/// The tier ladder, as a pure function of the detected capabilities.
+///
+/// `amx_os_ok` is the OS-level AMX gate (`simd_amx::amx_available()`),
+/// passed in so the ladder itself can be tested against synthetic
+/// capability sets for CPUs this host is not.
+///
+/// Each rung's gate names the instruction feature its kernel REQUIRES:
+/// the `avxvnni` tier runs VEX `VPDPBUSD` (`_mm256_dpbusd_avx_epi32`), so it
+/// is gated on `avxvnni` — never on `avxvnniint8`, which is a different
+/// feature (VPDPBSSD/VPDPBUUD) that Alder Lake does not have.
+fn select_cpu_ops(_caps: crate::hpc::simd_caps::SimdCaps, amx_os_ok: bool) -> &'static CpuOps {
+ let _ = amx_os_ok; // read only on x86_64
+ {
#[cfg(target_arch = "x86_64")]
{
// AMX tier selection: CPUID-reports-AMX is necessary but
@@ -188,7 +209,7 @@ pub fn cpu_ops() -> &'static CpuOps {
// being set. `simd_amx::amx_available()` runs the full
// four-step gate (CPUID + OSXSAVE + XCR0 + arch_prctl);
// demote to the AVX-512 path when the OS-check fails.
- if _caps.amx_int8 && crate::simd_amx::amx_available() {
+ if _caps.amx_int8 && amx_os_ok {
return &CPU_OPS_AMX_INT8;
}
if _caps.avx512f && _caps.avx512vnni {
@@ -197,7 +218,9 @@ pub fn cpu_ops() -> &'static CpuOps {
if _caps.avx512f {
return &CPU_OPS_AVX512F;
}
- if _caps.avx2 && _caps.avxvnniint8 {
+ // The `avxvnni` table's add_mul pointers are AVX2+FMA kernels,
+ // and AVX-VNNI does not imply FMA, so FMA is required here too.
+ if _caps.avx2 && _caps.fma && _caps.avxvnni {
return &CPU_OPS_AVXVNNI;
}
if _caps.avx2 && _caps.fma {
@@ -213,8 +236,7 @@ pub fn cpu_ops() -> &'static CpuOps {
}
&CPU_OPS_SCALAR
- });
- *SELECTED
+ }
}
/// Lookup `&'static CpuOps` by tier name string. Used for explicit-
@@ -414,4 +436,74 @@ mod tests {
assert!((v - 7.0).abs() < 1e-6, "add_mul_f32 via DTO: got {v}");
}
}
+
+ /// The tier ladder must gate each rung on the feature its kernel
+ /// REQUIRES. Synthetic capability sets, so the check runs on any x86
+ /// host — including ones that are not the CPU being modelled.
+ ///
+ /// Regression: the `avxvnni` rung used to be gated on `avxvnniint8`, so
+ /// an Alder Lake (AVX-VNNI, no AVX-VNNI-INT8) fell through to
+ /// `avx2_fma` while `cpu_tier_for_cpu("alderlake")` said `avxvnni`.
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn tier_ladder_gates_on_the_kernel_feature() {
+ use crate::hpc::simd_caps::SimdCaps;
+ let base = SimdCaps {
+ avx2: true,
+ fma: true,
+ avx512f: false,
+ avx512bw: false,
+ avx512vl: false,
+ avx512vnni: false,
+ amx_tile: false,
+ amx_int8: false,
+ avxvnni: false,
+ avxvnniint8: false,
+ ..crate::hpc::simd_caps::simd_caps()
+ };
+ // Alder Lake: AVX2 + FMA + AVX-VNNI, no AVX-512, no AVX-VNNI-INT8.
+ let alderlake = SimdCaps { avxvnni: true, ..base };
+ assert_eq!(select_cpu_ops(alderlake, false).tier, "avxvnni");
+ assert_eq!(cpu_tier_for_cpu("alderlake"), Some(select_cpu_ops(alderlake, false).tier));
+ // Arrow Lake: AVX-VNNI and AVX-VNNI-INT8.
+ let arrowlake = SimdCaps {
+ avxvnni: true,
+ avxvnniint8: true,
+ ..base
+ };
+ assert_eq!(select_cpu_ops(arrowlake, false).tier, "avxvnni");
+ // AVX-VNNI-INT8 alone does not license the VEX VPDPBUSD kernel.
+ let int8_only = SimdCaps {
+ avxvnniint8: true,
+ ..base
+ };
+ assert_eq!(select_cpu_ops(int8_only, false).tier, "avx2_fma");
+ // AVX-VNNI without FMA must not select a table whose add_mul runs FMA.
+ let no_fma = SimdCaps {
+ fma: false,
+ ..alderlake
+ };
+ assert_eq!(select_cpu_ops(no_fma, false).tier, "scalar");
+ // Haswell baseline.
+ assert_eq!(select_cpu_ops(base, false).tier, "avx2_fma");
+ assert_eq!(cpu_tier_for_cpu("haswell"), Some("avx2_fma"));
+ // Cascade Lake: AVX-512 + VNNI outranks AVX-VNNI.
+ let cascadelake = SimdCaps {
+ avx512f: true,
+ avx512bw: true,
+ avx512vl: true,
+ avx512vnni: true,
+ ..base
+ };
+ assert_eq!(select_cpu_ops(cascadelake, false).tier, "avx512vnni");
+ assert_eq!(cpu_tier_for_cpu("cascadelake"), Some("avx512vnni"));
+ // AMX needs both the CPUID bit and the OS gate.
+ let spr = SimdCaps {
+ amx_tile: true,
+ amx_int8: true,
+ ..cascadelake
+ };
+ assert_eq!(select_cpu_ops(spr, true).tier, "amx_int8");
+ assert_eq!(select_cpu_ops(spr, false).tier, "avx512vnni");
+ }
}
diff --git a/src/simd_runtime/vnni_dot.rs b/src/simd_runtime/vnni_dot.rs
index 4e9ba980..aaeb12e1 100644
--- a/src/simd_runtime/vnni_dot.rs
+++ b/src/simd_runtime/vnni_dot.rs
@@ -7,7 +7,7 @@
//! `n - (n%64)` lanes and silently drops the K-tail (its current
//! matvec caller pre-aligns rows so the tail is never the bug;
//! a general-purpose dispatch surface cannot assume that).
-//! 2. **AVX-VNNI ymm** (`avx2 + avxvnniint8`) → `simd_amx::vnni2_dot_u8_i8`
+//! 2. **AVX-VNNI ymm** (`avx2 + avxvnni`) → `simd_amx::vnni2_dot_u8_i8`
//! which already includes its own tail handling.
//! 3. **Scalar** → `simd_amx::vnni_dot_u8_i8_scalar` (always correct,
//! no SIMD, the safety floor).
@@ -30,7 +30,7 @@ static VNNI_DOT_U8_I8_DISPATCH: LazyLock = LazyLock::new(|| {
if _caps.avx512f && _caps.avx512vnni {
return vnni_dot_u8_i8_avx512_with_tail as VnniDotFn;
}
- if _caps.avx2 && _caps.avxvnniint8 {
+ if _caps.avx2 && _caps.avxvnni {
return vnni2_dot_u8_i8_safe_wrapper as VnniDotFn;
}
}
@@ -101,9 +101,9 @@ unsafe fn vnni_dot_u8_i8_avx512_with_tail(a: &[u8], b: &[i8]) -> i32 {
/// the dispatch table a `VnniDotFn` pointer with the right signature.
///
/// # Safety
-/// Caller must have feature-detected `avx2 + avxvnniint8` at runtime.
+/// Caller must have feature-detected `avx2 + avxvnni` at runtime.
#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2,avxvnniint8")]
+#[target_feature(enable = "avx2,avxvnni")]
unsafe fn vnni2_dot_u8_i8_safe_wrapper(a: &[u8], b: &[i8]) -> i32 {
crate::simd_amx::vnni2_dot_u8_i8(a, b)
}
@@ -139,7 +139,7 @@ pub(super) unsafe fn vnni_dot_u8_i8_avx512_with_tail_safe(a: &[u8], b: &[i8]) ->
#[cfg(target_arch = "x86_64")]
pub(super) unsafe fn vnni2_dot_u8_i8_safe(a: &[u8], b: &[i8]) -> i32 {
- // SAFETY: dispatch closure verified avx2 + avxvnniint8.
+ // SAFETY: dispatch closure verified avx2 + avxvnni.
vnni2_dot_u8_i8_safe_wrapper(a, b)
}
@@ -151,6 +151,26 @@ pub(super) unsafe fn vnni_dot_u8_i8_scalar_wrapper(a: &[u8], b: &[i8]) -> i32 {
mod tests {
use super::*;
+ /// The 256-bit AVX-VNNI kernel against the scalar reference, called
+ /// directly (not via the dispatcher, which on an AVX-512 host never picks
+ /// it). Skips — and says so — on hosts without AVX-VNNI.
+ #[cfg(target_arch = "x86_64")]
+ #[test]
+ fn avxvnni_kernel_matches_scalar_reference() {
+ if !(is_x86_feature_detected!("avx2") && is_x86_feature_detected!("avxvnni")) {
+ eprintln!("SKIPPED: host has no AVX-VNNI; kernel not executed");
+ return;
+ }
+ for k in [0usize, 1, 31, 32, 33, 64, 100, 257] {
+ let a: Vec = (0..k).map(|i| ((i * 29 + 7) % 256) as u8).collect();
+ let b: Vec = (0..k).map(|i| ((i * 31 + 3) % 256) as u8 as i8).collect();
+ // SAFETY: avx2 + avxvnni verified above.
+ let got = unsafe { vnni2_dot_u8_i8_safe_wrapper(&a, &b) };
+ let want: i32 = (0..k).map(|i| a[i] as i32 * b[i] as i32).sum();
+ assert_eq!(got, want, "K={k}");
+ }
+ }
+
/// Verify the runtime trampoline produces the same result as the
/// scalar reference for a few representative shapes — aligned and
/// non-aligned K (which exercises the AVX-512 wrapper's scalar