# FineWeb-10B slice benchmark — Qdrant vs Milvus vs Elasticsearch, all scored against one shared exact ground truth and swept across each engine's search-effort knob for a recall/latency curve

> Source: <https://gist.github.com/andrisgauracs/f44d5dd040844f4e83ccae60f0d8ef5e>
> Published: 2026-09-24 09:29:43+00:00

|  | #!/usr/bin/env bash | 
|  | # ============================================================================= | 
|  | # FineWeb-10B slice benchmark — Qdrant vs Milvus vs Elasticsearch | 
|  | # | 
|  | # One exact ground truth (brute force over YOUR slice), reused by every engine, | 
|  | # so all three are scored against identical correct answers. Each engine is then | 
|  | # swept across its search-effort knob to produce a recall/latency CURVE rather | 
|  | # than a single point. | 
|  | # | 
|  | # ./bench.sh all # everything, Qdrant only | 
|  | # ENGINES="qdrant milvus elastic" ./bench.sh all | 
|  | # ./bench.sh probe milvus # validate one config, fast | 
|  | # ./bench.sh report | 
|  | # | 
|  | # REQUIRES, on PATH | 
|  | # nova with `bf` and `sweep` subcommands | 
|  | # nova-load built with --features elastic,milvus for the non-Qdrant paths | 
|  | # docker with a compose file in the working directory (see SERVICES) | 
|  | # hf HuggingFace CLI, for the dataset | 
|  | # python with pyarrow and pandas | 
|  | # | 
|  | # CONFIGURE entirely through the environment — nothing here is tied to a | 
|  | # particular machine, and ROOT is the only directory the script writes to: | 
|  | # | 
|  | # ROOT where everything lands (default $HOME/qfw-bench) | 
|  | # ENGINES which to run (default "qdrant") | 
|  | # SHARDS dataset shards to download (default 1) | 
|  | # QUERY_LIMIT / TOP_K / GT_K / DURATION_S / CONCURRENCY | 
|  | # QDRANT_URL / MILVUS_URL / ELASTIC_URL point at your own endpoints | 
|  | # NO_DOCKER=1 use already-running services and skip compose entirely | 
|  | # FORCE=1 redo a step that would otherwise be skipped as cached | 
|  | # | 
|  | # Verified end to end on the Qdrant path. The Milvus and Elasticsearch sweep | 
|  | # schemas are less travelled — run `./bench.sh probe <engine>` first; it fails | 
|  | # fast and names the offending field. | 
|  | # ============================================================================= | 
|  | set -euo pipefail | 
|  | ROOT="${ROOT:-$HOME/qfw-bench}" | 
|  | ENGINES="${ENGINES:-qdrant}" | 
|  | SHARDS="${SHARDS:-1}" | 
|  | QUERY_LIMIT="${QUERY_LIMIT:-1000}" | 
|  | TOP_K="${TOP_K:-10}" | 
|  | GT_K="${GT_K:-100}" | 
|  | DURATION_S="${DURATION_S:-60}" | 
|  | CONCURRENCY="${CONCURRENCY:-8}" | 
|  | SLICE="$ROOT/slice" | 
|  | QUERIES="$ROOT/qfw-queries" | 
|  | REGEN="$ROOT/regenerated/dense_regenerated.parquet" | 
|  | BFOUT="$ROOT/bf-out" | 
|  | BFFILE="$BFOUT/bf_dense_regenerated_dense_k${GT_K}.parquet" | 
|  | SWEEPOUT="$ROOT/sweep-out" | 
|  | export QDRANT_URL="${QDRANT_URL:-http://localhost:6334}" | 
|  | export MILVUS_URL="${MILVUS_URL:-http://localhost:19530}" | 
|  | export ELASTIC_URL="${ELASTIC_URL:-http://localhost:9200}" | 
|  | mkdir -p "$ROOT" "$BFOUT" "$SWEEPOUT" "$(dirname "$REGEN")" | 
|  | # cargo-installed binaries are not always on PATH in a non-login shell | 
|  | export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:$PATH" | 
|  | # ============================================================================= | 
|  | # PER-BACKEND SCHEMA | 
|  | # | 
|  | # Every engine spells its search-effort knob differently, and the sweep config | 
|  | # rejects unknown keys outright — so a wrong guess fails at config load with the | 
|  | # offending field named rather than silently producing a wrong number. If your | 
|  | # toolchain disagrees with the spellings below, this is the block to fix. | 
|  | # ============================================================================= | 
|  | # Search-effort knob per engine: Qdrant HNSW takes hnsw_ef, Milvus HNSW takes | 
|  | # ef (IVF takes nprobe instead), Elasticsearch kNN takes num_candidates. | 
|  | effort_key() { | 
|  | case "$1" in | 
|  | qdrant) echo "hnsw_ef" ;; | 
|  | milvus) echo "ef" ;; | 
|  | elastic) echo "num_candidates" ;; | 
|  | esac | 
|  | } | 
|  | effort_values() { | 
|  | case "$1" in | 
|  | elastic) echo "[16, 32, 64, 128, 256]" ;; # must be >= top_k | 
|  | *) echo "[16, 32, 64, 128, 256]" ;; | 
|  | esac | 
|  | } | 
|  | # Index-build knobs, from nova_sweep/backends/*.py. Qdrant nests them under a | 
|  | # `params:` sub-block, so it takes dotted paths. Milvus and Elastic vectorstore | 
|  | # blocks are FLAT — variant keys land directly on the block, no dots. Elastic | 
|  | # holds HNSW mapping params in `index_options`; Milvus in `index_type` + | 
|  | # `index_params`. | 
|  | index_block() { | 
|  | case "$1" in | 
|  | qdrant) printf ' hnsw.m: [16]\n hnsw.ef_construct: [100]\n' ;; | 
|  | elastic) printf ' index_options: [{type: hnsw, m: 16, ef_construction: 100}]\n' ;; | 
|  | milvus) printf ' index_type: [HNSW]\n index_params: [{M: 16, efConstruction: 100}]\n' ;; | 
|  | esac | 
|  | } | 
|  | target_block() { | 
|  | case "$1" in | 
|  | qdrant) printf ' type: qdrant\n url: ${QDRANT_URL}\n recreate: always\n' ;; | 
|  | milvus) printf ' type: milvus\n url: ${MILVUS_URL}\n recreate: always\n' ;; | 
|  | elastic) printf ' type: elastic\n url: ${ELASTIC_URL}\n tls_insecure: true\n recreate: always\n' ;; | 
|  | esac | 
|  | } | 
|  | # ============================================================================= | 
|  | # ============================================================================= | 
|  | # Fail before spending five minutes getting to a failure. | 
|  | preflight() { | 
|  | local missing=0 | 
|  | for c in hf docker curl python nova; do | 
|  | command -v "$c" >/dev/null \|\| { echo "MISSING: $c"; missing=1; } | 
|  | done | 
|  | nova --help 2>/dev/null \| grep -q '^ bf ' \ | 
|  | \|\| { echo "MISSING: the 'bf' subcommand of nova"; missing=1; } | 
|  | for e in $ENGINES; do | 
|  | case "$e" in qdrant) continue ;; esac | 
|  | nova-load --help 2>&1 \| grep -qi "$e" \ | 
|  | \|\| echo "WARNING: nova-load may lack $e support (rebuild with --features elastic,milvus)" | 
|  | done | 
|  | python -c "import pyarrow, pandas" 2>/dev/null \ | 
|  | \|\| { echo "MISSING: pyarrow/pandas (pip install pyarrow pandas)"; missing=1; } | 
|  | # SERVICES: compose must define qdrant / etcd minio milvus / elastic, matching | 
|  | # engine_services() below. NO_DOCKER=1 if you run them some other way. | 
|  | if [ -z "${NO_DOCKER:-}" ] && ! docker compose config --services >/dev/null 2>&1; then | 
|  | echo "MISSING: a docker compose file here defining the engine services" | 
|  | echo " (or set NO_DOCKER=1 to use services you start yourself)" | 
|  | missing=1 | 
|  | fi | 
|  | [ "$missing" -eq 0 ] \|\| { echo "preflight failed"; exit 1; } | 
|  | echo "preflight OK — engines: $ENGINES" | 
|  | } | 
|  | # Only the Python side. The nova toolchain is a prerequisite — install it | 
|  | # however you normally do and make sure it is on PATH before running this. | 
|  | setup() { | 
|  | pip install "transformers<5" sentence-transformers requests huggingface_hub \ | 
|  | pyarrow pandas | 
|  | nova --help | 
|  | } | 
|  | fetch() { | 
|  | # FORCE=1 to re-download regardless. | 
|  | if [ -z "${FORCE:-}" ] && [ -d "$SLICE/data" ] && \ | 
|  | [ "$(find "$SLICE/data" -name '*.parquet' \| wc -l)" -ge "$SHARDS" ]; then | 
|  | echo "skip fetch: $SHARDS shard(s) already present (FORCE=1 to redo)" | 
|  | return 0 | 
|  | fi | 
|  | hf download Qdrant/FineWeb-10B --repo-type dataset \ | 
|  | --include "queries/gt_dense_k1000.parquet" "queries/scripts/*" "queries/_checksums.json" \ | 
|  | --local-dir "$QUERIES" | 
|  | python - "$SHARDS" "$SLICE" <<'PY' | 
|  | import subprocess, sys | 
|  | from huggingface_hub import HfApi | 
|  | n, dest = int(sys.argv[1]), sys.argv[2] | 
|  | files = [f for f in HfApi().list_repo_files('Qdrant/FineWeb-10B', repo_type='dataset') | 
|  | if f.startswith('data/')][:n] | 
|  | print(f"fetching {len(files)} shard(s):", *files, sep="\n ") | 
|  | subprocess.run(["hf","download","Qdrant/FineWeb-10B","--repo-type","dataset", | 
|  | "--local-dir", dest, "--include", *files], check=True) | 
|  | PY | 
|  | } | 
|  | queries() { | 
|  | if [ -z "${FORCE:-}" ] && [ -f "$REGEN" ]; then | 
|  | echo "skip queries: $REGEN exists (FORCE=1 to redo)" | 
|  | return 0 | 
|  | fi | 
|  | cd "$QUERIES/queries" | 
|  | python scripts/regenerate_queries.py \ | 
|  | --in gt_dense_k1000.parquet --out "$REGEN" \ | 
|  | --vectors dense --limit "$QUERY_LIMIT" --device cpu | 
|  | # MS MARCO queries are licensed for non-commercial research use only. | 
|  | } | 
|  | groundtruth() { | 
|  | if [ -z "${FORCE:-}" ] && [ -f "$BFFILE" ]; then | 
|  | echo "skip groundtruth: $BFFILE exists (FORCE=1 to redo)" | 
|  | return 0 | 
|  | fi | 
|  | cat > "$ROOT/bf.yaml" <<YAML | 
|  | corpus: | 
|  | path: $SLICE/data | 
|  | dense_column: dense_embedding | 
|  | queries: | 
|  | path: $REGEN | 
|  | dense_column: dense_embedding | 
|  | id_column: msmarco_query_id | 
|  | payload_fields: | 
|  | - query | 
|  | - dense_embedding | 
|  | output: | 
|  | path: $BFOUT | 
|  | params: | 
|  | io_workers: 4 | 
|  | dense_batch_size: 4096 | 
|  | searches: | 
|  | - name: dense | 
|  | vector_type: dense | 
|  | metric: cosine | 
|  | k: $GT_K | 
|  | YAML | 
|  | nova bf compute "$ROOT/bf.yaml" | 
|  | python - "$BFFILE" <<'PY' | 
|  | import sys, pyarrow.parquet as pq | 
|  | t = pq.read_table(sys.argv[1]); s = t.column("hit_scores")[0].as_py() | 
|  | assert "dense_embedding" in t.schema.names, "query vectors missing from bf output" | 
|  | print(f"ground truth OK: {t.num_rows} queries, top {s[0]:.4f}, k-th {s[-1]:.4f}") | 
|  | PY | 
|  | } | 
|  | write_config() { | 
|  | local e="$1" | 
|  | cat > "$ROOT/sweep-$e.yaml" <<YAML | 
|  | collection_name: qfineweb_$e | 
|  | corpus: | 
|  | path: $SLICE/data | 
|  | dense_column: dense_embedding | 
|  | queries: | 
|  | uri: $BFFILE | 
|  | column: dense_embedding | 
|  | ground_truth_column: hit_ids | 
|  | limit: $QUERY_LIMIT | 
|  | target: | 
|  | $(target_block "$e") | 
|  | # The ground truth was computed with metric: cosine. If an engine's collection | 
|  | # is built with a different distance, recall drops for a reason that has | 
|  | # nothing to do with the engine — and it fails as a plausible number rather | 
|  | # than an error, so pin it explicitly rather than trusting per-backend defaults. | 
|  | data_layouts: | 
|  | vectors.dense.distance: [cosine] | 
|  | index_variants: | 
|  | $(index_block "$e") | 
|  | searches: | 
|  | top_k: [$TOP_K] | 
|  | $(effort_key "$e"): $(effort_values "$e") | 
|  | batch_size: [1] | 
|  | duration_s: [$DURATION_S] | 
|  | concurrency: [$CONCURRENCY] | 
|  | output: | 
|  | path: $SWEEPOUT/$e | 
|  | YAML | 
|  | echo "wrote $ROOT/sweep-$e.yaml" | 
|  | } | 
|  | # Validate a config in seconds instead of paying for a multi-minute ingest to | 
|  | # find out it was wrong. There is no --dry-run, so call the config loader | 
|  | # directly and let it name the offending field. | 
|  | # NOTE: this validates the SWEEP config only. The per-backend configs it | 
|  | # generates are checked further downstream, so a clean probe does not guarantee | 
|  | # a clean run. | 
|  | probe() { | 
|  | local e="${1:?usage: probe <engine>}" | 
|  | write_config "$e" | 
|  | python - "$ROOT/sweep-$e.yaml" <<'PY' | 
|  | import sys | 
|  | from nova_sweep.config import load_config | 
|  | try: | 
|  | load_config(sys.argv[1]) | 
|  | except Exception as exc: | 
|  | print(f"INVALID: {type(exc).__name__}") | 
|  | print(exc) | 
|  | sys.exit(1) | 
|  | print(f"config OK: {sys.argv[1]}") | 
|  | PY | 
|  | } | 
|  | # Block until the engine answers, instead of a fixed sleep. Milvus in | 
|  | # particular takes well over a minute to become ready, and a broken pipe | 
|  | # mid-ingest is what an unready (or OOM-killed) server looks like. | 
|  | wait_ready() { | 
|  | local e="$1" url deadline=$((SECONDS + 180)) | 
|  | case "$e" in | 
|  | qdrant) url="http://localhost:6333/readyz" ;; | 
|  | milvus) url="http://localhost:9091/healthz" ;; | 
|  | elastic) url="$ELASTIC_URL/_cluster/health" ;; | 
|  | esac | 
|  | echo "waiting for $e ..." | 
|  | until curl -sf "$url" >/dev/null 2>&1; do | 
|  | [ $SECONDS -lt $deadline ] \|\| { echo "ERROR: $e not ready after 180s"; return 1; } | 
|  | sleep 3 | 
|  | done | 
|  | echo "$e ready" | 
|  | } | 
|  | run_engine() { | 
|  | local e="$1" | 
|  | echo "=== $e ===" | 
|  | wait_ready "$e" \|\| { echo "SKIPPING $e"; return 0; } | 
|  | write_config "$e" | 
|  | # A sweep that fails still writes rows with ok=false; don't abort the others. | 
|  | nova sweep "$ROOT/sweep-$e.yaml" \|\| echo "WARNING: $e sweep reported errors" | 
|  | } | 
|  | compose_up() { docker compose up -d "$@" >/dev/null 2>&1 \|\| true; } | 
|  | compose_stop() { docker compose stop "$@" >/dev/null 2>&1 \|\| true; } | 
|  | engine_services() { | 
|  | case "$1" in | 
|  | qdrant) echo "qdrant" ;; | 
|  | milvus) echo "etcd minio milvus" ;; | 
|  | elastic) echo "elastic" ;; | 
|  | esac | 
|  | } | 
|  | engines() { | 
|  | for e in $ENGINES; do | 
|  | # One engine up at a time: concurrent engines contend for the same cores | 
|  | # and page cache, which makes every latency number incomparable. | 
|  | local svc; svc=$(engine_services "$e") | 
|  | [ -n "${NO_DOCKER:-}" ] \|\| compose_up $svc | 
|  | run_engine "$e" | 
|  | [ -n "${NO_DOCKER:-}" ] \|\| compose_stop $svc | 
|  | sleep 5 | 
|  | done | 
|  | } | 
|  | # Results without provenance can't be defended later. nova bf writes its own | 
|  | # manifest; the sweep output has none, so record the rest here. | 
|  | provenance() { | 
|  | { | 
|  | echo "date: $(date -u +%FT%TZ)" | 
|  | echo "host: $(uname -mrs)" | 
|  | echo "engines: $ENGINES" | 
|  | echo "shards: $SHARDS query_limit: $QUERY_LIMIT top_k: $TOP_K gt_k: $GT_K" | 
|  | echo "duration_s: $DURATION_S concurrency: $CONCURRENCY" | 
|  | echo "nova: $(nova --version 2>/dev/null \|\| echo unknown)" | 
|  | echo "nova-load: $(nova-load --version 2>/dev/null \|\| echo unknown)" | 
|  | for e in $ENGINES; do | 
|  | for s in $(engine_services "$e"); do | 
|  | echo "image[$s]: $(docker compose images -q "$s" 2>/dev/null \| head -1)" | 
|  | done | 
|  | done | 
|  | } \| tee "$ROOT/provenance.txt" | 
|  | } | 
|  | report() { | 
|  | python - "$SWEEPOUT" "$ROOT/results.csv" <<'PY' | 
|  | import glob, os, sys, pandas as pd, pyarrow.parquet as pq | 
|  | sweepout, outcsv = sys.argv[1], sys.argv[2] | 
|  | rows = [] | 
|  | for f in glob.glob(os.path.join(sweepout, "*", "sweep_results.parquet")): | 
|  | df = pq.read_table(f).to_pandas() | 
|  | df["engine"] = os.path.basename(os.path.dirname(f)) | 
|  | rows.append(df) | 
|  | if not rows: | 
|  | raise SystemExit("no sweep results yet") | 
|  | df = pd.concat(rows, ignore_index=True) | 
|  | # each backend names its effort knob differently; coalesce into one column | 
|  | keys = [c for c in ("search.hnsw_ef", "search.ef", "search.num_candidates") | 
|  | if c in df.columns] | 
|  | df["effort"] = df[keys].bfill(axis=1).iloc[:, 0] if keys else None | 
|  | # A failed sweep still writes one row per search point with ok=false and every | 
|  | # metric null. Reporting those as NaN alongside real numbers invites reading a | 
|  | # blank as a result, so split them out and say what actually broke. | 
|  | if "ok" in df.columns: | 
|  | failed, df = df[~df["ok"].fillna(False)], df[df["ok"].fillna(False)] | 
|  | for eng, grp in failed.groupby("engine"): | 
|  | msg = next((m for m in grp.get("error", []) if isinstance(m, str)), "unknown error") | 
|  | print(f"FAILED: {eng} — {len(grp)} point(s) did not run") | 
|  | print(f" {msg.strip().splitlines()[0][:200]}\n") | 
|  | if df.empty: | 
|  | raise SystemExit("every engine failed — nothing to report") | 
|  | cols = ["engine", "effort", "full_recall.mean", "p50_ms", "p95_ms", "p99_ms", | 
|  | "qps", "missing_from_gt", "reindex_seconds"] | 
|  | out = df[[c for c in cols if c in df.columns]].sort_values(["engine", "effort"]) | 
|  | print(out.to_string(index=False)) | 
|  | out.to_csv(outcsv, index=False) | 
|  | print(f"\nwrote {outcsv}") | 
|  | if "missing_from_gt" in df and (df["missing_from_gt"] > 0).any(): | 
|  | print("\nWARNING: missing_from_gt > 0 — ground truth and collection disagree.") | 
|  | print("Recall numbers from those rows are not trustworthy.") | 
|  | if df["engine"].nunique() > 1: | 
|  | print("\nCompare engines at EQUAL RECALL, not at equal effort — the knobs") | 
|  | print("are not on the same scale. Milvus cosine goes over REST rather than") | 
|  | print("its native SDK path, so its latency is not comparable to Qdrant's gRPC.") | 
|  | PY | 
|  | } | 
|  | all() { preflight; fetch; queries; groundtruth; engines; provenance; report; } | 
|  | "${@:-all}" |