| | #!/usr/bin/env bash |
| | # ============================================================================= |
| | # FineWeb-10B slice benchmark — Qdrant vs Milvus vs Elasticsearch |
| | # |
| | # One exact ground truth (brute force over YOUR slice), reused by every engine, |
| | # so all three are scored against identical correct answers. Each engine is then |
| | # swept across its search-effort knob to produce a recall/latency CURVE rather |
| | # than a single point. |
| | # |
| | # ./bench.sh all # everything, Qdrant only |
| | # ENGINES="qdrant milvus elastic" ./bench.sh all |
| | # ./bench.sh probe milvus # validate one config, fast |
| | # ./bench.sh report |
| | # |
| | # REQUIRES, on PATH |
| | # nova with bf and sweep subcommands |
| | # nova-load built with --features elastic,milvus for the non-Qdrant paths |
| | # docker with a compose file in the working directory (see SERVICES) |
| | # hf HuggingFace CLI, for the dataset |
| | # python with pyarrow and pandas |
| | # |
| | # CONFIGURE entirely through the environment — nothing here is tied to a |
| | # particular machine, and ROOT is the only directory the script writes to: |
| | # |
| | # ROOT where everything lands (default $HOME/qfw-bench) |
| | # ENGINES which to run (default "qdrant") |
| | # SHARDS dataset shards to download (default 1) |
| | # QUERY_LIMIT / TOP_K / GT_K / DURATION_S / CONCURRENCY |
| | # QDRANT_URL / MILVUS_URL / ELASTIC_URL point at your own endpoints |
| | # NO_DOCKER=1 use already-running services and skip compose entirely |
| | # FORCE=1 redo a step that would otherwise be skipped as cached |
| | # |
| | # Verified end to end on the Qdrant path. The Milvus and Elasticsearch sweep |
| | # schemas are less travelled — run ./bench.sh probe <engine> first; it fails |
| | # fast and names the offending field. |
| | # ============================================================================= |
| | set -euo pipefail |
| | ROOT="${ROOT:-$HOME/qfw-bench}" |
| | ENGINES="${ENGINES:-qdrant}" |
| | SHARDS="${SHARDS:-1}" |
| | QUERY_LIMIT="${QUERY_LIMIT:-1000}" |
| | TOP_K="${TOP_K:-10}" |
| | GT_K="${GT_K:-100}" |
| | DURATION_S="${DURATION_S:-60}" |
| | CONCURRENCY="${CONCURRENCY:-8}" |
| | SLICE="$ROOT/slice" | | | QUERIES="$ROOT/qfw-queries" | | | REGEN="$ROOT/regenerated/dense_regenerated.parquet" | | | BFOUT="$ROOT/bf-out" | | | BFFILE="$BFOUT/bf_dense_regenerated_dense_k${GT_K}.parquet" |
| | SWEEPOUT="$ROOT/sweep-out" |
| | export QDRANT_URL="${QDRANT_URL:-http://localhost:6334}" |
| | export MILVUS_URL="${MILVUS_URL:-http://localhost:19530}" |
| | export ELASTIC_URL="${ELASTIC_URL:-http://localhost:9200}" |
| | mkdir -p "$ROOT" "$BFOUT" "$SWEEPOUT" "$(dirname "$REGEN")" | | | # cargo-installed binaries are not always on PATH in a non-login shell |
| | export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:$PATH" |
| | # ============================================================================= |
| | # PER-BACKEND SCHEMA | | | # | | | # Every engine spells its search-effort knob differently, and the sweep config | | | # rejects unknown keys outright — so a wrong guess fails at config load with the | | | # offending field named rather than silently producing a wrong number. If your | | | # toolchain disagrees with the spellings below, this is the block to fix. | | | # ============================================================================= | | | # Search-effort knob per engine: Qdrant HNSW takes hnsw_ef, Milvus HNSW takes | | | # ef (IVF takes nprobe instead), Elasticsearch kNN takes num_candidates. | | | effort_key() { | | | case "$1" in |
| | qdrant) echo "hnsw_ef" ;; |
| | milvus) echo "ef" ;; |
| | elastic) echo "num_candidates" ;; |
| | esac | | | } | | | effort_values() { | | | case "$1" in |
| | elastic) echo "[16, 32, 64, 128, 256]" ;; # must be >= top_k |
| | *) echo "[16, 32, 64, 128, 256]" ;; |
| | esac |
| | } |
| | # Index-build knobs, from nova_sweep/backends/*.py. Qdrant nests them under a |
| | # params: sub-block, so it takes dotted paths. Milvus and Elastic vectorstore |
| | # blocks are FLAT — variant keys land directly on the block, no dots. Elastic |
| | # holds HNSW mapping params in index_options; Milvus in index_type + |
| | # index_params. |
| | index_block() { |
| | case "$1" in |
| | qdrant) printf ' hnsw.m: [16]\n hnsw.ef_construct: [100]\n' ;; |
| | elastic) printf ' index_options: [{type: hnsw, m: 16, ef_construction: 100}]\n' ;; |
| | milvus) printf ' index_type: [HNSW]\n index_params: [{M: 16, efConstruction: 100}]\n' ;; |
| | esac | | | } | | | target_block() { | | | case "$1" in |
| | qdrant) printf ' type: qdrant\n url: ${QDRANT_URL}\n recreate: always\n' ;; |
| | milvus) printf ' type: milvus\n url: ${MILVUS_URL}\n recreate: always\n' ;; |
| | elastic) printf ' type: elastic\n url: ${ELASTIC_URL}\n tls_insecure: true\n recreate: always\n' ;; |
| | esac | | | } |
| | # ============================================================================= |
| | # ============================================================================= |
| | # Fail before spending five minutes getting to a failure. | | | preflight() { | | | local missing=0 | | | for c in hf docker curl python nova; do | | | command -v "$c" >/dev/null || { echo "MISSING: $c"; missing=1; } | | | done |
| | nova --help 2>/dev/null \| grep -q '^ bf ' \ |
| | \|\| { echo "MISSING: the 'bf' subcommand of nova"; missing=1; } |
| | for e in $ENGINES; do |
| | case "$e" in qdrant) continue ;; esac |
| | nova-load --help 2>&1 \| grep -qi "$e" \ |
| | \|\| echo "WARNING: nova-load may lack $e support (rebuild with --features elastic,milvus)" |
| | done | | | python -c "import pyarrow, pandas" 2>/dev/null \ | | | || { echo "MISSING: pyarrow/pandas (pip install pyarrow pandas)"; missing=1; } | | | # SERVICES: compose must define qdrant / etcd minio milvus / elastic, matching | | | # engine_services() below. NO_DOCKER=1 if you run them some other way. | | | if [ -z "${NO_DOCKER:-}" ] && ! docker compose config --services >/dev/null 2>&1; then | | | echo "MISSING: a docker compose file here defining the engine services" | | | echo " (or set NO_DOCKER=1 to use services you start yourself)" | | | missing=1 | | | fi | | | [ "$missing" -eq 0 ] || { echo "preflight failed"; exit 1; } | | | echo "preflight OK — engines: $ENGINES" | | | } | | | # Only the Python side. The nova toolchain is a prerequisite — install it | | | # however you normally do and make sure it is on PATH before running this. | | | setup() { | | | pip install "transformers<5" sentence-transformers requests huggingface_hub \ | | | pyarrow pandas | | | nova --help | | | } | | | fetch() { | | | # FORCE=1 to re-download regardless. |
| | if [ -z "${FORCE:-}" ] && [ -d "$SLICE/data" ] && \ |
| | [ "$(find "$SLICE/data" -name '*.parquet' \| wc -l)" -ge "$SHARDS" ]; then |
| | echo "skip fetch: $SHARDS shard(s) already present (FORCE=1 to redo)" |
| | return 0 | | | fi | | | hf download Qdrant/FineWeb-10B --repo-type dataset \ | | | --include "queries/gt_dense_k1000.parquet" "queries/scripts/*" "queries/_checksums.json" \ | | | --local-dir "$QUERIES" | | | python - "$SHARDS" "$SLICE" <<'PY' | | | import subprocess, sys | | | from huggingface_hub import HfApi |
| | n, dest = int(sys.argv[1]), sys.argv[2] |
| | files = [f for f in HfApi().list_repo_files('Qdrant/FineWeb-10B', repo_type='dataset') |
| | if f.startswith('data/')][:n] |
| | print(f"fetching {len(files)} shard(s):", *files, sep="\n ") |
| | subprocess.run(["hf","download","Qdrant/FineWeb-10B","--repo-type","dataset", |
| | "--local-dir", dest, "--include", *files], check=True) |
| | PY | | | } |
| | queries() { |
| | if [ -z "${FORCE:-}" ] && [ -f "$REGEN" ]; then |
| | echo "skip queries: $REGEN exists (FORCE=1 to redo)" |
| | return 0 | | | fi | | | cd "$QUERIES/queries" | | | python scripts/regenerate_queries.py \ |
| | --in gt_dense_k1000.parquet --out "$REGEN" \ |
| | --vectors dense --limit "$QUERY_LIMIT" --device cpu |
| | # MS MARCO queries are licensed for non-commercial research use only. | | | } |
| | groundtruth() { |
| | if [ -z "${FORCE:-}" ] && [ -f "$BFFILE" ]; then |
| | echo "skip groundtruth: $BFFILE exists (FORCE=1 to redo)" |
| | return 0 | | | fi | | | cat > "$ROOT/bf.yaml" <<YAML | | | corpus: | | | path: $SLICE/data | | | dense_column: dense_embedding | | | queries: | | | path: $REGEN | | | dense_column: dense_embedding | | | id_column: msmarco_query_id | | | payload_fields: | | | - query | | | - dense_embedding | | | output: | | | path: $BFOUT | | | params: | | | io_workers: 4 | | | dense_batch_size: 4096 | | | searches: | | | - name: dense | | | vector_type: dense | | | metric: cosine | | | k: $GT_K | | | YAML | | | nova bf compute "$ROOT/bf.yaml" | | | python - "$BFFILE" <<'PY' | | | import sys, pyarrow.parquet as pq | | | t = pq.read_table(sys.argv[1]); s = t.column("hit_scores")[0].as_py() | | | assert "dense_embedding" in t.schema.names, "query vectors missing from bf output" | | | print(f"ground truth OK: {t.num_rows} queries, top {s[0]:.4f}, k-th {s[-1]:.4f}") | | | PY | | | } | | | write_config() { | | | local e="$1" | | | cat > "$ROOT/sweep-$e.yaml" <<YAML | | | collection_name: qfineweb_$e | | | corpus: | | | path: $SLICE/data | | | dense_column: dense_embedding | | | queries: | | | uri: $BFFILE | | | column: dense_embedding | | | ground_truth_column: hit_ids | | | limit: $QUERY_LIMIT | | | target: | | | $(target_block "$e") | | | # The ground truth was computed with metric: cosine. If an engine's collection | | | # is built with a different distance, recall drops for a reason that has | | | # nothing to do with the engine — and it fails as a plausible number rather | | | # than an error, so pin it explicitly rather than trusting per-backend defaults. | | | data_layouts: | | | vectors.dense.distance: [cosine] | | | index_variants: | | | $(index_block "$e") | | | searches: |
| | top_k: [$TOP_K] |
| | $(effort_key "$e"): $(effort_values "$e") |
| | batch_size: [1] |
| | duration_s: [$DURATION_S] |
| | concurrency: [$CONCURRENCY] |
| | output: | | | path: $SWEEPOUT/$e | | | YAML | | | echo "wrote $ROOT/sweep-$e.yaml" | | | } | | | # Validate a config in seconds instead of paying for a multi-minute ingest to | | | # find out it was wrong. There is no --dry-run, so call the config | | | # directly and let it name the offending field. | | | # NOTE: this validates the SWEEP config only. The per-backend configs it | | | # generates are checked further downstream, so a clean probe does not guarantee | | | # a clean run. |
| | probe() { |
| | local e="${1:?usage: probe <engine>}" |
| | write_config "$e" | | | python - "$ROOT/sweep-$e.yaml" <<'PY' | | | import sys | | | from nova_sweep.config import load_config | | | try: | | | load_config(sys.argv[1]) | | | except Exception as exc: |
| | print(f"INVALID: {type(exc).__name__}") |
| | print(exc) |
| | sys.exit(1) |
| | print(f"config OK: {sys.argv[1]}") |
| | PY | | | } | | | # Block until the engine answers, instead of a fixed sleep. Milvus in | | | # particular takes well over a minute to become ready, and a broken pipe |
| | # mid-ingest is what an unready (or OOM-killed) server looks like. |
| | wait_ready() { |
| | local e="$1" url deadline=$((SECONDS + 180)) |
| | case "$e" in |
| | qdrant) url="http://localhost:6333/readyz" ;; |
| | milvus) url="http://localhost:9091/healthz" ;; |
| | elastic) url="$ELASTIC_URL/_cluster/health" ;; |
| | esac | | | echo "waiting for $e ..." |
| | until curl -sf "$url" >/dev/null 2>&1; do |
| | [ $SECONDS -lt $deadline ] \|\| { echo "ERROR: $e not ready after 180s"; return 1; } |
| | sleep 3 | | | done | | | echo "$e ready" | | | } | | | run_engine() { | | | local e="$1" |
| | echo "=== $e ===" |
| | wait_ready "$e" \|\| { echo "SKIPPING $e"; return 0; } |
| | write_config "$e" | | | # A sweep that fails still writes rows with ok=false; don't abort the others. | | | nova sweep "$ROOT/sweep-$e.yaml" || echo "WARNING: $e sweep reported errors" | | | } |
| | compose_up() { docker compose up -d "$@" >/dev/null 2>&1 \|\| true; } |
| | compose_stop() { docker compose stop "$@" >/dev/null 2>&1 \|\| true; } |
| | engine_services() { |
| | case "$1" in |
| | qdrant) echo "qdrant" ;; |
| | milvus) echo "etcd minio milvus" ;; |
| | elastic) echo "elastic" ;; |
| | esac | | | } | | | engines() { | | | for e in $ENGINES; do | | | # One engine up at a time: concurrent engines contend for the same cores | | | # and page cache, which makes every latency number incomparable. |
| | local svc; svc=$(engine_services "$e") |
| | [ -n "${NO_DOCKER:-}" ] \|\| compose_up $svc |
| | run_engine "$e" | | | [ -n "${NO_DOCKER:-}" ] || compose_stop $svc | | | sleep 5 | | | done | | | } | | | # Results without provenance can't be defended later. nova bf writes its own | | | # manifest; the sweep output has none, so record the rest here. | | | provenance() { | | | { |
| | echo "date: $(date -u +%FT%TZ)" |
| | echo "host: $(uname -mrs)" |
| | echo "engines: $ENGINES" | | | echo "shards: $SHARDS query_limit: $QUERY_LIMIT top_k: $TOP_K gt_k: $GT_K" | | | echo "duration_s: $DURATION_S concurrency: $CONCURRENCY" |
| | echo "nova: $(nova --version 2>/dev/null \|\| echo unknown)" |
| | echo "nova-load: $(nova-load --version 2>/dev/null \|\| echo unknown)" |
| | for e in $ENGINES; do |
| | for s in $(engine_services "$e"); do |
| | echo "image[$s]: $(docker compose images -q "$s" 2>/dev/null \| head -1)" |
| | done | | | done | | | } | tee "$ROOT/provenance.txt" | | | } | | | report() { | | | python - "$SWEEPOUT" "$ROOT/results.csv" <<'PY' | | | import glob, os, sys, pandas as pd, pyarrow.parquet as pq |
| | sweepout, outcsv = sys.argv[1], sys.argv[2] |
| | rows = [] |
| | for f in glob.glob(os.path.join(sweepout, "*", "sweep_results.parquet")): |
| | df = pq.read_table(f).to_pandas() |
| | df["engine"] = os.path.basename(os.path.dirname(f)) |
| | rows.append(df) |
| | if not rows: | | | raise SystemExit("no sweep results yet") | | | df = pd.concat(rows, ignore_index=True) | | | # each backend names its effort knob differently; coalesce into one column | | | keys = [c for c in ("search.hnsw_ef", "search.ef", "search.num_candidates") | | | if c in df.columns] | | | df["effort"] = df[keys].bfill(axis=1).iloc[:, 0] if keys else None | | | # A failed sweep still writes one row per search point with ok=false and every | | | # metric null. Reporting those as NaN alongside real numbers invites reading a | | | # blank as a result, so split them out and say what actually broke. | | | if "ok" in df.columns: |
| | failed, df = df[~df["ok"].fillna(False)], df[df["ok"].fillna(False)] |
| | for eng, grp in failed.groupby("engine"): |
| | msg = next((m for m in grp.get("error", []) if isinstance(m, str)), "unknown error") |
| | print(f"FAILED: {eng} — {len(grp)} point(s) did not run") |
| | print(f" {msg.strip().splitlines()[0][:200]}\n") |
| | if df.empty: | | | raise SystemExit("every engine failed — nothing to report") | | | cols = ["engine", "effort", "full_recall.mean", "p50_ms", "p95_ms", "p99_ms", | | | "qps", "missing_from_gt", "reindex_seconds"] |
| | out = df[[c for c in cols if c in df.columns]].sort_values(["engine", "effort"]) |
| | print(out.to_string(index=False)) |
| | out.to_csv(outcsv, index=False) |
| | print(f"\nwrote {outcsv}") |
| | if "missing_from_gt" in df and (df["missing_from_gt"] > 0).any(): |
| | print("\nWARNING: missing_from_gt > 0 — ground truth and collection disagree.") | | | print("Recall numbers from those rows are not trustworthy.") | | | if df["engine"].nunique() > 1: | | | print("\nCompare engines at EQUAL RECALL, not at equal effort — the knobs") | | | print("are not on the same scale. Milvus cosine goes over REST rather than") | | | print("its native SDK path, so its latency is not comparable to Qdrant's gRPC.") | | | PY | | | } |
| | all() { preflight; fetch; queries; groundtruth; engines; provenance; report; } |
| | "${@:-all}" |