d2028a642b
Pipeline engine rewritten with combo-first loop: each combination is processed through all requested passes before moving to the next, with incremental DB saves after every step (crash-safe). Blocked combos now get result rows so they appear in the results page with constraint violation reasons. New pipeline_runs table tracks run lifecycle (pending/running/completed/failed/ cancelled). Web route launches pipeline in a background thread with its own DB connection. HTMX polling partial shows live progress with per-pass breakdown. Also: status guard prevents reviewed->scored downgrade, save_combination loads existing status on dedup for correct resume, per-metric scores show domain bounds + units + position bars, ensure_metric backfills units on existing rows. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
130 lines
4.3 KiB
Python
130 lines
4.3 KiB
Python
"""DDL, table creation, and schema initialization."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
DDL = """
|
|
CREATE TABLE IF NOT EXISTS dimensions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT UNIQUE NOT NULL,
|
|
description TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS entities (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
dimension_id INTEGER NOT NULL REFERENCES dimensions(id),
|
|
name TEXT NOT NULL,
|
|
description TEXT,
|
|
UNIQUE(dimension_id, name)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS dependencies (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
entity_id INTEGER NOT NULL REFERENCES entities(id),
|
|
category TEXT NOT NULL,
|
|
key TEXT NOT NULL,
|
|
value TEXT NOT NULL,
|
|
unit TEXT,
|
|
constraint_type TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS domains (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT UNIQUE NOT NULL,
|
|
description TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS metrics (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT UNIQUE NOT NULL,
|
|
unit TEXT,
|
|
description TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS domain_metric_weights (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
domain_id INTEGER NOT NULL REFERENCES domains(id),
|
|
metric_id INTEGER NOT NULL REFERENCES metrics(id),
|
|
weight REAL NOT NULL,
|
|
norm_min REAL,
|
|
norm_max REAL,
|
|
UNIQUE(domain_id, metric_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS combinations (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
hash TEXT UNIQUE NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
block_reason TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS combination_entities (
|
|
combination_id INTEGER NOT NULL REFERENCES combinations(id),
|
|
entity_id INTEGER NOT NULL REFERENCES entities(id),
|
|
PRIMARY KEY (combination_id, entity_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS combination_scores (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
combination_id INTEGER NOT NULL REFERENCES combinations(id),
|
|
domain_id INTEGER NOT NULL REFERENCES domains(id),
|
|
metric_id INTEGER NOT NULL REFERENCES metrics(id),
|
|
raw_value REAL,
|
|
normalized_score REAL,
|
|
estimation_method TEXT,
|
|
confidence REAL,
|
|
UNIQUE(combination_id, domain_id, metric_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS combination_results (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
combination_id INTEGER NOT NULL REFERENCES combinations(id),
|
|
domain_id INTEGER NOT NULL REFERENCES domains(id),
|
|
composite_score REAL,
|
|
novelty_flag TEXT,
|
|
llm_review TEXT,
|
|
human_notes TEXT,
|
|
pass_reached INTEGER,
|
|
UNIQUE(combination_id, domain_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS pipeline_runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
domain_id INTEGER NOT NULL REFERENCES domains(id),
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
config TEXT,
|
|
total_combos INTEGER DEFAULT 0,
|
|
combos_pass1 INTEGER DEFAULT 0,
|
|
combos_pass2 INTEGER DEFAULT 0,
|
|
combos_pass3 INTEGER DEFAULT 0,
|
|
combos_pass4 INTEGER DEFAULT 0,
|
|
current_pass INTEGER,
|
|
error_message TEXT,
|
|
started_at TIMESTAMP,
|
|
completed_at TIMESTAMP,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_deps_entity ON dependencies(entity_id);
|
|
CREATE INDEX IF NOT EXISTS idx_deps_category_key ON dependencies(category, key);
|
|
CREATE INDEX IF NOT EXISTS idx_combo_status ON combinations(status);
|
|
CREATE INDEX IF NOT EXISTS idx_scores_combo_domain ON combination_scores(combination_id, domain_id);
|
|
CREATE INDEX IF NOT EXISTS idx_results_domain_score ON combination_results(domain_id, composite_score DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_pipeline_runs_domain ON pipeline_runs(domain_id);
|
|
"""
|
|
|
|
|
|
def init_db(db_path: str | Path) -> sqlite3.Connection:
|
|
"""Create/open the database and ensure all tables exist."""
|
|
db_path = Path(db_path)
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
conn = sqlite3.connect(str(db_path))
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
conn.executescript(DDL)
|
|
conn.commit()
|
|
return conn
|