drop safety/availability from scoring, holistic p4 rating, phase-parallel pipeline
safety and availability don't reduce to physics formulas the way power_density/range_fuel/cost_efficiency do -- they're judgment calls (risk assessment, infrastructure prevalence), and running them through the same log-normalize() built for physical quantities produced incoherent results: safety's raw value is already a "0-1" score, and normalizing it again turned 0.6 into an unexplainable 0.678 that even the LLM reviewing it could only cite, never justify (see combo 1540). Removed both from domain_metric_weights (safety from 4 domains, availability from urban_commuting) and renormalized the remaining weights to sum to 1.0. Pass 4 now produces one holistic RATING (LOW/MEDIUM/HIGH) alongside the existing VERDICT, with safety and accessibility folded in as qualitative considerations feeding that single judgment rather than scored separately -- not a checklist of independent numbers. New qualitative_rating column, filterable in the results UI. Also added domain name/description to the review prompt so the LLM judges a metric like range against what the domain actually needs (urban_commuting: 1-50km) instead of generic real-world expectations for the platform category -- confirmed live on a combo where phi4 had called a 396km range "limited" by comparing to typical aircraft rather than a domain that needs 1-50km. Pass 2 is estimator-only now -- self.llm is never consulted there, reserved entirely for pass 4. Restructured Pipeline.run() from combo-first to phase-parallel: each pass now runs to completion across every combo before the next pass starts, rather than walking each combo through all four passes before the next combo. This surfaced a real bug: domain- blocked combos (status stays "valid" by design, not "_fail") were slipping past a naive status-based skip guard and getting silently re-processed by pass 2. Fixed with a shared dead-combo check that catches both generic failures and domain blocks correctly. Also fixes a results-page display bug found while reviewing a live combo: the per-metric "position" bar showed raw distance from norm_min without inverting for lower_is_better metrics, so an excellent cost score (near the good end) rendered as a ~0%, near-empty bar -- looked bad next to its own 0.99 normalized score. Validated live against phi4 (real Ollama calls, not mocked): full-domain phase-parallel run (2,970 combos, estimator-only p2, 1.6s) followed by a real pass-4 run (111 reviewed, 11m, 0 crashes, 0 null ratings). Two tests that relied on the old LLM-driven pass 2 to force deterministic outcomes were updated to test pass 4's verdict-wiring directly instead. All 100 tests pass. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -230,25 +230,38 @@ class Repository:
|
|||||||
self.conn.commit()
|
self.conn.commit()
|
||||||
return row["id"]
|
return row["id"]
|
||||||
|
|
||||||
def backfill_lower_is_better(self, domain_name: str, metric_name: str) -> None:
|
def sync_domain_metric_weights(self, domain: Domain) -> None:
|
||||||
"""Set lower_is_better=1 for an existing domain-metric row that still has the default 0."""
|
"""Make domain_metric_weights exactly match domain.metric_bounds on an
|
||||||
self.conn.execute(
|
already-seeded domain: upserts weight/norm_min/norm_max/unit for every
|
||||||
"""UPDATE domain_metric_weights SET lower_is_better = 1
|
currently-declared metric, and deletes any row for a metric that's been
|
||||||
WHERE lower_is_better = 0
|
removed from the domain (e.g. safety/availability dropped from the
|
||||||
AND domain_id = (SELECT id FROM domains WHERE name = ?)
|
scored set). Safe to call whether the domain was just freshly inserted
|
||||||
AND metric_id = (SELECT id FROM metrics WHERE name = ?)""",
|
or already existed.
|
||||||
(domain_name, metric_name),
|
"""
|
||||||
)
|
row = self.conn.execute(
|
||||||
self.conn.commit()
|
"SELECT id FROM domains WHERE name = ?", (domain.name,)
|
||||||
|
).fetchone()
|
||||||
def backfill_metric_unit(self, domain_name: str, metric_name: str, unit: str) -> None:
|
if not row:
|
||||||
"""Set this domain-metric row's unit — unit is domain-scoped, not global to the metric name."""
|
return
|
||||||
self.conn.execute(
|
domain_id = row["id"]
|
||||||
"""UPDATE domain_metric_weights SET unit = ?
|
keep_ids = []
|
||||||
WHERE domain_id = (SELECT id FROM domains WHERE name = ?)
|
for mb in domain.metric_bounds:
|
||||||
AND metric_id = (SELECT id FROM metrics WHERE name = ?)""",
|
metric_id = self.ensure_metric(mb.metric_name, unit=mb.unit)
|
||||||
(unit, domain_name, metric_name),
|
keep_ids.append(metric_id)
|
||||||
)
|
self.conn.execute(
|
||||||
|
"""INSERT OR REPLACE INTO domain_metric_weights
|
||||||
|
(domain_id, metric_id, weight, norm_min, norm_max, lower_is_better, unit)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||||
|
(domain_id, metric_id, mb.weight, mb.norm_min, mb.norm_max,
|
||||||
|
int(mb.lower_is_better), mb.unit),
|
||||||
|
)
|
||||||
|
if keep_ids:
|
||||||
|
placeholders = ",".join("?" * len(keep_ids))
|
||||||
|
self.conn.execute(
|
||||||
|
f"""DELETE FROM domain_metric_weights
|
||||||
|
WHERE domain_id = ? AND metric_id NOT IN ({placeholders})""",
|
||||||
|
(domain_id, *keep_ids),
|
||||||
|
)
|
||||||
self.conn.commit()
|
self.conn.commit()
|
||||||
|
|
||||||
def add_domain(self, domain: Domain) -> Domain:
|
def add_domain(self, domain: Domain) -> Domain:
|
||||||
@@ -593,15 +606,18 @@ class Repository:
|
|||||||
llm_review: str | None = None,
|
llm_review: str | None = None,
|
||||||
human_notes: str | None = None,
|
human_notes: str | None = None,
|
||||||
domain_block_reason: str | None = None,
|
domain_block_reason: str | None = None,
|
||||||
|
qualitative_rating: str | None = None,
|
||||||
commit: bool = True,
|
commit: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.conn.execute(
|
self.conn.execute(
|
||||||
"""INSERT OR REPLACE INTO combination_results
|
"""INSERT OR REPLACE INTO combination_results
|
||||||
(combination_id, domain_id, composite_score, novelty_flag,
|
(combination_id, domain_id, composite_score, novelty_flag,
|
||||||
llm_review, human_notes, pass_reached, domain_block_reason)
|
llm_review, human_notes, pass_reached, domain_block_reason,
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
qualitative_rating)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||||
(combo_id, domain_id, composite_score, novelty_flag,
|
(combo_id, domain_id, composite_score, novelty_flag,
|
||||||
llm_review, human_notes, pass_reached, domain_block_reason),
|
llm_review, human_notes, pass_reached, domain_block_reason,
|
||||||
|
qualitative_rating),
|
||||||
)
|
)
|
||||||
if commit:
|
if commit:
|
||||||
self.conn.commit()
|
self.conn.commit()
|
||||||
@@ -641,6 +657,20 @@ class Repository:
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
return {r["status"]: r["cnt"] for r in rows}
|
return {r["status"]: r["cnt"] for r in rows}
|
||||||
|
|
||||||
|
def count_results_by_rating(self, domain_name: str) -> dict[str, int]:
|
||||||
|
"""Count results by qualitative_rating (LOW/MEDIUM/HIGH) for a domain.
|
||||||
|
Rows with no rating (not yet pass-4 reviewed, or reviewed before this
|
||||||
|
existed) are excluded, not bucketed as a pseudo-status."""
|
||||||
|
rows = self.conn.execute(
|
||||||
|
"""SELECT cr.qualitative_rating as rating, COUNT(*) as cnt
|
||||||
|
FROM combination_results cr
|
||||||
|
JOIN domains d ON cr.domain_id = d.id
|
||||||
|
WHERE d.name = ? AND cr.qualitative_rating IS NOT NULL
|
||||||
|
GROUP BY cr.qualitative_rating""",
|
||||||
|
(domain_name,),
|
||||||
|
).fetchall()
|
||||||
|
return {r["rating"]: r["cnt"] for r in rows}
|
||||||
|
|
||||||
def get_pipeline_summary(self, domain_name: str) -> dict | None:
|
def get_pipeline_summary(self, domain_name: str) -> dict | None:
|
||||||
"""Return a summary of results for a domain, or None if no results."""
|
"""Return a summary of results for a domain, or None if no results."""
|
||||||
row = self.conn.execute(
|
row = self.conn.execute(
|
||||||
@@ -685,8 +715,12 @@ class Repository:
|
|||||||
).fetchone()
|
).fetchone()
|
||||||
return dict(row) if row else None
|
return dict(row) if row else None
|
||||||
|
|
||||||
def get_all_results(self, domain_name: str, status: str | None = None) -> list[dict]:
|
def get_all_results(
|
||||||
"""Return all results for a domain, optionally filtered by combo status."""
|
self, domain_name: str, status: str | None = None, rating: str | None = None
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Return all results for a domain, optionally filtered by combo
|
||||||
|
status and/or qualitative_rating (LOW/MEDIUM/HIGH, independent filters
|
||||||
|
that combine with AND)."""
|
||||||
query = """SELECT cr.*, c.hash, c.status as combo_status, d.name as domain_name
|
query = """SELECT cr.*, c.hash, c.status as combo_status, d.name as domain_name
|
||||||
FROM combination_results cr
|
FROM combination_results cr
|
||||||
JOIN combinations c ON cr.combination_id = c.id
|
JOIN combinations c ON cr.combination_id = c.id
|
||||||
@@ -698,6 +732,9 @@ class Repository:
|
|||||||
elif status:
|
elif status:
|
||||||
query += " AND c.status = ? AND cr.domain_block_reason IS NULL"
|
query += " AND c.status = ? AND cr.domain_block_reason IS NULL"
|
||||||
params.append(status)
|
params.append(status)
|
||||||
|
if rating:
|
||||||
|
query += " AND cr.qualitative_rating = ?"
|
||||||
|
params.append(rating)
|
||||||
query += " ORDER BY cr.composite_score DESC"
|
query += " ORDER BY cr.composite_score DESC"
|
||||||
rows = self.conn.execute(query, params).fetchall()
|
rows = self.conn.execute(query, params).fetchall()
|
||||||
combo_ids = [r["combination_id"] for r in rows]
|
combo_ids = [r["combination_id"] for r in rows]
|
||||||
@@ -712,6 +749,7 @@ class Repository:
|
|||||||
"pass_reached": r["pass_reached"],
|
"pass_reached": r["pass_reached"],
|
||||||
"domain_id": r["domain_id"],
|
"domain_id": r["domain_id"],
|
||||||
"domain_block_reason": r["domain_block_reason"],
|
"domain_block_reason": r["domain_block_reason"],
|
||||||
|
"qualitative_rating": r["qualitative_rating"],
|
||||||
}
|
}
|
||||||
for r in rows
|
for r in rows
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ CREATE TABLE IF NOT EXISTS combination_results (
|
|||||||
human_notes TEXT,
|
human_notes TEXT,
|
||||||
pass_reached INTEGER,
|
pass_reached INTEGER,
|
||||||
domain_block_reason TEXT,
|
domain_block_reason TEXT,
|
||||||
|
qualitative_rating TEXT,
|
||||||
UNIQUE(combination_id, domain_id)
|
UNIQUE(combination_id, domain_id)
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -165,6 +166,10 @@ def _migrate(conn: sqlite3.Connection) -> None:
|
|||||||
conn.execute(
|
conn.execute(
|
||||||
"ALTER TABLE combination_results ADD COLUMN domain_block_reason TEXT"
|
"ALTER TABLE combination_results ADD COLUMN domain_block_reason TEXT"
|
||||||
)
|
)
|
||||||
|
if "qualitative_rating" not in result_cols:
|
||||||
|
conn.execute(
|
||||||
|
"ALTER TABLE combination_results ADD COLUMN qualitative_rating TEXT"
|
||||||
|
)
|
||||||
|
|
||||||
# Backfill: cost_efficiency is lower-is-better in all domains
|
# Backfill: cost_efficiency is lower-is-better in all domains
|
||||||
conn.execute(
|
conn.execute(
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from physcom.engine.combinator import generate_combinations
|
|||||||
from physcom.engine.constraint_resolver import ConstraintResolver, ConstraintResult
|
from physcom.engine.constraint_resolver import ConstraintResolver, ConstraintResult
|
||||||
from physcom.engine.scorer import Scorer
|
from physcom.engine.scorer import Scorer
|
||||||
from physcom.llm.base import LLMProvider, LLMRateLimitError
|
from physcom.llm.base import LLMProvider, LLMRateLimitError
|
||||||
|
from physcom.llm.parsing import parse_rating
|
||||||
from physcom.models.combination import Combination, ScoredResult
|
from physcom.models.combination import Combination, ScoredResult
|
||||||
from physcom.models.domain import Domain, MetricBound
|
from physcom.models.domain import Domain, MetricBound
|
||||||
|
|
||||||
@@ -385,308 +386,57 @@ class Pipeline:
|
|||||||
# Prepare metric lookup
|
# Prepare metric lookup
|
||||||
bounds_by_name = {mb.metric_name: mb for mb in domain.metric_bounds}
|
bounds_by_name = {mb.metric_name: mb for mb in domain.metric_bounds}
|
||||||
|
|
||||||
# ── Combo-first loop ─────────────────────────────────────
|
# ── Phase-parallel: each pass runs to completion across every combo
|
||||||
# Deterministic passes (1, 3, and 2 without an LLM) defer commits and
|
# before the next pass starts, instead of walking each combo through
|
||||||
# get flushed periodically + in `finally` below -- a crash there costs
|
# every pass before moving to the next combo. This maximizes
|
||||||
# a cheap recompute, not lost work worth committing per write. Pass 4
|
# progress before the expensive/slow phase (pass 4's LLM calls) and
|
||||||
# (and pass 2 with an LLM) commit immediately after each call: those
|
# keeps that phase's cost visible on its own, separate from the
|
||||||
# are slow and crash-prone (see the QwQ timeout saga), so that result
|
# deterministic passes. It also removes any need for the two options
|
||||||
# is worth protecting the moment it lands.
|
# to reconcile: pass 2 is estimator-only now (no LLM call in it at
|
||||||
|
# all -- self.llm is reserved for pass 4), so there's no combo that
|
||||||
|
# touches an LLM in both pass 2 and pass 4, and nothing here needs a
|
||||||
|
# live/resumed conversation across passes.
|
||||||
|
#
|
||||||
|
# Deterministic passes (1, 2, 3) defer commits and get flushed
|
||||||
|
# periodically + in `finally` below -- a crash there costs a cheap
|
||||||
|
# recompute, not lost work worth committing per write. Pass 4
|
||||||
|
# commits immediately after each call: those are slow and
|
||||||
|
# crash-prone (see the QwQ timeout saga), so that result is worth
|
||||||
|
# protecting the moment it lands.
|
||||||
combos_since_commit = 0
|
combos_since_commit = 0
|
||||||
|
|
||||||
|
def _tick_commit() -> None:
|
||||||
|
nonlocal combos_since_commit
|
||||||
|
combos_since_commit += 1
|
||||||
|
if combos_since_commit >= 200:
|
||||||
|
self.repo.commit()
|
||||||
|
combos_since_commit = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for combo in combos:
|
if 1 in passes:
|
||||||
self._check_cancelled(run_id)
|
for combo in combos:
|
||||||
combos_since_commit += 1
|
self._check_cancelled(run_id)
|
||||||
if combos_since_commit >= 200:
|
_tick_commit()
|
||||||
self.repo.commit()
|
self._process_pass1(combo, domain, result, run_id)
|
||||||
combos_since_commit = 0
|
|
||||||
|
|
||||||
# Check existing progress for this combo in this domain
|
if 2 in passes:
|
||||||
existing_pass = self.repo.get_combo_pass_reached(
|
for combo in combos:
|
||||||
combo.id, domain.id
|
self._check_cancelled(run_id)
|
||||||
) or 0
|
_tick_commit()
|
||||||
|
self._process_pass2(combo, domain, bounds_by_name, result, run_id)
|
||||||
|
|
||||||
# Load existing result to preserve human review data
|
if 3 in passes:
|
||||||
existing_result = self.repo.get_existing_result(
|
for combo in combos:
|
||||||
combo.id, domain.id
|
self._check_cancelled(run_id)
|
||||||
)
|
_tick_commit()
|
||||||
|
self._process_pass3(
|
||||||
# ── Pass 1: Constraint Resolution ────────────────
|
combo, domain, bounds_by_name, result, score_threshold, run_id
|
||||||
if 1 in passes and existing_pass < 1:
|
|
||||||
cr: ConstraintResult = self.resolver.resolve(combo)
|
|
||||||
if cr.status == "p1_fail":
|
|
||||||
combo.status = "p1_fail"
|
|
||||||
combo.block_reason = "; ".join(cr.violations)
|
|
||||||
self.repo.update_combination_status(
|
|
||||||
combo.id, "p1_fail", combo.block_reason, commit=False
|
|
||||||
)
|
|
||||||
# Save a result row so failed combos appear in results
|
|
||||||
self.repo.save_result(
|
|
||||||
combo.id,
|
|
||||||
domain.id,
|
|
||||||
composite_score=0.0,
|
|
||||||
pass_reached=1,
|
|
||||||
commit=False,
|
|
||||||
)
|
|
||||||
result.pass1_failed += 1
|
|
||||||
self._update_run_counters(run_id, result, current_pass=1)
|
|
||||||
continue # p1_fail — skip remaining passes
|
|
||||||
else:
|
|
||||||
combo.status = "valid"
|
|
||||||
self.repo.update_combination_status(combo.id, "valid", commit=False)
|
|
||||||
|
|
||||||
# Domain constraint check (per-domain block only). combo.status
|
|
||||||
# stays "valid" here on purpose: it's domain-agnostic and the
|
|
||||||
# same combo can be blocked in this domain but valid in another.
|
|
||||||
# The per-domain block lives on combination_results.domain_block_reason
|
|
||||||
# (see count_combinations_by_status / get_all_results, which bucket on it).
|
|
||||||
if domain.constraints:
|
|
||||||
dc_result = self.resolver.check_domain_constraints(
|
|
||||||
combo, domain.constraints
|
|
||||||
)
|
|
||||||
if dc_result.status == "p1_fail":
|
|
||||||
self.repo.save_result(
|
|
||||||
combo.id, domain.id,
|
|
||||||
composite_score=0.0, pass_reached=1,
|
|
||||||
domain_block_reason="; ".join(
|
|
||||||
dc_result.violations
|
|
||||||
),
|
|
||||||
commit=False,
|
|
||||||
)
|
|
||||||
result.pass1_failed += 1
|
|
||||||
self._update_run_counters(
|
|
||||||
run_id, result, current_pass=1
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if cr.status == "conditional":
|
|
||||||
result.pass1_conditional += 1
|
|
||||||
else:
|
|
||||||
result.pass1_valid += 1
|
|
||||||
|
|
||||||
self._update_run_counters(run_id, result, current_pass=1)
|
|
||||||
elif 1 in passes:
|
|
||||||
# Already pass1'd — check if it failed
|
|
||||||
if combo.status.endswith("_fail"):
|
|
||||||
result.pass1_failed += 1
|
|
||||||
continue
|
|
||||||
# Check if domain-blocked from a prior run
|
|
||||||
if existing_result and existing_result["pass_reached"] == 1:
|
|
||||||
result.pass1_failed += 1
|
|
||||||
continue
|
|
||||||
result.pass1_valid += 1
|
|
||||||
else:
|
|
||||||
# Pass 1 not requested; check if failed from a prior run
|
|
||||||
if combo.status.endswith("_fail"):
|
|
||||||
result.pass1_failed += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# ── Pass 2: Physics Estimation ───────────────────
|
|
||||||
raw_metrics: dict[str, float] = {}
|
|
||||||
if 2 in passes and existing_pass < 2:
|
|
||||||
description = _describe_combination(combo)
|
|
||||||
if self.llm:
|
|
||||||
raw_metrics = self.llm.estimate_physics(
|
|
||||||
description, domain.metric_bounds
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raw_metrics = self._stub_estimate(combo, domain.metric_bounds)
|
|
||||||
|
|
||||||
# Save raw estimates immediately (crash-safe)
|
|
||||||
estimate_dicts = []
|
|
||||||
for mname, rval in raw_metrics.items():
|
|
||||||
mb = bounds_by_name.get(mname)
|
|
||||||
if mb and mb.metric_id:
|
|
||||||
estimate_dicts.append({
|
|
||||||
"metric_id": mb.metric_id,
|
|
||||||
"raw_value": rval,
|
|
||||||
"estimation_method": "llm" if self.llm else "stub",
|
|
||||||
"confidence": 1.0,
|
|
||||||
})
|
|
||||||
# LLM-produced estimates commit immediately (slow/crash-
|
|
||||||
# prone, worth protecting); stub estimates are instant
|
|
||||||
# and defer, same as the rest of the deterministic passes.
|
|
||||||
used_llm = self.llm is not None
|
|
||||||
if estimate_dicts:
|
|
||||||
self.repo.save_raw_estimates(
|
|
||||||
combo.id, domain.id, estimate_dicts, commit=used_llm
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check for all-zero estimates → p2_fail
|
|
||||||
if raw_metrics and all(v == 0.0 for v in raw_metrics.values()):
|
|
||||||
combo.status = "p2_fail"
|
|
||||||
combo.block_reason = "All metric estimates are zero"
|
|
||||||
self.repo.update_combination_status(
|
|
||||||
combo.id, "p2_fail", combo.block_reason, commit=used_llm
|
|
||||||
)
|
|
||||||
self.repo.save_result(
|
|
||||||
combo.id, domain.id,
|
|
||||||
composite_score=0.0, pass_reached=2,
|
|
||||||
commit=used_llm,
|
|
||||||
)
|
|
||||||
result.pass2_failed += 1
|
|
||||||
self._update_run_counters(run_id, result, current_pass=2)
|
|
||||||
continue
|
|
||||||
|
|
||||||
result.pass2_estimated += 1
|
|
||||||
self._update_run_counters(run_id, result, current_pass=2)
|
|
||||||
elif 2 in passes:
|
|
||||||
# Already estimated — reload raw values from DB
|
|
||||||
existing_scores = self.repo.get_combination_scores(
|
|
||||||
combo.id, domain.id
|
|
||||||
)
|
|
||||||
raw_metrics = {
|
|
||||||
s["metric_name"]: s["raw_value"] for s in existing_scores
|
|
||||||
}
|
|
||||||
result.pass2_estimated += 1
|
|
||||||
else:
|
|
||||||
# Pass 2 not requested, use empty metrics
|
|
||||||
raw_metrics = {}
|
|
||||||
|
|
||||||
# ── Pass 3: Scoring & Ranking ────────────────────
|
|
||||||
if 3 in passes and existing_pass < 3:
|
|
||||||
sr = self.scorer.score_combination(combo, raw_metrics)
|
|
||||||
|
|
||||||
# Persist per-metric scores with normalized values
|
|
||||||
score_dicts = []
|
|
||||||
for s in sr.scores:
|
|
||||||
mb = bounds_by_name.get(s.metric_name)
|
|
||||||
if mb and mb.metric_id:
|
|
||||||
score_dicts.append({
|
|
||||||
"metric_id": mb.metric_id,
|
|
||||||
"raw_value": s.raw_value,
|
|
||||||
"normalized_score": s.normalized_score,
|
|
||||||
"estimation_method": s.estimation_method,
|
|
||||||
"confidence": s.confidence,
|
|
||||||
})
|
|
||||||
if score_dicts:
|
|
||||||
self.repo.save_scores(combo.id, domain.id, score_dicts, commit=False)
|
|
||||||
|
|
||||||
# Preserve existing human data
|
|
||||||
novelty_flag = (
|
|
||||||
existing_result["novelty_flag"] if existing_result else None
|
|
||||||
)
|
|
||||||
human_notes = (
|
|
||||||
existing_result["human_notes"] if existing_result else None
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if sr.composite_score < score_threshold:
|
if 4 in passes and self.llm:
|
||||||
self.repo.save_result(
|
for combo in combos:
|
||||||
combo.id, domain.id,
|
self._check_cancelled(run_id)
|
||||||
sr.composite_score, pass_reached=3,
|
self._process_pass4(combo, domain, result, score_threshold, run_id)
|
||||||
novelty_flag=novelty_flag,
|
|
||||||
human_notes=human_notes,
|
|
||||||
commit=False,
|
|
||||||
)
|
|
||||||
combo.status = "p3_fail"
|
|
||||||
combo.block_reason = (
|
|
||||||
f"Composite score {sr.composite_score:.4f} "
|
|
||||||
f"below threshold {score_threshold}"
|
|
||||||
)
|
|
||||||
self.repo.update_combination_status(
|
|
||||||
combo.id, "p3_fail", combo.block_reason, commit=False
|
|
||||||
)
|
|
||||||
result.pass3_failed += 1
|
|
||||||
result.pass3_scored += 1
|
|
||||||
self._update_run_counters(run_id, result, current_pass=3)
|
|
||||||
continue
|
|
||||||
|
|
||||||
self.repo.save_result(
|
|
||||||
combo.id,
|
|
||||||
domain.id,
|
|
||||||
sr.composite_score,
|
|
||||||
pass_reached=3,
|
|
||||||
novelty_flag=novelty_flag,
|
|
||||||
human_notes=human_notes,
|
|
||||||
commit=False,
|
|
||||||
)
|
|
||||||
self.repo.update_combination_status(combo.id, "scored", commit=False)
|
|
||||||
|
|
||||||
result.pass3_scored += 1
|
|
||||||
result.pass3_above_threshold += 1
|
|
||||||
|
|
||||||
self._update_run_counters(run_id, result, current_pass=3)
|
|
||||||
elif 3 in passes and existing_pass >= 3:
|
|
||||||
# Already scored — count it
|
|
||||||
result.pass3_scored += 1
|
|
||||||
if existing_result and existing_result["composite_score"] is not None:
|
|
||||||
if existing_result["composite_score"] >= score_threshold:
|
|
||||||
result.pass3_above_threshold += 1
|
|
||||||
|
|
||||||
# ── Pass 4: LLM Review ───────────────────────────
|
|
||||||
if 4 in passes and self.llm:
|
|
||||||
cur_pass = self.repo.get_combo_pass_reached(
|
|
||||||
combo.id, domain.id
|
|
||||||
) or 0
|
|
||||||
if cur_pass < 4:
|
|
||||||
cur_result = self.repo.get_existing_result(
|
|
||||||
combo.id, domain.id
|
|
||||||
)
|
|
||||||
if (
|
|
||||||
cur_result
|
|
||||||
and cur_result["composite_score"] is not None
|
|
||||||
and cur_result["composite_score"] >= score_threshold
|
|
||||||
):
|
|
||||||
description = _describe_combination(combo)
|
|
||||||
db_scores = self.repo.get_combination_scores(
|
|
||||||
combo.id, domain.id
|
|
||||||
)
|
|
||||||
score_dict = {
|
|
||||||
s["metric_name"]: s["normalized_score"]
|
|
||||||
for s in db_scores
|
|
||||||
if s["normalized_score"] is not None
|
|
||||||
}
|
|
||||||
raw_dict = {
|
|
||||||
s["metric_name"]: s["raw_value"]
|
|
||||||
for s in db_scores
|
|
||||||
if s["raw_value"] is not None
|
|
||||||
}
|
|
||||||
review_result: tuple[str, bool] | None = None
|
|
||||||
try:
|
|
||||||
review_result = self.llm.review_plausibility(
|
|
||||||
description, raw_dict, score_dict, domain.metric_bounds
|
|
||||||
)
|
|
||||||
except LLMRateLimitError as exc:
|
|
||||||
self._wait_for_rate_limit(run_id, exc.retry_after)
|
|
||||||
try:
|
|
||||||
review_result = self.llm.review_plausibility(
|
|
||||||
description, raw_dict, score_dict, domain.metric_bounds
|
|
||||||
)
|
|
||||||
except LLMRateLimitError:
|
|
||||||
pass # still limited; skip, retry next run
|
|
||||||
if review_result is not None:
|
|
||||||
review_text, plausible = review_result
|
|
||||||
if not plausible:
|
|
||||||
self.repo.save_result(
|
|
||||||
combo.id, domain.id,
|
|
||||||
cur_result["composite_score"],
|
|
||||||
pass_reached=4,
|
|
||||||
novelty_flag=cur_result.get("novelty_flag"),
|
|
||||||
llm_review=review_text,
|
|
||||||
human_notes=cur_result.get("human_notes"),
|
|
||||||
)
|
|
||||||
combo.status = "p4_fail"
|
|
||||||
combo.block_reason = "LLM deemed implausible"
|
|
||||||
self.repo.update_combination_status(
|
|
||||||
combo.id, "p4_fail", combo.block_reason
|
|
||||||
)
|
|
||||||
result.pass4_failed += 1
|
|
||||||
else:
|
|
||||||
self.repo.save_result(
|
|
||||||
combo.id, domain.id,
|
|
||||||
cur_result["composite_score"],
|
|
||||||
pass_reached=4,
|
|
||||||
novelty_flag=cur_result.get("novelty_flag"),
|
|
||||||
llm_review=review_text,
|
|
||||||
human_notes=cur_result.get("human_notes"),
|
|
||||||
)
|
|
||||||
self.repo.update_combination_status(
|
|
||||||
combo.id, "llm_reviewed"
|
|
||||||
)
|
|
||||||
result.pass4_reviewed += 1
|
|
||||||
self._update_run_counters(
|
|
||||||
run_id, result, current_pass=4
|
|
||||||
)
|
|
||||||
|
|
||||||
except CancelledError:
|
except CancelledError:
|
||||||
if run_id is not None:
|
if run_id is not None:
|
||||||
@@ -716,6 +466,271 @@ class Pipeline:
|
|||||||
result.top_results = self.repo.get_top_results(domain.name, limit=20)
|
result.top_results = self.repo.get_top_results(domain.name, limit=20)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _already_dead(combo: Combination, existing_result: dict | None) -> bool:
|
||||||
|
"""True if this combo is dead for every pass after 1 -- either a
|
||||||
|
generic failure (status ends in _fail) or a domain-specific block.
|
||||||
|
The domain-block case needs the extra existing_result check:
|
||||||
|
combo.status stays "valid" on purpose for it (domain-agnostic,
|
||||||
|
see _process_pass1's own comment on this), so pass_reached==1 with
|
||||||
|
the block already recorded is what actually marks it dead --
|
||||||
|
status alone isn't enough to catch it."""
|
||||||
|
if combo.status.endswith("_fail"):
|
||||||
|
return True
|
||||||
|
return bool(existing_result and existing_result["pass_reached"] == 1)
|
||||||
|
|
||||||
|
def _process_pass1(
|
||||||
|
self, combo: Combination, domain: Domain, result: PipelineResult, run_id: int | None
|
||||||
|
) -> None:
|
||||||
|
"""Constraint resolution for one combo. All writes deferred (commit=False)."""
|
||||||
|
existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0
|
||||||
|
if existing_pass >= 1:
|
||||||
|
if combo.status.endswith("_fail"):
|
||||||
|
result.pass1_failed += 1
|
||||||
|
return
|
||||||
|
existing_result = self.repo.get_existing_result(combo.id, domain.id)
|
||||||
|
if existing_result and existing_result["pass_reached"] == 1:
|
||||||
|
result.pass1_failed += 1
|
||||||
|
return
|
||||||
|
result.pass1_valid += 1
|
||||||
|
return
|
||||||
|
|
||||||
|
cr: ConstraintResult = self.resolver.resolve(combo)
|
||||||
|
if cr.status == "p1_fail":
|
||||||
|
combo.status = "p1_fail"
|
||||||
|
combo.block_reason = "; ".join(cr.violations)
|
||||||
|
self.repo.update_combination_status(
|
||||||
|
combo.id, "p1_fail", combo.block_reason, commit=False
|
||||||
|
)
|
||||||
|
# Save a result row so failed combos appear in results
|
||||||
|
self.repo.save_result(
|
||||||
|
combo.id, domain.id, composite_score=0.0, pass_reached=1, commit=False
|
||||||
|
)
|
||||||
|
result.pass1_failed += 1
|
||||||
|
self._update_run_counters(run_id, result, current_pass=1)
|
||||||
|
return
|
||||||
|
|
||||||
|
combo.status = "valid"
|
||||||
|
self.repo.update_combination_status(combo.id, "valid", commit=False)
|
||||||
|
|
||||||
|
# Domain constraint check (per-domain block only). combo.status stays
|
||||||
|
# "valid" here on purpose: it's domain-agnostic and the same combo can
|
||||||
|
# be blocked in this domain but valid in another. The per-domain
|
||||||
|
# block lives on combination_results.domain_block_reason (see
|
||||||
|
# count_combinations_by_status / get_all_results, which bucket on it).
|
||||||
|
if domain.constraints:
|
||||||
|
dc_result = self.resolver.check_domain_constraints(combo, domain.constraints)
|
||||||
|
if dc_result.status == "p1_fail":
|
||||||
|
self.repo.save_result(
|
||||||
|
combo.id, domain.id,
|
||||||
|
composite_score=0.0, pass_reached=1,
|
||||||
|
domain_block_reason="; ".join(dc_result.violations),
|
||||||
|
commit=False,
|
||||||
|
)
|
||||||
|
result.pass1_failed += 1
|
||||||
|
self._update_run_counters(run_id, result, current_pass=1)
|
||||||
|
return
|
||||||
|
|
||||||
|
if cr.status == "conditional":
|
||||||
|
result.pass1_conditional += 1
|
||||||
|
else:
|
||||||
|
result.pass1_valid += 1
|
||||||
|
self._update_run_counters(run_id, result, current_pass=1)
|
||||||
|
|
||||||
|
def _process_pass2(
|
||||||
|
self,
|
||||||
|
combo: Combination,
|
||||||
|
domain: Domain,
|
||||||
|
bounds_by_name: dict[str, MetricBound],
|
||||||
|
result: PipelineResult,
|
||||||
|
run_id: int | None,
|
||||||
|
) -> None:
|
||||||
|
"""Physics estimation for one combo. Estimator-only -- self.llm is
|
||||||
|
reserved for pass 4, never consulted here. All writes deferred."""
|
||||||
|
existing_result = self.repo.get_existing_result(combo.id, domain.id)
|
||||||
|
if self._already_dead(combo, existing_result):
|
||||||
|
return
|
||||||
|
existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0
|
||||||
|
if existing_pass >= 2:
|
||||||
|
result.pass2_estimated += 1
|
||||||
|
return
|
||||||
|
|
||||||
|
raw_metrics = self._stub_estimate(combo, domain.metric_bounds)
|
||||||
|
|
||||||
|
estimate_dicts = []
|
||||||
|
for mname, rval in raw_metrics.items():
|
||||||
|
mb = bounds_by_name.get(mname)
|
||||||
|
if mb and mb.metric_id:
|
||||||
|
estimate_dicts.append({
|
||||||
|
"metric_id": mb.metric_id,
|
||||||
|
"raw_value": rval,
|
||||||
|
"estimation_method": "stub",
|
||||||
|
"confidence": 1.0,
|
||||||
|
})
|
||||||
|
if estimate_dicts:
|
||||||
|
self.repo.save_raw_estimates(combo.id, domain.id, estimate_dicts, commit=False)
|
||||||
|
|
||||||
|
# Check for all-zero estimates → p2_fail
|
||||||
|
if raw_metrics and all(v == 0.0 for v in raw_metrics.values()):
|
||||||
|
combo.status = "p2_fail"
|
||||||
|
combo.block_reason = "All metric estimates are zero"
|
||||||
|
self.repo.update_combination_status(
|
||||||
|
combo.id, "p2_fail", combo.block_reason, commit=False
|
||||||
|
)
|
||||||
|
self.repo.save_result(
|
||||||
|
combo.id, domain.id, composite_score=0.0, pass_reached=2, commit=False
|
||||||
|
)
|
||||||
|
result.pass2_failed += 1
|
||||||
|
self._update_run_counters(run_id, result, current_pass=2)
|
||||||
|
return
|
||||||
|
|
||||||
|
result.pass2_estimated += 1
|
||||||
|
self._update_run_counters(run_id, result, current_pass=2)
|
||||||
|
|
||||||
|
def _process_pass3(
|
||||||
|
self,
|
||||||
|
combo: Combination,
|
||||||
|
domain: Domain,
|
||||||
|
bounds_by_name: dict[str, MetricBound],
|
||||||
|
result: PipelineResult,
|
||||||
|
score_threshold: float,
|
||||||
|
run_id: int | None,
|
||||||
|
) -> None:
|
||||||
|
"""Scoring for one combo. Reloads raw estimates from the DB (pass 2
|
||||||
|
ran as its own separate phase, not in-memory from this iteration).
|
||||||
|
All writes deferred."""
|
||||||
|
existing_result = self.repo.get_existing_result(combo.id, domain.id)
|
||||||
|
if self._already_dead(combo, existing_result):
|
||||||
|
return
|
||||||
|
existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0
|
||||||
|
if existing_pass >= 3:
|
||||||
|
result.pass3_scored += 1
|
||||||
|
if existing_result and existing_result["composite_score"] is not None:
|
||||||
|
if existing_result["composite_score"] >= score_threshold:
|
||||||
|
result.pass3_above_threshold += 1
|
||||||
|
return
|
||||||
|
|
||||||
|
existing_scores = self.repo.get_combination_scores(combo.id, domain.id)
|
||||||
|
raw_metrics = {s["metric_name"]: s["raw_value"] for s in existing_scores}
|
||||||
|
sr = self.scorer.score_combination(combo, raw_metrics)
|
||||||
|
|
||||||
|
score_dicts = []
|
||||||
|
for s in sr.scores:
|
||||||
|
mb = bounds_by_name.get(s.metric_name)
|
||||||
|
if mb and mb.metric_id:
|
||||||
|
score_dicts.append({
|
||||||
|
"metric_id": mb.metric_id,
|
||||||
|
"raw_value": s.raw_value,
|
||||||
|
"normalized_score": s.normalized_score,
|
||||||
|
"estimation_method": s.estimation_method,
|
||||||
|
"confidence": s.confidence,
|
||||||
|
})
|
||||||
|
if score_dicts:
|
||||||
|
self.repo.save_scores(combo.id, domain.id, score_dicts, commit=False)
|
||||||
|
|
||||||
|
# Preserve existing human data
|
||||||
|
novelty_flag = existing_result["novelty_flag"] if existing_result else None
|
||||||
|
human_notes = existing_result["human_notes"] if existing_result else None
|
||||||
|
|
||||||
|
if sr.composite_score < score_threshold:
|
||||||
|
self.repo.save_result(
|
||||||
|
combo.id, domain.id, sr.composite_score, pass_reached=3,
|
||||||
|
novelty_flag=novelty_flag, human_notes=human_notes, commit=False,
|
||||||
|
)
|
||||||
|
combo.status = "p3_fail"
|
||||||
|
combo.block_reason = (
|
||||||
|
f"Composite score {sr.composite_score:.4f} below threshold {score_threshold}"
|
||||||
|
)
|
||||||
|
self.repo.update_combination_status(
|
||||||
|
combo.id, "p3_fail", combo.block_reason, commit=False
|
||||||
|
)
|
||||||
|
result.pass3_failed += 1
|
||||||
|
result.pass3_scored += 1
|
||||||
|
self._update_run_counters(run_id, result, current_pass=3)
|
||||||
|
return
|
||||||
|
|
||||||
|
self.repo.save_result(
|
||||||
|
combo.id, domain.id, sr.composite_score, pass_reached=3,
|
||||||
|
novelty_flag=novelty_flag, human_notes=human_notes, commit=False,
|
||||||
|
)
|
||||||
|
self.repo.update_combination_status(combo.id, "scored", commit=False)
|
||||||
|
result.pass3_scored += 1
|
||||||
|
result.pass3_above_threshold += 1
|
||||||
|
self._update_run_counters(run_id, result, current_pass=3)
|
||||||
|
|
||||||
|
def _process_pass4(
|
||||||
|
self,
|
||||||
|
combo: Combination,
|
||||||
|
domain: Domain,
|
||||||
|
result: PipelineResult,
|
||||||
|
score_threshold: float,
|
||||||
|
run_id: int | None,
|
||||||
|
) -> None:
|
||||||
|
"""LLM plausibility review for one combo. Writes commit immediately
|
||||||
|
(default commit=True) -- slow, crash-prone calls worth protecting
|
||||||
|
the moment a result lands."""
|
||||||
|
cur_result = self.repo.get_existing_result(combo.id, domain.id)
|
||||||
|
if self._already_dead(combo, cur_result):
|
||||||
|
return
|
||||||
|
cur_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0
|
||||||
|
if cur_pass >= 4:
|
||||||
|
return
|
||||||
|
if not (
|
||||||
|
cur_result
|
||||||
|
and cur_result["composite_score"] is not None
|
||||||
|
and cur_result["composite_score"] >= score_threshold
|
||||||
|
):
|
||||||
|
return
|
||||||
|
|
||||||
|
description = _describe_combination(combo)
|
||||||
|
db_scores = self.repo.get_combination_scores(combo.id, domain.id)
|
||||||
|
score_dict = {
|
||||||
|
s["metric_name"]: s["normalized_score"]
|
||||||
|
for s in db_scores if s["normalized_score"] is not None
|
||||||
|
}
|
||||||
|
raw_dict = {
|
||||||
|
s["metric_name"]: s["raw_value"]
|
||||||
|
for s in db_scores if s["raw_value"] is not None
|
||||||
|
}
|
||||||
|
|
||||||
|
review_result: tuple[str, bool] | None = None
|
||||||
|
try:
|
||||||
|
review_result = self.llm.review_plausibility(
|
||||||
|
description, raw_dict, score_dict, domain
|
||||||
|
)
|
||||||
|
except LLMRateLimitError as exc:
|
||||||
|
self._wait_for_rate_limit(run_id, exc.retry_after)
|
||||||
|
try:
|
||||||
|
review_result = self.llm.review_plausibility(
|
||||||
|
description, raw_dict, score_dict, domain.metric_bounds
|
||||||
|
)
|
||||||
|
except LLMRateLimitError:
|
||||||
|
return # still limited; skip, retry next run
|
||||||
|
|
||||||
|
if review_result is None:
|
||||||
|
return
|
||||||
|
review_text, plausible = review_result
|
||||||
|
rating = parse_rating(review_text)
|
||||||
|
if not plausible:
|
||||||
|
self.repo.save_result(
|
||||||
|
combo.id, domain.id, cur_result["composite_score"], pass_reached=4,
|
||||||
|
novelty_flag=cur_result.get("novelty_flag"), llm_review=review_text,
|
||||||
|
human_notes=cur_result.get("human_notes"), qualitative_rating=rating,
|
||||||
|
)
|
||||||
|
combo.status = "p4_fail"
|
||||||
|
combo.block_reason = "LLM deemed implausible"
|
||||||
|
self.repo.update_combination_status(combo.id, "p4_fail", combo.block_reason)
|
||||||
|
result.pass4_failed += 1
|
||||||
|
else:
|
||||||
|
self.repo.save_result(
|
||||||
|
combo.id, domain.id, cur_result["composite_score"], pass_reached=4,
|
||||||
|
novelty_flag=cur_result.get("novelty_flag"), llm_review=review_text,
|
||||||
|
human_notes=cur_result.get("human_notes"), qualitative_rating=rating,
|
||||||
|
)
|
||||||
|
self.repo.update_combination_status(combo.id, "llm_reviewed")
|
||||||
|
result.pass4_reviewed += 1
|
||||||
|
self._update_run_counters(run_id, result, current_pass=4)
|
||||||
|
|
||||||
def _wait_for_rate_limit(self, run_id: int | None, retry_after: int) -> None:
|
def _wait_for_rate_limit(self, run_id: int | None, retry_after: int) -> None:
|
||||||
"""Mark run rate_limited, sleep with cancel checks, then resume."""
|
"""Mark run rate_limited, sleep with cancel checks, then resume."""
|
||||||
if run_id is not None:
|
if run_id is not None:
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
from physcom.models.domain import MetricBound
|
from physcom.models.domain import Domain, MetricBound
|
||||||
|
|
||||||
|
|
||||||
class LLMRateLimitError(Exception):
|
class LLMRateLimitError(Exception):
|
||||||
@@ -40,7 +40,7 @@ class LLMProvider(ABC):
|
|||||||
combination_description: str,
|
combination_description: str,
|
||||||
raw_metrics: dict[str, float],
|
raw_metrics: dict[str, float],
|
||||||
normalized_scores: dict[str, float],
|
normalized_scores: dict[str, float],
|
||||||
metrics: list[MetricBound],
|
domain: Domain,
|
||||||
) -> tuple[str, bool]:
|
) -> tuple[str, bool]:
|
||||||
"""Given a combination, its raw physical estimates, and their
|
"""Given a combination, its raw physical estimates, and their
|
||||||
normalized scores, return a (text, is_plausible) tuple:
|
normalized scores, return a (text, is_plausible) tuple:
|
||||||
@@ -50,6 +50,11 @@ class LLMProvider(ABC):
|
|||||||
normalized score) so the review can reason from the actual physics
|
normalized score) so the review can reason from the actual physics
|
||||||
rather than only a compressed 0-1 number, which can look
|
rather than only a compressed 0-1 number, which can look
|
||||||
deceptively bad for a metric whose scale was built for a different
|
deceptively bad for a metric whose scale was built for a different
|
||||||
kind of vehicle. `metrics` carries each metric's unit for
|
kind of vehicle. `domain` carries both each metric's unit (via
|
||||||
formatting the raw value meaningfully."""
|
domain.metric_bounds, for formatting the raw value meaningfully)
|
||||||
|
and the domain's own name/description, so the review judges a
|
||||||
|
metric like range against what THIS domain actually needs rather
|
||||||
|
than generic real-world expectations for the platform category
|
||||||
|
(e.g. a short-hop domain shouldn't get judged against typical
|
||||||
|
long-haul aircraft range)."""
|
||||||
...
|
...
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ def parse_verdict(text: str) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def parse_rating(text: str) -> str | None:
|
||||||
|
"""Extract RATING: LOW/MEDIUM/HIGH from response; None if absent (older
|
||||||
|
reviews saved before this existed, or a malformed response)."""
|
||||||
|
m = re.search(r"RATING:\s*(LOW|MEDIUM|HIGH)", text, re.IGNORECASE)
|
||||||
|
return m.group(1).upper() if m else None
|
||||||
|
|
||||||
|
|
||||||
def parse_metric_json(text: str, metrics: list[MetricBound]) -> dict[str, float]:
|
def parse_metric_json(text: str, metrics: list[MetricBound]) -> dict[str, float]:
|
||||||
"""Strip markdown fences and parse JSON; fall back to each metric's own
|
"""Strip markdown fences and parse JSON; fall back to each metric's own
|
||||||
norm_min/norm_max midpoint on error — a flat constant like 0.5 is
|
norm_min/norm_max midpoint on error — a flat constant like 0.5 is
|
||||||
|
|||||||
@@ -116,6 +116,14 @@ You are reviewing a transportation concept for real-world viability — could th
|
|||||||
actually be built and operated safely. Whether it is new, exciting, or original
|
actually be built and operated safely. Whether it is new, exciting, or original
|
||||||
is NOT the question.
|
is NOT the question.
|
||||||
|
|
||||||
|
## Domain
|
||||||
|
This concept is being evaluated for "{domain_name}": {domain_description}
|
||||||
|
Judge every metric against what THIS domain actually needs, not general
|
||||||
|
expectations for the platform category. A range far beyond what this domain
|
||||||
|
requires is a strength or a non-issue, never a weakness -- don't reason about
|
||||||
|
range, speed, or capacity by comparing to what other vehicles of this type
|
||||||
|
typically have in general use; compare to what this specific domain calls for.
|
||||||
|
|
||||||
## Concept
|
## Concept
|
||||||
{description}
|
{description}
|
||||||
|
|
||||||
@@ -139,6 +147,14 @@ before treating the score as evidence of a problem.
|
|||||||
|
|
||||||
{scores}
|
{scores}
|
||||||
|
|
||||||
|
Safety and accessibility (infrastructure/regulatory availability) are NOT
|
||||||
|
among the scores above — neither reduces to a physics formula the way the
|
||||||
|
metrics above do, so nothing here estimates them numerically. Reason about
|
||||||
|
both directly from the concept description: does this combination carry a
|
||||||
|
specific safety hazard, and is the infrastructure/regulatory environment it
|
||||||
|
needs realistic? Both feed into the RATING below as qualitative judgment
|
||||||
|
calls, not as scores of their own.
|
||||||
|
|
||||||
## What makes something IMPLAUSIBLE
|
## What makes something IMPLAUSIBLE
|
||||||
Mark IMPLAUSIBLE if either of these is true:
|
Mark IMPLAUSIBLE if either of these is true:
|
||||||
- It is physically or engineering-wise impossible given the components
|
- It is physically or engineering-wise impossible given the components
|
||||||
@@ -150,15 +166,13 @@ Mark IMPLAUSIBLE if either of these is true:
|
|||||||
fatiguing a hull over time is a real structural risk, not just "explosives
|
fatiguing a hull over time is a real structural risk, not just "explosives
|
||||||
are dangerous in general"; a fuel that's fine in the open becoming
|
are dangerous in general"; a fuel that's fine in the open becoming
|
||||||
concentrated in a sealed tube is a real risk, not just "fuel is
|
concentrated in a sealed tube is a real risk, not just "fuel is
|
||||||
flammable"). A low given safety score is a signal the pipeline already
|
flammable").
|
||||||
found something concerning — treat it as evidence, not noise to explain
|
|
||||||
away.
|
|
||||||
- Or: a real regulatory/infrastructure barrier with no plausible workaround.
|
- Or: a real regulatory/infrastructure barrier with no plausible workaround.
|
||||||
|
|
||||||
None of these make something implausible on their own:
|
None of these make something implausible on their own:
|
||||||
- being unoriginal or something like it already exists
|
- being unoriginal or something like it already exists
|
||||||
- being expensive, slow, or short-range
|
- being expensive, slow, or short-range
|
||||||
- a single mediocre score on one metric that isn't safety-related
|
- a single mediocre score on one metric
|
||||||
|
|
||||||
Most concepts that reach this review are ordinary and workable; reserve
|
Most concepts that reach this review are ordinary and workable; reserve
|
||||||
IMPLAUSIBLE for a real, specific problem you can name — but don't require
|
IMPLAUSIBLE for a real, specific problem you can name — but don't require
|
||||||
@@ -167,14 +181,26 @@ Reason from the physics and engineering actually described here, not from
|
|||||||
whether something like it already exists — novelty or lack of it is not
|
whether something like it already exists — novelty or lack of it is not
|
||||||
evidence either way.
|
evidence either way.
|
||||||
|
|
||||||
|
## Overall Rating
|
||||||
|
Separately from the plausibility verdict, give ONE holistic rating —
|
||||||
|
LOW, MEDIUM, or HIGH — for how good this combination is overall. This is a
|
||||||
|
single combined judgment, not a separate score per attribute: weigh the
|
||||||
|
metric scores above together with your own qualitative read on safety and
|
||||||
|
accessibility into one rating, the way a person sizing up the whole concept
|
||||||
|
would, not a checklist of independent numbers.
|
||||||
|
|
||||||
## What to write
|
## What to write
|
||||||
In 2-4 sentences, give your reasoning, then check it against the scores
|
In 2-4 sentences, give your reasoning, then check it against the scores
|
||||||
above: if your reasoning conflicts with a score (e.g. you believe this is
|
above: if your reasoning conflicts with a score (e.g. you believe cost is
|
||||||
hazardous but its safety score is high), name the metric and say so
|
a serious problem but its cost score is high), name the metric and say so
|
||||||
explicitly — don't silently contradict a given score.
|
explicitly — don't silently contradict a given score.
|
||||||
|
|
||||||
Finish with exactly one line:
|
Finish with exactly two lines. For the first, pick exactly one:
|
||||||
|
RATING: LOW
|
||||||
|
RATING: MEDIUM
|
||||||
|
RATING: HIGH
|
||||||
|
|
||||||
|
Then, for the second, pick exactly one:
|
||||||
VERDICT: PLAUSIBLE
|
VERDICT: PLAUSIBLE
|
||||||
or
|
|
||||||
VERDICT: IMPLAUSIBLE
|
VERDICT: IMPLAUSIBLE
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from physcom.llm.prompts import (
|
|||||||
format_metrics_for_prompt,
|
format_metrics_for_prompt,
|
||||||
format_scores_for_prompt,
|
format_scores_for_prompt,
|
||||||
)
|
)
|
||||||
from physcom.models.domain import MetricBound
|
from physcom.models.domain import Domain, MetricBound
|
||||||
|
|
||||||
|
|
||||||
class GeminiLLMProvider(LLMProvider):
|
class GeminiLLMProvider(LLMProvider):
|
||||||
@@ -51,12 +51,14 @@ class GeminiLLMProvider(LLMProvider):
|
|||||||
combination_description: str,
|
combination_description: str,
|
||||||
raw_metrics: dict[str, float],
|
raw_metrics: dict[str, float],
|
||||||
normalized_scores: dict[str, float],
|
normalized_scores: dict[str, float],
|
||||||
metrics: list[MetricBound],
|
domain: Domain,
|
||||||
) -> tuple[str, bool]:
|
) -> tuple[str, bool]:
|
||||||
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, metrics)
|
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, domain.metric_bounds)
|
||||||
prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
|
prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
|
||||||
description=combination_description,
|
description=combination_description,
|
||||||
scores=scores_str,
|
scores=scores_str,
|
||||||
|
domain_name=domain.name,
|
||||||
|
domain_description=domain.description,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
response = self._client.models.generate_content(
|
response = self._client.models.generate_content(
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from physcom.llm.base import LLMProvider
|
from physcom.llm.base import LLMProvider
|
||||||
from physcom.models.domain import MetricBound
|
from physcom.models.domain import Domain, MetricBound
|
||||||
|
|
||||||
|
|
||||||
class MockLLMProvider(LLMProvider):
|
class MockLLMProvider(LLMProvider):
|
||||||
@@ -25,7 +25,7 @@ class MockLLMProvider(LLMProvider):
|
|||||||
combination_description: str,
|
combination_description: str,
|
||||||
raw_metrics: dict[str, float],
|
raw_metrics: dict[str, float],
|
||||||
normalized_scores: dict[str, float],
|
normalized_scores: dict[str, float],
|
||||||
metrics: list[MetricBound],
|
domain: Domain,
|
||||||
) -> tuple[str, bool]:
|
) -> tuple[str, bool]:
|
||||||
avg = sum(normalized_scores.values()) / max(len(normalized_scores), 1)
|
avg = sum(normalized_scores.values()) / max(len(normalized_scores), 1)
|
||||||
if avg > 0.5:
|
if avg > 0.5:
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from physcom.llm.prompts import (
|
|||||||
format_metrics_for_prompt,
|
format_metrics_for_prompt,
|
||||||
format_scores_for_prompt,
|
format_scores_for_prompt,
|
||||||
)
|
)
|
||||||
from physcom.models.domain import MetricBound
|
from physcom.models.domain import Domain, MetricBound
|
||||||
|
|
||||||
|
|
||||||
class OllamaLLMProvider(LLMProvider):
|
class OllamaLLMProvider(LLMProvider):
|
||||||
@@ -39,12 +39,14 @@ class OllamaLLMProvider(LLMProvider):
|
|||||||
combination_description: str,
|
combination_description: str,
|
||||||
raw_metrics: dict[str, float],
|
raw_metrics: dict[str, float],
|
||||||
normalized_scores: dict[str, float],
|
normalized_scores: dict[str, float],
|
||||||
metrics: list[MetricBound],
|
domain: Domain,
|
||||||
) -> tuple[str, bool]:
|
) -> tuple[str, bool]:
|
||||||
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, metrics)
|
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, domain.metric_bounds)
|
||||||
prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
|
prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
|
||||||
description=combination_description,
|
description=combination_description,
|
||||||
scores=scores_str,
|
scores=scores_str,
|
||||||
|
domain_name=domain.name,
|
||||||
|
domain_description=domain.description,
|
||||||
)
|
)
|
||||||
text = self._generate(prompt, json_mode=False).strip()
|
text = self._generate(prompt, json_mode=False).strip()
|
||||||
return (text, parse_verdict(text))
|
return (text, parse_verdict(text))
|
||||||
|
|||||||
@@ -728,11 +728,22 @@ URBAN_COMMUTING = Domain(
|
|||||||
name="urban_commuting",
|
name="urban_commuting",
|
||||||
description="Daily travel within a city, 1-50km range",
|
description="Daily travel within a city, 1-50km range",
|
||||||
metric_bounds=[
|
metric_bounds=[
|
||||||
MetricBound("power_density", weight=0.25, norm_min=1, norm_max=2000, unit="W/kg"),
|
# safety and availability removed from the scored/weighted metric set:
|
||||||
MetricBound("cost_efficiency", weight=0.25, norm_min=1e-5, norm_max=2e-3, unit="$/m", lower_is_better=True),
|
# both are judgment calls (risk assessment, infrastructure prevalence),
|
||||||
MetricBound("safety", weight=0.25, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
# not physics quantities with a formula, and running them through the
|
||||||
MetricBound("availability", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
# same log-normalize() built for physical quantities produced
|
||||||
MetricBound("range_fuel", weight=0.10, norm_min=5000, norm_max=500000, unit="m"),
|
# incoherent results (a safety raw value already declared as "0-1"
|
||||||
|
# getting re-normalized into a different, unexplainable 0-1 number --
|
||||||
|
# see combo 1540's review, where phi4 could only cite the post-
|
||||||
|
# normalization number with no way to justify it). Safety is now a
|
||||||
|
# qualitative consideration folded into pass 4's holistic RATING
|
||||||
|
# instead. Availability needs real per-infrastructure-type research
|
||||||
|
# this project hasn't done -- not scored anywhere for now rather than
|
||||||
|
# pretend a quick formula or an equally uninformed LLM guess settles it.
|
||||||
|
# Weights renormalized to sum to 1.0 across the remaining metrics.
|
||||||
|
MetricBound("power_density", weight=0.4167, norm_min=1, norm_max=2000, unit="W/kg"),
|
||||||
|
MetricBound("cost_efficiency", weight=0.4167, norm_min=1e-5, norm_max=2e-3, unit="$/m", lower_is_better=True),
|
||||||
|
MetricBound("range_fuel", weight=0.1666, norm_min=5000, norm_max=500000, unit="m"),
|
||||||
],
|
],
|
||||||
constraints=[DomainConstraint("medium", ["ground", "air"])],
|
constraints=[DomainConstraint("medium", ["ground", "air"])],
|
||||||
)
|
)
|
||||||
@@ -741,11 +752,12 @@ INTERPLANETARY = Domain(
|
|||||||
name="interplanetary_travel",
|
name="interplanetary_travel",
|
||||||
description="Travel between planets within a solar system",
|
description="Travel between planets within a solar system",
|
||||||
metric_bounds=[
|
metric_bounds=[
|
||||||
MetricBound("power_density", weight=0.30, norm_min=10, norm_max=10000, unit="W/kg"),
|
# safety removed -- see URBAN_COMMUTING comment above. Weights
|
||||||
MetricBound("range_fuel", weight=0.30, norm_min=1e9, norm_max=1e13, unit="m"),
|
# renormalized across the remaining metrics.
|
||||||
MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
MetricBound("power_density", weight=0.375, norm_min=10, norm_max=10000, unit="W/kg"),
|
||||||
MetricBound("cost_efficiency", weight=0.10, norm_min=1.0, norm_max=1e6, unit="$/m", lower_is_better=True),
|
MetricBound("range_fuel", weight=0.375, norm_min=1e9, norm_max=1e13, unit="m"),
|
||||||
MetricBound("range_degradation", weight=0.10, norm_min=8640000, norm_max=3.1536e9, unit="s"),
|
MetricBound("cost_efficiency", weight=0.125, norm_min=1.0, norm_max=1e6, unit="$/m", lower_is_better=True),
|
||||||
|
MetricBound("range_degradation", weight=0.125, norm_min=8640000, norm_max=3.1536e9, unit="s"),
|
||||||
],
|
],
|
||||||
constraints=[DomainConstraint("medium", ["space"])],
|
constraints=[DomainConstraint("medium", ["space"])],
|
||||||
)
|
)
|
||||||
@@ -754,11 +766,12 @@ MARITIME_SHIPPING = Domain(
|
|||||||
name="maritime_shipping",
|
name="maritime_shipping",
|
||||||
description="Ocean cargo transport between ports, 100-40000km range",
|
description="Ocean cargo transport between ports, 100-40000km range",
|
||||||
metric_bounds=[
|
metric_bounds=[
|
||||||
MetricBound("power_density", weight=0.15, norm_min=1, norm_max=1000, unit="W/kg"),
|
# safety removed -- see URBAN_COMMUTING comment above. Weights
|
||||||
MetricBound("cargo_capacity", weight=0.25, norm_min=1000, norm_max=2e8, unit="kg"),
|
# renormalized across the remaining metrics.
|
||||||
MetricBound("cost_efficiency", weight=0.25, norm_min=1e-9, norm_max=1e-6, unit="$/(kg\u00b7m)", lower_is_better=True),
|
MetricBound("power_density", weight=0.1875, norm_min=1, norm_max=1000, unit="W/kg"),
|
||||||
MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
MetricBound("cargo_capacity", weight=0.3125, norm_min=1000, norm_max=2e8, unit="kg"),
|
||||||
MetricBound("range_fuel", weight=0.15, norm_min=100000, norm_max=40000000, unit="m"),
|
MetricBound("cost_efficiency", weight=0.3125, norm_min=1e-9, norm_max=1e-6, unit="$/(kg\u00b7m)", lower_is_better=True),
|
||||||
|
MetricBound("range_fuel", weight=0.1875, norm_min=100000, norm_max=40000000, unit="m"),
|
||||||
],
|
],
|
||||||
constraints=[DomainConstraint("medium", ["water"])],
|
constraints=[DomainConstraint("medium", ["water"])],
|
||||||
)
|
)
|
||||||
@@ -767,11 +780,12 @@ LAST_MILE_DELIVERY = Domain(
|
|||||||
name="last_mile_delivery",
|
name="last_mile_delivery",
|
||||||
description="Short-range package delivery within neighborhoods, 0.5-15km",
|
description="Short-range package delivery within neighborhoods, 0.5-15km",
|
||||||
metric_bounds=[
|
metric_bounds=[
|
||||||
MetricBound("power_density", weight=0.25, norm_min=1, norm_max=500, unit="W/kg"),
|
# safety removed -- see URBAN_COMMUTING comment above. Weights
|
||||||
MetricBound("cost_efficiency", weight=0.30, norm_min=1e-5, norm_max=5e-3, unit="$/m", lower_is_better=True),
|
# renormalized across the remaining metrics.
|
||||||
MetricBound("cargo_capacity_kg", weight=0.20, norm_min=1, norm_max=500, unit="kg"),
|
MetricBound("power_density", weight=0.2941, norm_min=1, norm_max=500, unit="W/kg"),
|
||||||
MetricBound("safety", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
MetricBound("cost_efficiency", weight=0.3529, norm_min=1e-5, norm_max=5e-3, unit="$/m", lower_is_better=True),
|
||||||
MetricBound("environmental_impact", weight=0.10, norm_min=0, norm_max=5e-4, unit="kg/m", lower_is_better=True),
|
MetricBound("cargo_capacity_kg", weight=0.2353, norm_min=1, norm_max=500, unit="kg"),
|
||||||
|
MetricBound("environmental_impact", weight=0.1177, norm_min=0, norm_max=5e-4, unit="kg/m", lower_is_better=True),
|
||||||
],
|
],
|
||||||
constraints=[DomainConstraint("medium", ["ground", "air"])],
|
constraints=[DomainConstraint("medium", ["ground", "air"])],
|
||||||
)
|
)
|
||||||
@@ -839,12 +853,11 @@ def load_transport_seed(repo) -> dict:
|
|||||||
counts["domains"] += 1
|
counts["domains"] += 1
|
||||||
except sqlite3.IntegrityError:
|
except sqlite3.IntegrityError:
|
||||||
pass
|
pass
|
||||||
# Backfill metric units and lower_is_better on existing DBs.
|
# Sync domain_metric_weights to exactly match this domain's current
|
||||||
for mb in domain.metric_bounds:
|
# metric_bounds on existing DBs -- upserts weight/norm_min/norm_max/
|
||||||
repo.ensure_metric(mb.metric_name, unit=mb.unit)
|
# unit for current metrics and removes any that were dropped (e.g.
|
||||||
repo.backfill_metric_unit(domain.name, mb.metric_name, mb.unit)
|
# safety/availability no longer scored).
|
||||||
if mb.lower_is_better:
|
repo.sync_domain_metric_weights(domain)
|
||||||
repo.backfill_lower_is_better(domain.name, mb.metric_name)
|
|
||||||
# Backfill domain constraints
|
# Backfill domain constraints
|
||||||
repo.replace_domain_constraints(domain)
|
repo.replace_domain_constraints(domain)
|
||||||
|
|
||||||
|
|||||||
@@ -25,9 +25,11 @@ def results_domain(domain_name: str):
|
|||||||
return redirect(url_for("results.results_index"))
|
return redirect(url_for("results.results_index"))
|
||||||
|
|
||||||
status_filter = request.args.get("status")
|
status_filter = request.args.get("status")
|
||||||
results = repo.get_all_results(domain_name, status=status_filter)
|
rating_filter = request.args.get("rating")
|
||||||
|
results = repo.get_all_results(domain_name, status=status_filter, rating=rating_filter)
|
||||||
# Domain-scoped status counts (only combos that have results in this domain)
|
# Domain-scoped status counts (only combos that have results in this domain)
|
||||||
statuses = repo.count_combinations_by_status(domain_name=domain_name)
|
statuses = repo.count_combinations_by_status(domain_name=domain_name)
|
||||||
|
ratings = repo.count_results_by_rating(domain_name)
|
||||||
|
|
||||||
return render_template(
|
return render_template(
|
||||||
"results/list.html",
|
"results/list.html",
|
||||||
@@ -35,7 +37,9 @@ def results_domain(domain_name: str):
|
|||||||
domain=domain,
|
domain=domain,
|
||||||
results=results,
|
results=results,
|
||||||
status_filter=status_filter,
|
status_filter=status_filter,
|
||||||
|
rating_filter=rating_filter,
|
||||||
statuses=statuses,
|
statuses=statuses,
|
||||||
|
ratings=ratings,
|
||||||
total_results=sum(statuses.values()),
|
total_results=sum(statuses.values()),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -101,6 +105,7 @@ def submit_review(domain_name: str, combo_id: int):
|
|||||||
novelty_flag=novelty_flag,
|
novelty_flag=novelty_flag,
|
||||||
llm_review=existing.get("llm_review") if existing else None,
|
llm_review=existing.get("llm_review") if existing else None,
|
||||||
human_notes=human_notes,
|
human_notes=human_notes,
|
||||||
|
qualitative_rating=existing.get("qualitative_rating") if existing else None,
|
||||||
)
|
)
|
||||||
repo.update_combination_status(combo_id, "reviewed")
|
repo.update_combination_status(combo_id, "reviewed")
|
||||||
|
|
||||||
|
|||||||
@@ -214,6 +214,9 @@ table.compact th, table.compact td { padding: 0.25rem 0.4rem; font-size: 0.83rem
|
|||||||
.badge-llm_reviewed { background: rgba(107,163,160,0.12); color: var(--accent-teal); border-color: rgba(107,163,160,0.25); }
|
.badge-llm_reviewed { background: rgba(107,163,160,0.12); color: var(--accent-teal); border-color: rgba(107,163,160,0.25); }
|
||||||
.badge-reviewed { background: rgba(155,142,196,0.12); color: var(--accent-violet); border-color: rgba(155,142,196,0.25); }
|
.badge-reviewed { background: rgba(155,142,196,0.12); color: var(--accent-violet); border-color: rgba(155,142,196,0.25); }
|
||||||
.badge-pending { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); }
|
.badge-pending { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); }
|
||||||
|
.badge-rating-low { background: rgba(184,92,92,0.12); color: var(--accent-red); border-color: rgba(184,92,92,0.25); }
|
||||||
|
.badge-rating-medium { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); }
|
||||||
|
.badge-rating-high { background: rgba(122,171,138,0.12); color: var(--accent-green); border-color: rgba(122,171,138,0.25); }
|
||||||
|
|
||||||
/* ── Buttons ─────────────────────────────────────────────── */
|
/* ── Buttons ─────────────────────────────────────────────── */
|
||||||
.btn {
|
.btn {
|
||||||
|
|||||||
@@ -50,12 +50,13 @@
|
|||||||
<div class="step-body">
|
<div class="step-body">
|
||||||
<h3>Physics Estimation</h3>
|
<h3>Physics Estimation</h3>
|
||||||
<p>
|
<p>
|
||||||
Surviving combinations get raw metric estimates — speed, cost,
|
Surviving combinations get raw metric estimates — power
|
||||||
safety, range — via heuristic stubs or an LLM provider that
|
density, cost, range — from a deterministic physics engine
|
||||||
reasons about the physical properties of each pairing.
|
that sizes each combination from its own declared attributes, not
|
||||||
|
a guess.
|
||||||
</p>
|
</p>
|
||||||
<div class="step-example">
|
<div class="step-example">
|
||||||
Bicycle + Human Pedalling → speed: 20 km/h, cost: $0.01/km
|
Bicycle + Human Muscle → power density: 4.4 W/kg, range: 500km
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -72,8 +73,8 @@
|
|||||||
Combinations are ranked within their domain.
|
Combinations are ranked within their domain.
|
||||||
</p>
|
</p>
|
||||||
<div class="step-example">
|
<div class="step-example">
|
||||||
Domain <code>urban_commuting</code> weights: speed 25%, cost 25%,
|
Domain <code>urban_commuting</code> weights: power density 42%,
|
||||||
safety 25%, availability 15%, range 10%
|
cost 42%, range 17%
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -85,9 +86,11 @@
|
|||||||
<div class="step-body">
|
<div class="step-body">
|
||||||
<h3>LLM Review</h3>
|
<h3>LLM Review</h3>
|
||||||
<p>
|
<p>
|
||||||
Top-scoring combinations are sent to a language model for plausibility
|
Top-scoring combinations are sent to a language model for a
|
||||||
and novelty assessment — catching physically valid but practically
|
plausibility verdict plus a holistic LOW/MEDIUM/HIGH rating —
|
||||||
absurd pairings.
|
weighing safety and accessibility as qualitative judgment calls
|
||||||
|
alongside the physics scores, catching physically valid but
|
||||||
|
practically absurd pairings.
|
||||||
</p>
|
</p>
|
||||||
<div class="step-example">
|
<div class="step-example">
|
||||||
"Train + Solar Sail: structurally valid constraints, but solar radiation
|
"Train + Solar Sail: structurally valid constraints, but solar radiation
|
||||||
@@ -163,14 +166,15 @@
|
|||||||
<div class="card concept-card">
|
<div class="card concept-card">
|
||||||
<h3>Metrics</h3>
|
<h3>Metrics</h3>
|
||||||
<p>
|
<p>
|
||||||
Quantitative axes like speed, cost, safety, and range. Each metric
|
Quantitative physics axes like power density, cost, and range. Each
|
||||||
has a domain-specific weight and normalization range. Some are
|
metric has a domain-specific weight and normalization range. Some
|
||||||
inverted — lower cost is better.
|
are inverted — lower cost is better. Safety and accessibility
|
||||||
|
are judgment calls, not physics quantities — they're weighed
|
||||||
|
qualitatively in the LLM review pass instead of scored here.
|
||||||
</p>
|
</p>
|
||||||
<div class="concept-examples">
|
<div class="concept-examples">
|
||||||
<span class="badge">speed</span>
|
<span class="badge">power_density</span>
|
||||||
<span class="badge">cost_efficiency</span>
|
<span class="badge">cost_efficiency</span>
|
||||||
<span class="badge">safety</span>
|
|
||||||
<span class="badge">range_fuel</span>
|
<span class="badge">range_fuel</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -27,6 +27,9 @@
|
|||||||
{% if result %}
|
{% if result %}
|
||||||
<dt>Composite Score</dt><dd class="score-cell">{{ "%.4f"|format(result.composite_score) }}</dd>
|
<dt>Composite Score</dt><dd class="score-cell">{{ "%.4f"|format(result.composite_score) }}</dd>
|
||||||
<dt>Pass Reached</dt><dd>{{ result.pass_reached }}</dd>
|
<dt>Pass Reached</dt><dd>{{ result.pass_reached }}</dd>
|
||||||
|
{% if result.qualitative_rating %}
|
||||||
|
<dt>Rating</dt><dd><span class="badge badge-rating-{{ result.qualitative_rating|lower }}">{{ result.qualitative_rating }}</span></dd>
|
||||||
|
{% endif %}
|
||||||
{% if result.novelty_flag %}
|
{% if result.novelty_flag %}
|
||||||
<dt>Novelty</dt><dd>{{ result.novelty_flag }}</dd>
|
<dt>Novelty</dt><dd>{{ result.novelty_flag }}</dd>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
@@ -102,11 +105,16 @@
|
|||||||
{%- elif s.raw_value >= mb.norm_max -%}
|
{%- elif s.raw_value >= mb.norm_max -%}
|
||||||
<span class="badge badge-{{ 'p1_fail' if mb.lower_is_better else 'valid' }}">at/above max{{ ' (worst)' if mb.lower_is_better else '' }}</span>
|
<span class="badge badge-{{ 'p1_fail' if mb.lower_is_better else 'valid' }}">at/above max{{ ' (worst)' if mb.lower_is_better else '' }}</span>
|
||||||
{%- else -%}
|
{%- else -%}
|
||||||
{% set pct = ((s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100) | int %}
|
{% set raw_pct = (s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100 %}
|
||||||
|
{# For lower_is_better metrics, raw_pct alone measures distance from norm_min,
|
||||||
|
not quality -- a value near norm_min (excellent, cost near its floor) would
|
||||||
|
otherwise render as a near-empty bar. Invert so the bar and percentage always
|
||||||
|
mean "how good", matching the normalized score's own higher-is-better convention. #}
|
||||||
|
{% set pct = ((100 - raw_pct) if mb.lower_is_better else raw_pct) | int %}
|
||||||
<div class="metric-bar-container">
|
<div class="metric-bar-container">
|
||||||
<div class="metric-bar" style="width: {{ pct }}%"></div>
|
<div class="metric-bar" style="width: {{ pct }}%"></div>
|
||||||
</div>
|
</div>
|
||||||
<span class="metric-bar-label">~{{ pct }}%{{ ' ↓' if mb.lower_is_better else '' }}</span>
|
<span class="metric-bar-label">~{{ pct }}%{{ ' (lower is better)' if mb.lower_is_better else '' }}</span>
|
||||||
{%- endif -%}
|
{%- endif -%}
|
||||||
{%- else -%}
|
{%- else -%}
|
||||||
—
|
—
|
||||||
|
|||||||
@@ -26,11 +26,11 @@
|
|||||||
|
|
||||||
{% if statuses %}
|
{% if statuses %}
|
||||||
<div class="filter-row">
|
<div class="filter-row">
|
||||||
<span>Filter:</span>
|
<span>Status:</span>
|
||||||
<a href="{{ url_for('results.results_domain', domain_name=domain.name) }}"
|
<a href="{{ url_for('results.results_domain', domain_name=domain.name, rating=rating_filter) }}"
|
||||||
class="btn btn-sm {{ '' if status_filter else 'btn-primary' }}">All ({{ total_results }})</a>
|
class="btn btn-sm {{ '' if status_filter else 'btn-primary' }}">All ({{ total_results }})</a>
|
||||||
{% for s, cnt in statuses.items() %}
|
{% for s, cnt in statuses.items() %}
|
||||||
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=s) }}"
|
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=s, rating=rating_filter) }}"
|
||||||
class="btn btn-sm {{ 'btn-primary' if status_filter == s else '' }}">
|
class="btn btn-sm {{ 'btn-primary' if status_filter == s else '' }}">
|
||||||
{{ s }} ({{ cnt }})
|
{{ s }} ({{ cnt }})
|
||||||
</a>
|
</a>
|
||||||
@@ -38,9 +38,25 @@
|
|||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% if ratings %}
|
||||||
|
<div class="filter-row">
|
||||||
|
<span>Rating:</span>
|
||||||
|
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=status_filter) }}"
|
||||||
|
class="btn btn-sm {{ '' if not rating_filter else 'btn-primary' }}">All</a>
|
||||||
|
{% for rt in ['HIGH', 'MEDIUM', 'LOW'] %}
|
||||||
|
{% if rt in ratings %}
|
||||||
|
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=status_filter, rating=rt) }}"
|
||||||
|
class="btn btn-sm {{ 'btn-primary' if rating_filter == rt else '' }}">
|
||||||
|
{{ rt }} ({{ ratings[rt] }})
|
||||||
|
</a>
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% if not results %}
|
{% if not results %}
|
||||||
{% if status_filter %}
|
{% if status_filter or rating_filter %}
|
||||||
<p class="empty">No results with status "{{ status_filter }}" in this domain.</p>
|
<p class="empty">No results matching that filter in this domain.</p>
|
||||||
{% else %}
|
{% else %}
|
||||||
<p class="empty">No results for this domain yet. <a href="{{ url_for('pipeline.pipeline_form') }}">Run the pipeline</a> first.</p>
|
<p class="empty">No results for this domain yet. <a href="{{ url_for('pipeline.pipeline_form') }}">Run the pipeline</a> first.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
@@ -52,6 +68,7 @@
|
|||||||
<th>Score</th>
|
<th>Score</th>
|
||||||
<th>Entities</th>
|
<th>Entities</th>
|
||||||
<th>Status</th>
|
<th>Status</th>
|
||||||
|
<th>Rating</th>
|
||||||
<th>Details</th>
|
<th>Details</th>
|
||||||
<th></th>
|
<th></th>
|
||||||
</tr>
|
</tr>
|
||||||
@@ -69,6 +86,13 @@
|
|||||||
<span class="badge badge-{{ r.combination.status }}">{{ r.combination.status }}</span>
|
<span class="badge badge-{{ r.combination.status }}">{{ r.combination.status }}</span>
|
||||||
{%- endif -%}
|
{%- endif -%}
|
||||||
</td>
|
</td>
|
||||||
|
<td>
|
||||||
|
{%- if r.qualitative_rating -%}
|
||||||
|
<span class="badge badge-rating-{{ r.qualitative_rating|lower }}">{{ r.qualitative_rating }}</span>
|
||||||
|
{%- else -%}
|
||||||
|
—
|
||||||
|
{%- endif -%}
|
||||||
|
</td>
|
||||||
<td class="block-reason-cell">
|
<td class="block-reason-cell">
|
||||||
{%- if r.domain_block_reason -%}
|
{%- if r.domain_block_reason -%}
|
||||||
{{ r.domain_block_reason }}
|
{{ r.domain_block_reason }}
|
||||||
|
|||||||
@@ -335,28 +335,34 @@ def test_p3_fail_below_threshold(seeded_repo):
|
|||||||
|
|
||||||
|
|
||||||
def test_p4_fail_implausible(seeded_repo):
|
def test_p4_fail_implausible(seeded_repo):
|
||||||
"""Combos deemed implausible by LLM should get p4_fail status."""
|
"""Combos deemed implausible by LLM should get p4_fail status.
|
||||||
|
|
||||||
|
Pass 2 is estimator-only now (never calls the LLM), so there's no way
|
||||||
|
to force every combo's raw estimates toward a controlled low/high value
|
||||||
|
the way MockLLMProvider's default_estimates used to. Force the pass-4
|
||||||
|
verdict directly instead -- what's under test here is pipeline.py's
|
||||||
|
wiring of review_plausibility's return value to status/counters, not
|
||||||
|
MockLLMProvider's avg-based heuristic.
|
||||||
|
"""
|
||||||
from physcom.llm.providers.mock import MockLLMProvider
|
from physcom.llm.providers.mock import MockLLMProvider
|
||||||
|
|
||||||
|
class AlwaysImplausibleLLM(MockLLMProvider):
|
||||||
|
def review_plausibility(self, description, raw_metrics, normalized_scores, domain):
|
||||||
|
return ("Always implausible for testing.", False)
|
||||||
|
|
||||||
repo = seeded_repo
|
repo = seeded_repo
|
||||||
domain = repo.get_domain("urban_commuting")
|
domain = repo.get_domain("urban_commuting")
|
||||||
|
|
||||||
resolver = ConstraintResolver()
|
resolver = ConstraintResolver()
|
||||||
scorer = Scorer(domain)
|
scorer = Scorer(domain)
|
||||||
# Low estimates → normalized scores avg <= 0.5 → MockLLMProvider returns (text, False)
|
|
||||||
# Use threshold=0.0 so no combo gets p3_fail and all reach pass 4
|
# Use threshold=0.0 so no combo gets p3_fail and all reach pass 4
|
||||||
mock_llm = MockLLMProvider(default_estimates={
|
pipeline = Pipeline(repo, resolver, scorer, llm=AlwaysImplausibleLLM())
|
||||||
"power_density": 0.1, "cost_efficiency": 0.1, "safety": 0.1,
|
|
||||||
"availability": 0.1, "range_fuel": 0.1,
|
|
||||||
})
|
|
||||||
pipeline = Pipeline(repo, resolver, scorer, llm=mock_llm)
|
|
||||||
|
|
||||||
result = pipeline.run(
|
result = pipeline.run(
|
||||||
domain, ["platform", "actuator", "energy_storage"],
|
domain, ["platform", "actuator", "energy_storage"],
|
||||||
score_threshold=0.0, passes=[1, 2, 3, 4],
|
score_threshold=0.0, passes=[1, 2, 3, 4],
|
||||||
)
|
)
|
||||||
|
|
||||||
# With low normalized scores (avg <= 0.5), reviewed combos should be p4_fail
|
|
||||||
assert result.pass4_failed > 0
|
assert result.pass4_failed > 0
|
||||||
assert result.pass4_reviewed == 0
|
assert result.pass4_reviewed == 0
|
||||||
|
|
||||||
@@ -367,20 +373,23 @@ def test_p4_fail_implausible(seeded_repo):
|
|||||||
|
|
||||||
|
|
||||||
def test_p4_pass_plausible(seeded_repo):
|
def test_p4_pass_plausible(seeded_repo):
|
||||||
"""Combos deemed plausible by LLM should get llm_reviewed status."""
|
"""Combos deemed plausible by LLM should get llm_reviewed status.
|
||||||
|
|
||||||
|
See test_p4_fail_implausible on why the verdict is forced directly
|
||||||
|
rather than via controlled pass-2 estimates.
|
||||||
|
"""
|
||||||
from physcom.llm.providers.mock import MockLLMProvider
|
from physcom.llm.providers.mock import MockLLMProvider
|
||||||
|
|
||||||
|
class AlwaysPlausibleLLM(MockLLMProvider):
|
||||||
|
def review_plausibility(self, description, raw_metrics, normalized_scores, domain):
|
||||||
|
return ("Always plausible for testing.", True)
|
||||||
|
|
||||||
repo = seeded_repo
|
repo = seeded_repo
|
||||||
domain = repo.get_domain("urban_commuting")
|
domain = repo.get_domain("urban_commuting")
|
||||||
|
|
||||||
resolver = ConstraintResolver()
|
resolver = ConstraintResolver()
|
||||||
scorer = Scorer(domain)
|
scorer = Scorer(domain)
|
||||||
# High estimates → avg > 0.5 → MockLLMProvider returns (text, True)
|
pipeline = Pipeline(repo, resolver, scorer, llm=AlwaysPlausibleLLM())
|
||||||
mock_llm = MockLLMProvider(default_estimates={
|
|
||||||
"power_density": 500.0, "cost_efficiency": 5e-4, "safety": 0.6,
|
|
||||||
"availability": 0.7, "range_fuel": 200000.0,
|
|
||||||
})
|
|
||||||
pipeline = Pipeline(repo, resolver, scorer, llm=mock_llm)
|
|
||||||
|
|
||||||
result = pipeline.run(
|
result = pipeline.run(
|
||||||
domain, ["platform", "actuator", "energy_storage"],
|
domain, ["platform", "actuator", "energy_storage"],
|
||||||
|
|||||||
Reference in New Issue
Block a user