drop safety/availability from scoring, holistic p4 rating, phase-parallel pipeline

safety and availability don't reduce to physics formulas the way
power_density/range_fuel/cost_efficiency do -- they're judgment calls
(risk assessment, infrastructure prevalence), and running them through the
same log-normalize() built for physical quantities produced incoherent
results: safety's raw value is already a "0-1" score, and normalizing it
again turned 0.6 into an unexplainable 0.678 that even the LLM reviewing
it could only cite, never justify (see combo 1540). Removed both from
domain_metric_weights (safety from 4 domains, availability from
urban_commuting) and renormalized the remaining weights to sum to 1.0.

Pass 4 now produces one holistic RATING (LOW/MEDIUM/HIGH) alongside the
existing VERDICT, with safety and accessibility folded in as qualitative
considerations feeding that single judgment rather than scored
separately -- not a checklist of independent numbers. New
qualitative_rating column, filterable in the results UI. Also added
domain name/description to the review prompt so the LLM judges a metric
like range against what the domain actually needs (urban_commuting:
1-50km) instead of generic real-world expectations for the platform
category -- confirmed live on a combo where phi4 had called a 396km range
"limited" by comparing to typical aircraft rather than a domain that
needs 1-50km.

Pass 2 is estimator-only now -- self.llm is never consulted there,
reserved entirely for pass 4. Restructured Pipeline.run() from combo-first
to phase-parallel: each pass now runs to completion across every combo
before the next pass starts, rather than walking each combo through all
four passes before the next combo. This surfaced a real bug: domain-
blocked combos (status stays "valid" by design, not "_fail") were
slipping past a naive status-based skip guard and getting silently
re-processed by pass 2. Fixed with a shared dead-combo check that catches
both generic failures and domain blocks correctly.

Also fixes a results-page display bug found while reviewing a live combo:
the per-metric "position" bar showed raw distance from norm_min without
inverting for lower_is_better metrics, so an excellent cost score (near
the good end) rendered as a ~0%, near-empty bar -- looked bad next to its
own 0.99 normalized score.

Validated live against phi4 (real Ollama calls, not mocked): full-domain
phase-parallel run (2,970 combos, estimator-only p2, 1.6s) followed by a
real pass-4 run (111 reviewed, 11m, 0 crashes, 0 null ratings). Two tests
that relied on the old LLM-driven pass 2 to force deterministic outcomes
were updated to test pass 4's verdict-wiring directly instead. All 100
tests pass.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-15 16:52:26 -05:00
parent 730a23bac3
commit 76f460499a
16 changed files with 569 additions and 403 deletions

View File

@@ -230,24 +230,37 @@ class Repository:
self.conn.commit() self.conn.commit()
return row["id"] return row["id"]
def backfill_lower_is_better(self, domain_name: str, metric_name: str) -> None: def sync_domain_metric_weights(self, domain: Domain) -> None:
"""Set lower_is_better=1 for an existing domain-metric row that still has the default 0.""" """Make domain_metric_weights exactly match domain.metric_bounds on an
already-seeded domain: upserts weight/norm_min/norm_max/unit for every
currently-declared metric, and deletes any row for a metric that's been
removed from the domain (e.g. safety/availability dropped from the
scored set). Safe to call whether the domain was just freshly inserted
or already existed.
"""
row = self.conn.execute(
"SELECT id FROM domains WHERE name = ?", (domain.name,)
).fetchone()
if not row:
return
domain_id = row["id"]
keep_ids = []
for mb in domain.metric_bounds:
metric_id = self.ensure_metric(mb.metric_name, unit=mb.unit)
keep_ids.append(metric_id)
self.conn.execute( self.conn.execute(
"""UPDATE domain_metric_weights SET lower_is_better = 1 """INSERT OR REPLACE INTO domain_metric_weights
WHERE lower_is_better = 0 (domain_id, metric_id, weight, norm_min, norm_max, lower_is_better, unit)
AND domain_id = (SELECT id FROM domains WHERE name = ?) VALUES (?, ?, ?, ?, ?, ?, ?)""",
AND metric_id = (SELECT id FROM metrics WHERE name = ?)""", (domain_id, metric_id, mb.weight, mb.norm_min, mb.norm_max,
(domain_name, metric_name), int(mb.lower_is_better), mb.unit),
) )
self.conn.commit() if keep_ids:
placeholders = ",".join("?" * len(keep_ids))
def backfill_metric_unit(self, domain_name: str, metric_name: str, unit: str) -> None:
"""Set this domain-metric row's unit — unit is domain-scoped, not global to the metric name."""
self.conn.execute( self.conn.execute(
"""UPDATE domain_metric_weights SET unit = ? f"""DELETE FROM domain_metric_weights
WHERE domain_id = (SELECT id FROM domains WHERE name = ?) WHERE domain_id = ? AND metric_id NOT IN ({placeholders})""",
AND metric_id = (SELECT id FROM metrics WHERE name = ?)""", (domain_id, *keep_ids),
(unit, domain_name, metric_name),
) )
self.conn.commit() self.conn.commit()
@@ -593,15 +606,18 @@ class Repository:
llm_review: str | None = None, llm_review: str | None = None,
human_notes: str | None = None, human_notes: str | None = None,
domain_block_reason: str | None = None, domain_block_reason: str | None = None,
qualitative_rating: str | None = None,
commit: bool = True, commit: bool = True,
) -> None: ) -> None:
self.conn.execute( self.conn.execute(
"""INSERT OR REPLACE INTO combination_results """INSERT OR REPLACE INTO combination_results
(combination_id, domain_id, composite_score, novelty_flag, (combination_id, domain_id, composite_score, novelty_flag,
llm_review, human_notes, pass_reached, domain_block_reason) llm_review, human_notes, pass_reached, domain_block_reason,
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", qualitative_rating)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(combo_id, domain_id, composite_score, novelty_flag, (combo_id, domain_id, composite_score, novelty_flag,
llm_review, human_notes, pass_reached, domain_block_reason), llm_review, human_notes, pass_reached, domain_block_reason,
qualitative_rating),
) )
if commit: if commit:
self.conn.commit() self.conn.commit()
@@ -641,6 +657,20 @@ class Repository:
).fetchall() ).fetchall()
return {r["status"]: r["cnt"] for r in rows} return {r["status"]: r["cnt"] for r in rows}
def count_results_by_rating(self, domain_name: str) -> dict[str, int]:
"""Count results by qualitative_rating (LOW/MEDIUM/HIGH) for a domain.
Rows with no rating (not yet pass-4 reviewed, or reviewed before this
existed) are excluded, not bucketed as a pseudo-status."""
rows = self.conn.execute(
"""SELECT cr.qualitative_rating as rating, COUNT(*) as cnt
FROM combination_results cr
JOIN domains d ON cr.domain_id = d.id
WHERE d.name = ? AND cr.qualitative_rating IS NOT NULL
GROUP BY cr.qualitative_rating""",
(domain_name,),
).fetchall()
return {r["rating"]: r["cnt"] for r in rows}
def get_pipeline_summary(self, domain_name: str) -> dict | None: def get_pipeline_summary(self, domain_name: str) -> dict | None:
"""Return a summary of results for a domain, or None if no results.""" """Return a summary of results for a domain, or None if no results."""
row = self.conn.execute( row = self.conn.execute(
@@ -685,8 +715,12 @@ class Repository:
).fetchone() ).fetchone()
return dict(row) if row else None return dict(row) if row else None
def get_all_results(self, domain_name: str, status: str | None = None) -> list[dict]: def get_all_results(
"""Return all results for a domain, optionally filtered by combo status.""" self, domain_name: str, status: str | None = None, rating: str | None = None
) -> list[dict]:
"""Return all results for a domain, optionally filtered by combo
status and/or qualitative_rating (LOW/MEDIUM/HIGH, independent filters
that combine with AND)."""
query = """SELECT cr.*, c.hash, c.status as combo_status, d.name as domain_name query = """SELECT cr.*, c.hash, c.status as combo_status, d.name as domain_name
FROM combination_results cr FROM combination_results cr
JOIN combinations c ON cr.combination_id = c.id JOIN combinations c ON cr.combination_id = c.id
@@ -698,6 +732,9 @@ class Repository:
elif status: elif status:
query += " AND c.status = ? AND cr.domain_block_reason IS NULL" query += " AND c.status = ? AND cr.domain_block_reason IS NULL"
params.append(status) params.append(status)
if rating:
query += " AND cr.qualitative_rating = ?"
params.append(rating)
query += " ORDER BY cr.composite_score DESC" query += " ORDER BY cr.composite_score DESC"
rows = self.conn.execute(query, params).fetchall() rows = self.conn.execute(query, params).fetchall()
combo_ids = [r["combination_id"] for r in rows] combo_ids = [r["combination_id"] for r in rows]
@@ -712,6 +749,7 @@ class Repository:
"pass_reached": r["pass_reached"], "pass_reached": r["pass_reached"],
"domain_id": r["domain_id"], "domain_id": r["domain_id"],
"domain_block_reason": r["domain_block_reason"], "domain_block_reason": r["domain_block_reason"],
"qualitative_rating": r["qualitative_rating"],
} }
for r in rows for r in rows
] ]

View File

@@ -91,6 +91,7 @@ CREATE TABLE IF NOT EXISTS combination_results (
human_notes TEXT, human_notes TEXT,
pass_reached INTEGER, pass_reached INTEGER,
domain_block_reason TEXT, domain_block_reason TEXT,
qualitative_rating TEXT,
UNIQUE(combination_id, domain_id) UNIQUE(combination_id, domain_id)
); );
@@ -165,6 +166,10 @@ def _migrate(conn: sqlite3.Connection) -> None:
conn.execute( conn.execute(
"ALTER TABLE combination_results ADD COLUMN domain_block_reason TEXT" "ALTER TABLE combination_results ADD COLUMN domain_block_reason TEXT"
) )
if "qualitative_rating" not in result_cols:
conn.execute(
"ALTER TABLE combination_results ADD COLUMN qualitative_rating TEXT"
)
# Backfill: cost_efficiency is lower-is-better in all domains # Backfill: cost_efficiency is lower-is-better in all domains
conn.execute( conn.execute(

View File

@@ -12,6 +12,7 @@ from physcom.engine.combinator import generate_combinations
from physcom.engine.constraint_resolver import ConstraintResolver, ConstraintResult from physcom.engine.constraint_resolver import ConstraintResolver, ConstraintResult
from physcom.engine.scorer import Scorer from physcom.engine.scorer import Scorer
from physcom.llm.base import LLMProvider, LLMRateLimitError from physcom.llm.base import LLMProvider, LLMRateLimitError
from physcom.llm.parsing import parse_rating
from physcom.models.combination import Combination, ScoredResult from physcom.models.combination import Combination, ScoredResult
from physcom.models.domain import Domain, MetricBound from physcom.models.domain import Domain, MetricBound
@@ -385,308 +386,57 @@ class Pipeline:
# Prepare metric lookup # Prepare metric lookup
bounds_by_name = {mb.metric_name: mb for mb in domain.metric_bounds} bounds_by_name = {mb.metric_name: mb for mb in domain.metric_bounds}
# ── Combo-first loop ───────────────────────────────────── # ── Phase-parallel: each pass runs to completion across every combo
# Deterministic passes (1, 3, and 2 without an LLM) defer commits and # before the next pass starts, instead of walking each combo through
# get flushed periodically + in `finally` below -- a crash there costs # every pass before moving to the next combo. This maximizes
# a cheap recompute, not lost work worth committing per write. Pass 4 # progress before the expensive/slow phase (pass 4's LLM calls) and
# (and pass 2 with an LLM) commit immediately after each call: those # keeps that phase's cost visible on its own, separate from the
# are slow and crash-prone (see the QwQ timeout saga), so that result # deterministic passes. It also removes any need for the two options
# is worth protecting the moment it lands. # to reconcile: pass 2 is estimator-only now (no LLM call in it at
# all -- self.llm is reserved for pass 4), so there's no combo that
# touches an LLM in both pass 2 and pass 4, and nothing here needs a
# live/resumed conversation across passes.
#
# Deterministic passes (1, 2, 3) defer commits and get flushed
# periodically + in `finally` below -- a crash there costs a cheap
# recompute, not lost work worth committing per write. Pass 4
# commits immediately after each call: those are slow and
# crash-prone (see the QwQ timeout saga), so that result is worth
# protecting the moment it lands.
combos_since_commit = 0 combos_since_commit = 0
try:
for combo in combos: def _tick_commit() -> None:
self._check_cancelled(run_id) nonlocal combos_since_commit
combos_since_commit += 1 combos_since_commit += 1
if combos_since_commit >= 200: if combos_since_commit >= 200:
self.repo.commit() self.repo.commit()
combos_since_commit = 0 combos_since_commit = 0
# Check existing progress for this combo in this domain try:
existing_pass = self.repo.get_combo_pass_reached( if 1 in passes:
combo.id, domain.id for combo in combos:
) or 0 self._check_cancelled(run_id)
_tick_commit()
self._process_pass1(combo, domain, result, run_id)
# Load existing result to preserve human review data if 2 in passes:
existing_result = self.repo.get_existing_result( for combo in combos:
combo.id, domain.id self._check_cancelled(run_id)
_tick_commit()
self._process_pass2(combo, domain, bounds_by_name, result, run_id)
if 3 in passes:
for combo in combos:
self._check_cancelled(run_id)
_tick_commit()
self._process_pass3(
combo, domain, bounds_by_name, result, score_threshold, run_id
) )
# ── Pass 1: Constraint Resolution ────────────────
if 1 in passes and existing_pass < 1:
cr: ConstraintResult = self.resolver.resolve(combo)
if cr.status == "p1_fail":
combo.status = "p1_fail"
combo.block_reason = "; ".join(cr.violations)
self.repo.update_combination_status(
combo.id, "p1_fail", combo.block_reason, commit=False
)
# Save a result row so failed combos appear in results
self.repo.save_result(
combo.id,
domain.id,
composite_score=0.0,
pass_reached=1,
commit=False,
)
result.pass1_failed += 1
self._update_run_counters(run_id, result, current_pass=1)
continue # p1_fail — skip remaining passes
else:
combo.status = "valid"
self.repo.update_combination_status(combo.id, "valid", commit=False)
# Domain constraint check (per-domain block only). combo.status
# stays "valid" here on purpose: it's domain-agnostic and the
# same combo can be blocked in this domain but valid in another.
# The per-domain block lives on combination_results.domain_block_reason
# (see count_combinations_by_status / get_all_results, which bucket on it).
if domain.constraints:
dc_result = self.resolver.check_domain_constraints(
combo, domain.constraints
)
if dc_result.status == "p1_fail":
self.repo.save_result(
combo.id, domain.id,
composite_score=0.0, pass_reached=1,
domain_block_reason="; ".join(
dc_result.violations
),
commit=False,
)
result.pass1_failed += 1
self._update_run_counters(
run_id, result, current_pass=1
)
continue
if cr.status == "conditional":
result.pass1_conditional += 1
else:
result.pass1_valid += 1
self._update_run_counters(run_id, result, current_pass=1)
elif 1 in passes:
# Already pass1'd — check if it failed
if combo.status.endswith("_fail"):
result.pass1_failed += 1
continue
# Check if domain-blocked from a prior run
if existing_result and existing_result["pass_reached"] == 1:
result.pass1_failed += 1
continue
result.pass1_valid += 1
else:
# Pass 1 not requested; check if failed from a prior run
if combo.status.endswith("_fail"):
result.pass1_failed += 1
continue
# ── Pass 2: Physics Estimation ───────────────────
raw_metrics: dict[str, float] = {}
if 2 in passes and existing_pass < 2:
description = _describe_combination(combo)
if self.llm:
raw_metrics = self.llm.estimate_physics(
description, domain.metric_bounds
)
else:
raw_metrics = self._stub_estimate(combo, domain.metric_bounds)
# Save raw estimates immediately (crash-safe)
estimate_dicts = []
for mname, rval in raw_metrics.items():
mb = bounds_by_name.get(mname)
if mb and mb.metric_id:
estimate_dicts.append({
"metric_id": mb.metric_id,
"raw_value": rval,
"estimation_method": "llm" if self.llm else "stub",
"confidence": 1.0,
})
# LLM-produced estimates commit immediately (slow/crash-
# prone, worth protecting); stub estimates are instant
# and defer, same as the rest of the deterministic passes.
used_llm = self.llm is not None
if estimate_dicts:
self.repo.save_raw_estimates(
combo.id, domain.id, estimate_dicts, commit=used_llm
)
# Check for all-zero estimates → p2_fail
if raw_metrics and all(v == 0.0 for v in raw_metrics.values()):
combo.status = "p2_fail"
combo.block_reason = "All metric estimates are zero"
self.repo.update_combination_status(
combo.id, "p2_fail", combo.block_reason, commit=used_llm
)
self.repo.save_result(
combo.id, domain.id,
composite_score=0.0, pass_reached=2,
commit=used_llm,
)
result.pass2_failed += 1
self._update_run_counters(run_id, result, current_pass=2)
continue
result.pass2_estimated += 1
self._update_run_counters(run_id, result, current_pass=2)
elif 2 in passes:
# Already estimated — reload raw values from DB
existing_scores = self.repo.get_combination_scores(
combo.id, domain.id
)
raw_metrics = {
s["metric_name"]: s["raw_value"] for s in existing_scores
}
result.pass2_estimated += 1
else:
# Pass 2 not requested, use empty metrics
raw_metrics = {}
# ── Pass 3: Scoring & Ranking ────────────────────
if 3 in passes and existing_pass < 3:
sr = self.scorer.score_combination(combo, raw_metrics)
# Persist per-metric scores with normalized values
score_dicts = []
for s in sr.scores:
mb = bounds_by_name.get(s.metric_name)
if mb and mb.metric_id:
score_dicts.append({
"metric_id": mb.metric_id,
"raw_value": s.raw_value,
"normalized_score": s.normalized_score,
"estimation_method": s.estimation_method,
"confidence": s.confidence,
})
if score_dicts:
self.repo.save_scores(combo.id, domain.id, score_dicts, commit=False)
# Preserve existing human data
novelty_flag = (
existing_result["novelty_flag"] if existing_result else None
)
human_notes = (
existing_result["human_notes"] if existing_result else None
)
if sr.composite_score < score_threshold:
self.repo.save_result(
combo.id, domain.id,
sr.composite_score, pass_reached=3,
novelty_flag=novelty_flag,
human_notes=human_notes,
commit=False,
)
combo.status = "p3_fail"
combo.block_reason = (
f"Composite score {sr.composite_score:.4f} "
f"below threshold {score_threshold}"
)
self.repo.update_combination_status(
combo.id, "p3_fail", combo.block_reason, commit=False
)
result.pass3_failed += 1
result.pass3_scored += 1
self._update_run_counters(run_id, result, current_pass=3)
continue
self.repo.save_result(
combo.id,
domain.id,
sr.composite_score,
pass_reached=3,
novelty_flag=novelty_flag,
human_notes=human_notes,
commit=False,
)
self.repo.update_combination_status(combo.id, "scored", commit=False)
result.pass3_scored += 1
result.pass3_above_threshold += 1
self._update_run_counters(run_id, result, current_pass=3)
elif 3 in passes and existing_pass >= 3:
# Already scored — count it
result.pass3_scored += 1
if existing_result and existing_result["composite_score"] is not None:
if existing_result["composite_score"] >= score_threshold:
result.pass3_above_threshold += 1
# ── Pass 4: LLM Review ───────────────────────────
if 4 in passes and self.llm: if 4 in passes and self.llm:
cur_pass = self.repo.get_combo_pass_reached( for combo in combos:
combo.id, domain.id self._check_cancelled(run_id)
) or 0 self._process_pass4(combo, domain, result, score_threshold, run_id)
if cur_pass < 4:
cur_result = self.repo.get_existing_result(
combo.id, domain.id
)
if (
cur_result
and cur_result["composite_score"] is not None
and cur_result["composite_score"] >= score_threshold
):
description = _describe_combination(combo)
db_scores = self.repo.get_combination_scores(
combo.id, domain.id
)
score_dict = {
s["metric_name"]: s["normalized_score"]
for s in db_scores
if s["normalized_score"] is not None
}
raw_dict = {
s["metric_name"]: s["raw_value"]
for s in db_scores
if s["raw_value"] is not None
}
review_result: tuple[str, bool] | None = None
try:
review_result = self.llm.review_plausibility(
description, raw_dict, score_dict, domain.metric_bounds
)
except LLMRateLimitError as exc:
self._wait_for_rate_limit(run_id, exc.retry_after)
try:
review_result = self.llm.review_plausibility(
description, raw_dict, score_dict, domain.metric_bounds
)
except LLMRateLimitError:
pass # still limited; skip, retry next run
if review_result is not None:
review_text, plausible = review_result
if not plausible:
self.repo.save_result(
combo.id, domain.id,
cur_result["composite_score"],
pass_reached=4,
novelty_flag=cur_result.get("novelty_flag"),
llm_review=review_text,
human_notes=cur_result.get("human_notes"),
)
combo.status = "p4_fail"
combo.block_reason = "LLM deemed implausible"
self.repo.update_combination_status(
combo.id, "p4_fail", combo.block_reason
)
result.pass4_failed += 1
else:
self.repo.save_result(
combo.id, domain.id,
cur_result["composite_score"],
pass_reached=4,
novelty_flag=cur_result.get("novelty_flag"),
llm_review=review_text,
human_notes=cur_result.get("human_notes"),
)
self.repo.update_combination_status(
combo.id, "llm_reviewed"
)
result.pass4_reviewed += 1
self._update_run_counters(
run_id, result, current_pass=4
)
except CancelledError: except CancelledError:
if run_id is not None: if run_id is not None:
@@ -716,6 +466,271 @@ class Pipeline:
result.top_results = self.repo.get_top_results(domain.name, limit=20) result.top_results = self.repo.get_top_results(domain.name, limit=20)
return result return result
@staticmethod
def _already_dead(combo: Combination, existing_result: dict | None) -> bool:
"""True if this combo is dead for every pass after 1 -- either a
generic failure (status ends in _fail) or a domain-specific block.
The domain-block case needs the extra existing_result check:
combo.status stays "valid" on purpose for it (domain-agnostic,
see _process_pass1's own comment on this), so pass_reached==1 with
the block already recorded is what actually marks it dead --
status alone isn't enough to catch it."""
if combo.status.endswith("_fail"):
return True
return bool(existing_result and existing_result["pass_reached"] == 1)
def _process_pass1(
self, combo: Combination, domain: Domain, result: PipelineResult, run_id: int | None
) -> None:
"""Constraint resolution for one combo. All writes deferred (commit=False)."""
existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0
if existing_pass >= 1:
if combo.status.endswith("_fail"):
result.pass1_failed += 1
return
existing_result = self.repo.get_existing_result(combo.id, domain.id)
if existing_result and existing_result["pass_reached"] == 1:
result.pass1_failed += 1
return
result.pass1_valid += 1
return
cr: ConstraintResult = self.resolver.resolve(combo)
if cr.status == "p1_fail":
combo.status = "p1_fail"
combo.block_reason = "; ".join(cr.violations)
self.repo.update_combination_status(
combo.id, "p1_fail", combo.block_reason, commit=False
)
# Save a result row so failed combos appear in results
self.repo.save_result(
combo.id, domain.id, composite_score=0.0, pass_reached=1, commit=False
)
result.pass1_failed += 1
self._update_run_counters(run_id, result, current_pass=1)
return
combo.status = "valid"
self.repo.update_combination_status(combo.id, "valid", commit=False)
# Domain constraint check (per-domain block only). combo.status stays
# "valid" here on purpose: it's domain-agnostic and the same combo can
# be blocked in this domain but valid in another. The per-domain
# block lives on combination_results.domain_block_reason (see
# count_combinations_by_status / get_all_results, which bucket on it).
if domain.constraints:
dc_result = self.resolver.check_domain_constraints(combo, domain.constraints)
if dc_result.status == "p1_fail":
self.repo.save_result(
combo.id, domain.id,
composite_score=0.0, pass_reached=1,
domain_block_reason="; ".join(dc_result.violations),
commit=False,
)
result.pass1_failed += 1
self._update_run_counters(run_id, result, current_pass=1)
return
if cr.status == "conditional":
result.pass1_conditional += 1
else:
result.pass1_valid += 1
self._update_run_counters(run_id, result, current_pass=1)
def _process_pass2(
self,
combo: Combination,
domain: Domain,
bounds_by_name: dict[str, MetricBound],
result: PipelineResult,
run_id: int | None,
) -> None:
"""Physics estimation for one combo. Estimator-only -- self.llm is
reserved for pass 4, never consulted here. All writes deferred."""
existing_result = self.repo.get_existing_result(combo.id, domain.id)
if self._already_dead(combo, existing_result):
return
existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0
if existing_pass >= 2:
result.pass2_estimated += 1
return
raw_metrics = self._stub_estimate(combo, domain.metric_bounds)
estimate_dicts = []
for mname, rval in raw_metrics.items():
mb = bounds_by_name.get(mname)
if mb and mb.metric_id:
estimate_dicts.append({
"metric_id": mb.metric_id,
"raw_value": rval,
"estimation_method": "stub",
"confidence": 1.0,
})
if estimate_dicts:
self.repo.save_raw_estimates(combo.id, domain.id, estimate_dicts, commit=False)
# Check for all-zero estimates → p2_fail
if raw_metrics and all(v == 0.0 for v in raw_metrics.values()):
combo.status = "p2_fail"
combo.block_reason = "All metric estimates are zero"
self.repo.update_combination_status(
combo.id, "p2_fail", combo.block_reason, commit=False
)
self.repo.save_result(
combo.id, domain.id, composite_score=0.0, pass_reached=2, commit=False
)
result.pass2_failed += 1
self._update_run_counters(run_id, result, current_pass=2)
return
result.pass2_estimated += 1
self._update_run_counters(run_id, result, current_pass=2)
def _process_pass3(
self,
combo: Combination,
domain: Domain,
bounds_by_name: dict[str, MetricBound],
result: PipelineResult,
score_threshold: float,
run_id: int | None,
) -> None:
"""Scoring for one combo. Reloads raw estimates from the DB (pass 2
ran as its own separate phase, not in-memory from this iteration).
All writes deferred."""
existing_result = self.repo.get_existing_result(combo.id, domain.id)
if self._already_dead(combo, existing_result):
return
existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0
if existing_pass >= 3:
result.pass3_scored += 1
if existing_result and existing_result["composite_score"] is not None:
if existing_result["composite_score"] >= score_threshold:
result.pass3_above_threshold += 1
return
existing_scores = self.repo.get_combination_scores(combo.id, domain.id)
raw_metrics = {s["metric_name"]: s["raw_value"] for s in existing_scores}
sr = self.scorer.score_combination(combo, raw_metrics)
score_dicts = []
for s in sr.scores:
mb = bounds_by_name.get(s.metric_name)
if mb and mb.metric_id:
score_dicts.append({
"metric_id": mb.metric_id,
"raw_value": s.raw_value,
"normalized_score": s.normalized_score,
"estimation_method": s.estimation_method,
"confidence": s.confidence,
})
if score_dicts:
self.repo.save_scores(combo.id, domain.id, score_dicts, commit=False)
# Preserve existing human data
novelty_flag = existing_result["novelty_flag"] if existing_result else None
human_notes = existing_result["human_notes"] if existing_result else None
if sr.composite_score < score_threshold:
self.repo.save_result(
combo.id, domain.id, sr.composite_score, pass_reached=3,
novelty_flag=novelty_flag, human_notes=human_notes, commit=False,
)
combo.status = "p3_fail"
combo.block_reason = (
f"Composite score {sr.composite_score:.4f} below threshold {score_threshold}"
)
self.repo.update_combination_status(
combo.id, "p3_fail", combo.block_reason, commit=False
)
result.pass3_failed += 1
result.pass3_scored += 1
self._update_run_counters(run_id, result, current_pass=3)
return
self.repo.save_result(
combo.id, domain.id, sr.composite_score, pass_reached=3,
novelty_flag=novelty_flag, human_notes=human_notes, commit=False,
)
self.repo.update_combination_status(combo.id, "scored", commit=False)
result.pass3_scored += 1
result.pass3_above_threshold += 1
self._update_run_counters(run_id, result, current_pass=3)
def _process_pass4(
self,
combo: Combination,
domain: Domain,
result: PipelineResult,
score_threshold: float,
run_id: int | None,
) -> None:
"""LLM plausibility review for one combo. Writes commit immediately
(default commit=True) -- slow, crash-prone calls worth protecting
the moment a result lands."""
cur_result = self.repo.get_existing_result(combo.id, domain.id)
if self._already_dead(combo, cur_result):
return
cur_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0
if cur_pass >= 4:
return
if not (
cur_result
and cur_result["composite_score"] is not None
and cur_result["composite_score"] >= score_threshold
):
return
description = _describe_combination(combo)
db_scores = self.repo.get_combination_scores(combo.id, domain.id)
score_dict = {
s["metric_name"]: s["normalized_score"]
for s in db_scores if s["normalized_score"] is not None
}
raw_dict = {
s["metric_name"]: s["raw_value"]
for s in db_scores if s["raw_value"] is not None
}
review_result: tuple[str, bool] | None = None
try:
review_result = self.llm.review_plausibility(
description, raw_dict, score_dict, domain
)
except LLMRateLimitError as exc:
self._wait_for_rate_limit(run_id, exc.retry_after)
try:
review_result = self.llm.review_plausibility(
description, raw_dict, score_dict, domain.metric_bounds
)
except LLMRateLimitError:
return # still limited; skip, retry next run
if review_result is None:
return
review_text, plausible = review_result
rating = parse_rating(review_text)
if not plausible:
self.repo.save_result(
combo.id, domain.id, cur_result["composite_score"], pass_reached=4,
novelty_flag=cur_result.get("novelty_flag"), llm_review=review_text,
human_notes=cur_result.get("human_notes"), qualitative_rating=rating,
)
combo.status = "p4_fail"
combo.block_reason = "LLM deemed implausible"
self.repo.update_combination_status(combo.id, "p4_fail", combo.block_reason)
result.pass4_failed += 1
else:
self.repo.save_result(
combo.id, domain.id, cur_result["composite_score"], pass_reached=4,
novelty_flag=cur_result.get("novelty_flag"), llm_review=review_text,
human_notes=cur_result.get("human_notes"), qualitative_rating=rating,
)
self.repo.update_combination_status(combo.id, "llm_reviewed")
result.pass4_reviewed += 1
self._update_run_counters(run_id, result, current_pass=4)
def _wait_for_rate_limit(self, run_id: int | None, retry_after: int) -> None: def _wait_for_rate_limit(self, run_id: int | None, retry_after: int) -> None:
"""Mark run rate_limited, sleep with cancel checks, then resume.""" """Mark run rate_limited, sleep with cancel checks, then resume."""
if run_id is not None: if run_id is not None:

View File

@@ -4,7 +4,7 @@ from __future__ import annotations
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from physcom.models.domain import MetricBound from physcom.models.domain import Domain, MetricBound
class LLMRateLimitError(Exception): class LLMRateLimitError(Exception):
@@ -40,7 +40,7 @@ class LLMProvider(ABC):
combination_description: str, combination_description: str,
raw_metrics: dict[str, float], raw_metrics: dict[str, float],
normalized_scores: dict[str, float], normalized_scores: dict[str, float],
metrics: list[MetricBound], domain: Domain,
) -> tuple[str, bool]: ) -> tuple[str, bool]:
"""Given a combination, its raw physical estimates, and their """Given a combination, its raw physical estimates, and their
normalized scores, return a (text, is_plausible) tuple: normalized scores, return a (text, is_plausible) tuple:
@@ -50,6 +50,11 @@ class LLMProvider(ABC):
normalized score) so the review can reason from the actual physics normalized score) so the review can reason from the actual physics
rather than only a compressed 0-1 number, which can look rather than only a compressed 0-1 number, which can look
deceptively bad for a metric whose scale was built for a different deceptively bad for a metric whose scale was built for a different
kind of vehicle. `metrics` carries each metric's unit for kind of vehicle. `domain` carries both each metric's unit (via
formatting the raw value meaningfully.""" domain.metric_bounds, for formatting the raw value meaningfully)
and the domain's own name/description, so the review judges a
metric like range against what THIS domain actually needs rather
than generic real-world expectations for the platform category
(e.g. a short-hop domain shouldn't get judged against typical
long-haul aircraft range)."""
... ...

View File

@@ -16,6 +16,13 @@ def parse_verdict(text: str) -> bool:
return True return True
def parse_rating(text: str) -> str | None:
"""Extract RATING: LOW/MEDIUM/HIGH from response; None if absent (older
reviews saved before this existed, or a malformed response)."""
m = re.search(r"RATING:\s*(LOW|MEDIUM|HIGH)", text, re.IGNORECASE)
return m.group(1).upper() if m else None
def parse_metric_json(text: str, metrics: list[MetricBound]) -> dict[str, float]: def parse_metric_json(text: str, metrics: list[MetricBound]) -> dict[str, float]:
"""Strip markdown fences and parse JSON; fall back to each metric's own """Strip markdown fences and parse JSON; fall back to each metric's own
norm_min/norm_max midpoint on error — a flat constant like 0.5 is norm_min/norm_max midpoint on error — a flat constant like 0.5 is

View File

@@ -116,6 +116,14 @@ You are reviewing a transportation concept for real-world viability — could th
actually be built and operated safely. Whether it is new, exciting, or original actually be built and operated safely. Whether it is new, exciting, or original
is NOT the question. is NOT the question.
## Domain
This concept is being evaluated for "{domain_name}": {domain_description}
Judge every metric against what THIS domain actually needs, not general
expectations for the platform category. A range far beyond what this domain
requires is a strength or a non-issue, never a weakness -- don't reason about
range, speed, or capacity by comparing to what other vehicles of this type
typically have in general use; compare to what this specific domain calls for.
## Concept ## Concept
{description} {description}
@@ -139,6 +147,14 @@ before treating the score as evidence of a problem.
{scores} {scores}
Safety and accessibility (infrastructure/regulatory availability) are NOT
among the scores above — neither reduces to a physics formula the way the
metrics above do, so nothing here estimates them numerically. Reason about
both directly from the concept description: does this combination carry a
specific safety hazard, and is the infrastructure/regulatory environment it
needs realistic? Both feed into the RATING below as qualitative judgment
calls, not as scores of their own.
## What makes something IMPLAUSIBLE ## What makes something IMPLAUSIBLE
Mark IMPLAUSIBLE if either of these is true: Mark IMPLAUSIBLE if either of these is true:
- It is physically or engineering-wise impossible given the components - It is physically or engineering-wise impossible given the components
@@ -150,15 +166,13 @@ Mark IMPLAUSIBLE if either of these is true:
fatiguing a hull over time is a real structural risk, not just "explosives fatiguing a hull over time is a real structural risk, not just "explosives
are dangerous in general"; a fuel that's fine in the open becoming are dangerous in general"; a fuel that's fine in the open becoming
concentrated in a sealed tube is a real risk, not just "fuel is concentrated in a sealed tube is a real risk, not just "fuel is
flammable"). A low given safety score is a signal the pipeline already flammable").
found something concerning — treat it as evidence, not noise to explain
away.
- Or: a real regulatory/infrastructure barrier with no plausible workaround. - Or: a real regulatory/infrastructure barrier with no plausible workaround.
None of these make something implausible on their own: None of these make something implausible on their own:
- being unoriginal or something like it already exists - being unoriginal or something like it already exists
- being expensive, slow, or short-range - being expensive, slow, or short-range
- a single mediocre score on one metric that isn't safety-related - a single mediocre score on one metric
Most concepts that reach this review are ordinary and workable; reserve Most concepts that reach this review are ordinary and workable; reserve
IMPLAUSIBLE for a real, specific problem you can name — but don't require IMPLAUSIBLE for a real, specific problem you can name — but don't require
@@ -167,14 +181,26 @@ Reason from the physics and engineering actually described here, not from
whether something like it already exists — novelty or lack of it is not whether something like it already exists — novelty or lack of it is not
evidence either way. evidence either way.
## Overall Rating
Separately from the plausibility verdict, give ONE holistic rating —
LOW, MEDIUM, or HIGH — for how good this combination is overall. This is a
single combined judgment, not a separate score per attribute: weigh the
metric scores above together with your own qualitative read on safety and
accessibility into one rating, the way a person sizing up the whole concept
would, not a checklist of independent numbers.
## What to write ## What to write
In 2-4 sentences, give your reasoning, then check it against the scores In 2-4 sentences, give your reasoning, then check it against the scores
above: if your reasoning conflicts with a score (e.g. you believe this is above: if your reasoning conflicts with a score (e.g. you believe cost is
hazardous but its safety score is high), name the metric and say so a serious problem but its cost score is high), name the metric and say so
explicitly — don't silently contradict a given score. explicitly — don't silently contradict a given score.
Finish with exactly one line: Finish with exactly two lines. For the first, pick exactly one:
RATING: LOW
RATING: MEDIUM
RATING: HIGH
Then, for the second, pick exactly one:
VERDICT: PLAUSIBLE VERDICT: PLAUSIBLE
or
VERDICT: IMPLAUSIBLE VERDICT: IMPLAUSIBLE
""" """

View File

@@ -13,7 +13,7 @@ from physcom.llm.prompts import (
format_metrics_for_prompt, format_metrics_for_prompt,
format_scores_for_prompt, format_scores_for_prompt,
) )
from physcom.models.domain import MetricBound from physcom.models.domain import Domain, MetricBound
class GeminiLLMProvider(LLMProvider): class GeminiLLMProvider(LLMProvider):
@@ -51,12 +51,14 @@ class GeminiLLMProvider(LLMProvider):
combination_description: str, combination_description: str,
raw_metrics: dict[str, float], raw_metrics: dict[str, float],
normalized_scores: dict[str, float], normalized_scores: dict[str, float],
metrics: list[MetricBound], domain: Domain,
) -> tuple[str, bool]: ) -> tuple[str, bool]:
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, metrics) scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, domain.metric_bounds)
prompt = PLAUSIBILITY_REVIEW_PROMPT.format( prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
description=combination_description, description=combination_description,
scores=scores_str, scores=scores_str,
domain_name=domain.name,
domain_description=domain.description,
) )
try: try:
response = self._client.models.generate_content( response = self._client.models.generate_content(

View File

@@ -3,7 +3,7 @@
from __future__ import annotations from __future__ import annotations
from physcom.llm.base import LLMProvider from physcom.llm.base import LLMProvider
from physcom.models.domain import MetricBound from physcom.models.domain import Domain, MetricBound
class MockLLMProvider(LLMProvider): class MockLLMProvider(LLMProvider):
@@ -25,7 +25,7 @@ class MockLLMProvider(LLMProvider):
combination_description: str, combination_description: str,
raw_metrics: dict[str, float], raw_metrics: dict[str, float],
normalized_scores: dict[str, float], normalized_scores: dict[str, float],
metrics: list[MetricBound], domain: Domain,
) -> tuple[str, bool]: ) -> tuple[str, bool]:
avg = sum(normalized_scores.values()) / max(len(normalized_scores), 1) avg = sum(normalized_scores.values()) / max(len(normalized_scores), 1)
if avg > 0.5: if avg > 0.5:

View File

@@ -14,7 +14,7 @@ from physcom.llm.prompts import (
format_metrics_for_prompt, format_metrics_for_prompt,
format_scores_for_prompt, format_scores_for_prompt,
) )
from physcom.models.domain import MetricBound from physcom.models.domain import Domain, MetricBound
class OllamaLLMProvider(LLMProvider): class OllamaLLMProvider(LLMProvider):
@@ -39,12 +39,14 @@ class OllamaLLMProvider(LLMProvider):
combination_description: str, combination_description: str,
raw_metrics: dict[str, float], raw_metrics: dict[str, float],
normalized_scores: dict[str, float], normalized_scores: dict[str, float],
metrics: list[MetricBound], domain: Domain,
) -> tuple[str, bool]: ) -> tuple[str, bool]:
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, metrics) scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, domain.metric_bounds)
prompt = PLAUSIBILITY_REVIEW_PROMPT.format( prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
description=combination_description, description=combination_description,
scores=scores_str, scores=scores_str,
domain_name=domain.name,
domain_description=domain.description,
) )
text = self._generate(prompt, json_mode=False).strip() text = self._generate(prompt, json_mode=False).strip()
return (text, parse_verdict(text)) return (text, parse_verdict(text))

View File

@@ -728,11 +728,22 @@ URBAN_COMMUTING = Domain(
name="urban_commuting", name="urban_commuting",
description="Daily travel within a city, 1-50km range", description="Daily travel within a city, 1-50km range",
metric_bounds=[ metric_bounds=[
MetricBound("power_density", weight=0.25, norm_min=1, norm_max=2000, unit="W/kg"), # safety and availability removed from the scored/weighted metric set:
MetricBound("cost_efficiency", weight=0.25, norm_min=1e-5, norm_max=2e-3, unit="$/m", lower_is_better=True), # both are judgment calls (risk assessment, infrastructure prevalence),
MetricBound("safety", weight=0.25, norm_min=0.0, norm_max=1.0, unit="0-1"), # not physics quantities with a formula, and running them through the
MetricBound("availability", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"), # same log-normalize() built for physical quantities produced
MetricBound("range_fuel", weight=0.10, norm_min=5000, norm_max=500000, unit="m"), # incoherent results (a safety raw value already declared as "0-1"
# getting re-normalized into a different, unexplainable 0-1 number --
# see combo 1540's review, where phi4 could only cite the post-
# normalization number with no way to justify it). Safety is now a
# qualitative consideration folded into pass 4's holistic RATING
# instead. Availability needs real per-infrastructure-type research
# this project hasn't done -- not scored anywhere for now rather than
# pretend a quick formula or an equally uninformed LLM guess settles it.
# Weights renormalized to sum to 1.0 across the remaining metrics.
MetricBound("power_density", weight=0.4167, norm_min=1, norm_max=2000, unit="W/kg"),
MetricBound("cost_efficiency", weight=0.4167, norm_min=1e-5, norm_max=2e-3, unit="$/m", lower_is_better=True),
MetricBound("range_fuel", weight=0.1666, norm_min=5000, norm_max=500000, unit="m"),
], ],
constraints=[DomainConstraint("medium", ["ground", "air"])], constraints=[DomainConstraint("medium", ["ground", "air"])],
) )
@@ -741,11 +752,12 @@ INTERPLANETARY = Domain(
name="interplanetary_travel", name="interplanetary_travel",
description="Travel between planets within a solar system", description="Travel between planets within a solar system",
metric_bounds=[ metric_bounds=[
MetricBound("power_density", weight=0.30, norm_min=10, norm_max=10000, unit="W/kg"), # safety removed -- see URBAN_COMMUTING comment above. Weights
MetricBound("range_fuel", weight=0.30, norm_min=1e9, norm_max=1e13, unit="m"), # renormalized across the remaining metrics.
MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"), MetricBound("power_density", weight=0.375, norm_min=10, norm_max=10000, unit="W/kg"),
MetricBound("cost_efficiency", weight=0.10, norm_min=1.0, norm_max=1e6, unit="$/m", lower_is_better=True), MetricBound("range_fuel", weight=0.375, norm_min=1e9, norm_max=1e13, unit="m"),
MetricBound("range_degradation", weight=0.10, norm_min=8640000, norm_max=3.1536e9, unit="s"), MetricBound("cost_efficiency", weight=0.125, norm_min=1.0, norm_max=1e6, unit="$/m", lower_is_better=True),
MetricBound("range_degradation", weight=0.125, norm_min=8640000, norm_max=3.1536e9, unit="s"),
], ],
constraints=[DomainConstraint("medium", ["space"])], constraints=[DomainConstraint("medium", ["space"])],
) )
@@ -754,11 +766,12 @@ MARITIME_SHIPPING = Domain(
name="maritime_shipping", name="maritime_shipping",
description="Ocean cargo transport between ports, 100-40000km range", description="Ocean cargo transport between ports, 100-40000km range",
metric_bounds=[ metric_bounds=[
MetricBound("power_density", weight=0.15, norm_min=1, norm_max=1000, unit="W/kg"), # safety removed -- see URBAN_COMMUTING comment above. Weights
MetricBound("cargo_capacity", weight=0.25, norm_min=1000, norm_max=2e8, unit="kg"), # renormalized across the remaining metrics.
MetricBound("cost_efficiency", weight=0.25, norm_min=1e-9, norm_max=1e-6, unit="$/(kg\u00b7m)", lower_is_better=True), MetricBound("power_density", weight=0.1875, norm_min=1, norm_max=1000, unit="W/kg"),
MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"), MetricBound("cargo_capacity", weight=0.3125, norm_min=1000, norm_max=2e8, unit="kg"),
MetricBound("range_fuel", weight=0.15, norm_min=100000, norm_max=40000000, unit="m"), MetricBound("cost_efficiency", weight=0.3125, norm_min=1e-9, norm_max=1e-6, unit="$/(kg\u00b7m)", lower_is_better=True),
MetricBound("range_fuel", weight=0.1875, norm_min=100000, norm_max=40000000, unit="m"),
], ],
constraints=[DomainConstraint("medium", ["water"])], constraints=[DomainConstraint("medium", ["water"])],
) )
@@ -767,11 +780,12 @@ LAST_MILE_DELIVERY = Domain(
name="last_mile_delivery", name="last_mile_delivery",
description="Short-range package delivery within neighborhoods, 0.5-15km", description="Short-range package delivery within neighborhoods, 0.5-15km",
metric_bounds=[ metric_bounds=[
MetricBound("power_density", weight=0.25, norm_min=1, norm_max=500, unit="W/kg"), # safety removed -- see URBAN_COMMUTING comment above. Weights
MetricBound("cost_efficiency", weight=0.30, norm_min=1e-5, norm_max=5e-3, unit="$/m", lower_is_better=True), # renormalized across the remaining metrics.
MetricBound("cargo_capacity_kg", weight=0.20, norm_min=1, norm_max=500, unit="kg"), MetricBound("power_density", weight=0.2941, norm_min=1, norm_max=500, unit="W/kg"),
MetricBound("safety", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"), MetricBound("cost_efficiency", weight=0.3529, norm_min=1e-5, norm_max=5e-3, unit="$/m", lower_is_better=True),
MetricBound("environmental_impact", weight=0.10, norm_min=0, norm_max=5e-4, unit="kg/m", lower_is_better=True), MetricBound("cargo_capacity_kg", weight=0.2353, norm_min=1, norm_max=500, unit="kg"),
MetricBound("environmental_impact", weight=0.1177, norm_min=0, norm_max=5e-4, unit="kg/m", lower_is_better=True),
], ],
constraints=[DomainConstraint("medium", ["ground", "air"])], constraints=[DomainConstraint("medium", ["ground", "air"])],
) )
@@ -839,12 +853,11 @@ def load_transport_seed(repo) -> dict:
counts["domains"] += 1 counts["domains"] += 1
except sqlite3.IntegrityError: except sqlite3.IntegrityError:
pass pass
# Backfill metric units and lower_is_better on existing DBs. # Sync domain_metric_weights to exactly match this domain's current
for mb in domain.metric_bounds: # metric_bounds on existing DBs -- upserts weight/norm_min/norm_max/
repo.ensure_metric(mb.metric_name, unit=mb.unit) # unit for current metrics and removes any that were dropped (e.g.
repo.backfill_metric_unit(domain.name, mb.metric_name, mb.unit) # safety/availability no longer scored).
if mb.lower_is_better: repo.sync_domain_metric_weights(domain)
repo.backfill_lower_is_better(domain.name, mb.metric_name)
# Backfill domain constraints # Backfill domain constraints
repo.replace_domain_constraints(domain) repo.replace_domain_constraints(domain)

View File

@@ -25,9 +25,11 @@ def results_domain(domain_name: str):
return redirect(url_for("results.results_index")) return redirect(url_for("results.results_index"))
status_filter = request.args.get("status") status_filter = request.args.get("status")
results = repo.get_all_results(domain_name, status=status_filter) rating_filter = request.args.get("rating")
results = repo.get_all_results(domain_name, status=status_filter, rating=rating_filter)
# Domain-scoped status counts (only combos that have results in this domain) # Domain-scoped status counts (only combos that have results in this domain)
statuses = repo.count_combinations_by_status(domain_name=domain_name) statuses = repo.count_combinations_by_status(domain_name=domain_name)
ratings = repo.count_results_by_rating(domain_name)
return render_template( return render_template(
"results/list.html", "results/list.html",
@@ -35,7 +37,9 @@ def results_domain(domain_name: str):
domain=domain, domain=domain,
results=results, results=results,
status_filter=status_filter, status_filter=status_filter,
rating_filter=rating_filter,
statuses=statuses, statuses=statuses,
ratings=ratings,
total_results=sum(statuses.values()), total_results=sum(statuses.values()),
) )
@@ -101,6 +105,7 @@ def submit_review(domain_name: str, combo_id: int):
novelty_flag=novelty_flag, novelty_flag=novelty_flag,
llm_review=existing.get("llm_review") if existing else None, llm_review=existing.get("llm_review") if existing else None,
human_notes=human_notes, human_notes=human_notes,
qualitative_rating=existing.get("qualitative_rating") if existing else None,
) )
repo.update_combination_status(combo_id, "reviewed") repo.update_combination_status(combo_id, "reviewed")

View File

@@ -214,6 +214,9 @@ table.compact th, table.compact td { padding: 0.25rem 0.4rem; font-size: 0.83rem
.badge-llm_reviewed { background: rgba(107,163,160,0.12); color: var(--accent-teal); border-color: rgba(107,163,160,0.25); } .badge-llm_reviewed { background: rgba(107,163,160,0.12); color: var(--accent-teal); border-color: rgba(107,163,160,0.25); }
.badge-reviewed { background: rgba(155,142,196,0.12); color: var(--accent-violet); border-color: rgba(155,142,196,0.25); } .badge-reviewed { background: rgba(155,142,196,0.12); color: var(--accent-violet); border-color: rgba(155,142,196,0.25); }
.badge-pending { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); } .badge-pending { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); }
.badge-rating-low { background: rgba(184,92,92,0.12); color: var(--accent-red); border-color: rgba(184,92,92,0.25); }
.badge-rating-medium { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); }
.badge-rating-high { background: rgba(122,171,138,0.12); color: var(--accent-green); border-color: rgba(122,171,138,0.25); }
/* ── Buttons ─────────────────────────────────────────────── */ /* ── Buttons ─────────────────────────────────────────────── */
.btn { .btn {

View File

@@ -50,12 +50,13 @@
<div class="step-body"> <div class="step-body">
<h3>Physics Estimation</h3> <h3>Physics Estimation</h3>
<p> <p>
Surviving combinations get raw metric estimates &mdash; speed, cost, Surviving combinations get raw metric estimates &mdash; power
safety, range &mdash; via heuristic stubs or an LLM provider that density, cost, range &mdash; from a deterministic physics engine
reasons about the physical properties of each pairing. that sizes each combination from its own declared attributes, not
a guess.
</p> </p>
<div class="step-example"> <div class="step-example">
Bicycle + Human Pedalling &rarr; speed: 20 km/h, cost: $0.01/km Bicycle + Human Muscle &rarr; power density: 4.4 W/kg, range: 500km
</div> </div>
</div> </div>
</div> </div>
@@ -72,8 +73,8 @@
Combinations are ranked within their domain. Combinations are ranked within their domain.
</p> </p>
<div class="step-example"> <div class="step-example">
Domain <code>urban_commuting</code> weights: speed 25%, cost 25%, Domain <code>urban_commuting</code> weights: power density 42%,
safety 25%, availability 15%, range 10% cost 42%, range 17%
</div> </div>
</div> </div>
</div> </div>
@@ -85,9 +86,11 @@
<div class="step-body"> <div class="step-body">
<h3>LLM Review</h3> <h3>LLM Review</h3>
<p> <p>
Top-scoring combinations are sent to a language model for plausibility Top-scoring combinations are sent to a language model for a
and novelty assessment &mdash; catching physically valid but practically plausibility verdict plus a holistic LOW/MEDIUM/HIGH rating &mdash;
absurd pairings. weighing safety and accessibility as qualitative judgment calls
alongside the physics scores, catching physically valid but
practically absurd pairings.
</p> </p>
<div class="step-example"> <div class="step-example">
"Train + Solar Sail: structurally valid constraints, but solar radiation "Train + Solar Sail: structurally valid constraints, but solar radiation
@@ -163,14 +166,15 @@
<div class="card concept-card"> <div class="card concept-card">
<h3>Metrics</h3> <h3>Metrics</h3>
<p> <p>
Quantitative axes like speed, cost, safety, and range. Each metric Quantitative physics axes like power density, cost, and range. Each
has a domain-specific weight and normalization range. Some are metric has a domain-specific weight and normalization range. Some
inverted &mdash; lower cost is better. are inverted &mdash; lower cost is better. Safety and accessibility
are judgment calls, not physics quantities &mdash; they're weighed
qualitatively in the LLM review pass instead of scored here.
</p> </p>
<div class="concept-examples"> <div class="concept-examples">
<span class="badge">speed</span> <span class="badge">power_density</span>
<span class="badge">cost_efficiency</span> <span class="badge">cost_efficiency</span>
<span class="badge">safety</span>
<span class="badge">range_fuel</span> <span class="badge">range_fuel</span>
</div> </div>
</div> </div>

View File

@@ -27,6 +27,9 @@
{% if result %} {% if result %}
<dt>Composite Score</dt><dd class="score-cell">{{ "%.4f"|format(result.composite_score) }}</dd> <dt>Composite Score</dt><dd class="score-cell">{{ "%.4f"|format(result.composite_score) }}</dd>
<dt>Pass Reached</dt><dd>{{ result.pass_reached }}</dd> <dt>Pass Reached</dt><dd>{{ result.pass_reached }}</dd>
{% if result.qualitative_rating %}
<dt>Rating</dt><dd><span class="badge badge-rating-{{ result.qualitative_rating|lower }}">{{ result.qualitative_rating }}</span></dd>
{% endif %}
{% if result.novelty_flag %} {% if result.novelty_flag %}
<dt>Novelty</dt><dd>{{ result.novelty_flag }}</dd> <dt>Novelty</dt><dd>{{ result.novelty_flag }}</dd>
{% endif %} {% endif %}
@@ -102,11 +105,16 @@
{%- elif s.raw_value >= mb.norm_max -%} {%- elif s.raw_value >= mb.norm_max -%}
<span class="badge badge-{{ 'p1_fail' if mb.lower_is_better else 'valid' }}">at/above max{{ ' (worst)' if mb.lower_is_better else '' }}</span> <span class="badge badge-{{ 'p1_fail' if mb.lower_is_better else 'valid' }}">at/above max{{ ' (worst)' if mb.lower_is_better else '' }}</span>
{%- else -%} {%- else -%}
{% set pct = ((s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100) | int %} {% set raw_pct = (s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100 %}
{# For lower_is_better metrics, raw_pct alone measures distance from norm_min,
not quality -- a value near norm_min (excellent, cost near its floor) would
otherwise render as a near-empty bar. Invert so the bar and percentage always
mean "how good", matching the normalized score's own higher-is-better convention. #}
{% set pct = ((100 - raw_pct) if mb.lower_is_better else raw_pct) | int %}
<div class="metric-bar-container"> <div class="metric-bar-container">
<div class="metric-bar" style="width: {{ pct }}%"></div> <div class="metric-bar" style="width: {{ pct }}%"></div>
</div> </div>
<span class="metric-bar-label">~{{ pct }}%{{ ' ' if mb.lower_is_better else '' }}</span> <span class="metric-bar-label">~{{ pct }}%{{ ' (lower is better)' if mb.lower_is_better else '' }}</span>
{%- endif -%} {%- endif -%}
{%- else -%} {%- else -%}

View File

@@ -26,11 +26,11 @@
{% if statuses %} {% if statuses %}
<div class="filter-row"> <div class="filter-row">
<span>Filter:</span> <span>Status:</span>
<a href="{{ url_for('results.results_domain', domain_name=domain.name) }}" <a href="{{ url_for('results.results_domain', domain_name=domain.name, rating=rating_filter) }}"
class="btn btn-sm {{ '' if status_filter else 'btn-primary' }}">All ({{ total_results }})</a> class="btn btn-sm {{ '' if status_filter else 'btn-primary' }}">All ({{ total_results }})</a>
{% for s, cnt in statuses.items() %} {% for s, cnt in statuses.items() %}
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=s) }}" <a href="{{ url_for('results.results_domain', domain_name=domain.name, status=s, rating=rating_filter) }}"
class="btn btn-sm {{ 'btn-primary' if status_filter == s else '' }}"> class="btn btn-sm {{ 'btn-primary' if status_filter == s else '' }}">
{{ s }} ({{ cnt }}) {{ s }} ({{ cnt }})
</a> </a>
@@ -38,9 +38,25 @@
</div> </div>
{% endif %} {% endif %}
{% if ratings %}
<div class="filter-row">
<span>Rating:</span>
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=status_filter) }}"
class="btn btn-sm {{ '' if not rating_filter else 'btn-primary' }}">All</a>
{% for rt in ['HIGH', 'MEDIUM', 'LOW'] %}
{% if rt in ratings %}
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=status_filter, rating=rt) }}"
class="btn btn-sm {{ 'btn-primary' if rating_filter == rt else '' }}">
{{ rt }} ({{ ratings[rt] }})
</a>
{% endif %}
{% endfor %}
</div>
{% endif %}
{% if not results %} {% if not results %}
{% if status_filter %} {% if status_filter or rating_filter %}
<p class="empty">No results with status "{{ status_filter }}" in this domain.</p> <p class="empty">No results matching that filter in this domain.</p>
{% else %} {% else %}
<p class="empty">No results for this domain yet. <a href="{{ url_for('pipeline.pipeline_form') }}">Run the pipeline</a> first.</p> <p class="empty">No results for this domain yet. <a href="{{ url_for('pipeline.pipeline_form') }}">Run the pipeline</a> first.</p>
{% endif %} {% endif %}
@@ -52,6 +68,7 @@
<th>Score</th> <th>Score</th>
<th>Entities</th> <th>Entities</th>
<th>Status</th> <th>Status</th>
<th>Rating</th>
<th>Details</th> <th>Details</th>
<th></th> <th></th>
</tr> </tr>
@@ -69,6 +86,13 @@
<span class="badge badge-{{ r.combination.status }}">{{ r.combination.status }}</span> <span class="badge badge-{{ r.combination.status }}">{{ r.combination.status }}</span>
{%- endif -%} {%- endif -%}
</td> </td>
<td>
{%- if r.qualitative_rating -%}
<span class="badge badge-rating-{{ r.qualitative_rating|lower }}">{{ r.qualitative_rating }}</span>
{%- else -%}
{%- endif -%}
</td>
<td class="block-reason-cell"> <td class="block-reason-cell">
{%- if r.domain_block_reason -%} {%- if r.domain_block_reason -%}
{{ r.domain_block_reason }} {{ r.domain_block_reason }}

View File

@@ -335,28 +335,34 @@ def test_p3_fail_below_threshold(seeded_repo):
def test_p4_fail_implausible(seeded_repo): def test_p4_fail_implausible(seeded_repo):
"""Combos deemed implausible by LLM should get p4_fail status.""" """Combos deemed implausible by LLM should get p4_fail status.
Pass 2 is estimator-only now (never calls the LLM), so there's no way
to force every combo's raw estimates toward a controlled low/high value
the way MockLLMProvider's default_estimates used to. Force the pass-4
verdict directly instead -- what's under test here is pipeline.py's
wiring of review_plausibility's return value to status/counters, not
MockLLMProvider's avg-based heuristic.
"""
from physcom.llm.providers.mock import MockLLMProvider from physcom.llm.providers.mock import MockLLMProvider
class AlwaysImplausibleLLM(MockLLMProvider):
def review_plausibility(self, description, raw_metrics, normalized_scores, domain):
return ("Always implausible for testing.", False)
repo = seeded_repo repo = seeded_repo
domain = repo.get_domain("urban_commuting") domain = repo.get_domain("urban_commuting")
resolver = ConstraintResolver() resolver = ConstraintResolver()
scorer = Scorer(domain) scorer = Scorer(domain)
# Low estimates → normalized scores avg <= 0.5 → MockLLMProvider returns (text, False)
# Use threshold=0.0 so no combo gets p3_fail and all reach pass 4 # Use threshold=0.0 so no combo gets p3_fail and all reach pass 4
mock_llm = MockLLMProvider(default_estimates={ pipeline = Pipeline(repo, resolver, scorer, llm=AlwaysImplausibleLLM())
"power_density": 0.1, "cost_efficiency": 0.1, "safety": 0.1,
"availability": 0.1, "range_fuel": 0.1,
})
pipeline = Pipeline(repo, resolver, scorer, llm=mock_llm)
result = pipeline.run( result = pipeline.run(
domain, ["platform", "actuator", "energy_storage"], domain, ["platform", "actuator", "energy_storage"],
score_threshold=0.0, passes=[1, 2, 3, 4], score_threshold=0.0, passes=[1, 2, 3, 4],
) )
# With low normalized scores (avg <= 0.5), reviewed combos should be p4_fail
assert result.pass4_failed > 0 assert result.pass4_failed > 0
assert result.pass4_reviewed == 0 assert result.pass4_reviewed == 0
@@ -367,20 +373,23 @@ def test_p4_fail_implausible(seeded_repo):
def test_p4_pass_plausible(seeded_repo): def test_p4_pass_plausible(seeded_repo):
"""Combos deemed plausible by LLM should get llm_reviewed status.""" """Combos deemed plausible by LLM should get llm_reviewed status.
See test_p4_fail_implausible on why the verdict is forced directly
rather than via controlled pass-2 estimates.
"""
from physcom.llm.providers.mock import MockLLMProvider from physcom.llm.providers.mock import MockLLMProvider
class AlwaysPlausibleLLM(MockLLMProvider):
def review_plausibility(self, description, raw_metrics, normalized_scores, domain):
return ("Always plausible for testing.", True)
repo = seeded_repo repo = seeded_repo
domain = repo.get_domain("urban_commuting") domain = repo.get_domain("urban_commuting")
resolver = ConstraintResolver() resolver = ConstraintResolver()
scorer = Scorer(domain) scorer = Scorer(domain)
# High estimates → avg > 0.5 → MockLLMProvider returns (text, True) pipeline = Pipeline(repo, resolver, scorer, llm=AlwaysPlausibleLLM())
mock_llm = MockLLMProvider(default_estimates={
"power_density": 500.0, "cost_efficiency": 5e-4, "safety": 0.6,
"availability": 0.7, "range_fuel": 200000.0,
})
pipeline = Pipeline(repo, resolver, scorer, llm=mock_llm)
result = pipeline.run( result = pipeline.run(
domain, ["platform", "actuator", "energy_storage"], domain, ["platform", "actuator", "energy_storage"],