From 76f460499a0f8e45b6b1759876b9bf017cdffc21 Mon Sep 17 00:00:00 2001 From: Andrew Simonson Date: Sat, 15 Aug 2026 16:52:26 -0500 Subject: [PATCH] drop safety/availability from scoring, holistic p4 rating, phase-parallel pipeline safety and availability don't reduce to physics formulas the way power_density/range_fuel/cost_efficiency do -- they're judgment calls (risk assessment, infrastructure prevalence), and running them through the same log-normalize() built for physical quantities produced incoherent results: safety's raw value is already a "0-1" score, and normalizing it again turned 0.6 into an unexplainable 0.678 that even the LLM reviewing it could only cite, never justify (see combo 1540). Removed both from domain_metric_weights (safety from 4 domains, availability from urban_commuting) and renormalized the remaining weights to sum to 1.0. Pass 4 now produces one holistic RATING (LOW/MEDIUM/HIGH) alongside the existing VERDICT, with safety and accessibility folded in as qualitative considerations feeding that single judgment rather than scored separately -- not a checklist of independent numbers. New qualitative_rating column, filterable in the results UI. Also added domain name/description to the review prompt so the LLM judges a metric like range against what the domain actually needs (urban_commuting: 1-50km) instead of generic real-world expectations for the platform category -- confirmed live on a combo where phi4 had called a 396km range "limited" by comparing to typical aircraft rather than a domain that needs 1-50km. Pass 2 is estimator-only now -- self.llm is never consulted there, reserved entirely for pass 4. Restructured Pipeline.run() from combo-first to phase-parallel: each pass now runs to completion across every combo before the next pass starts, rather than walking each combo through all four passes before the next combo. This surfaced a real bug: domain- blocked combos (status stays "valid" by design, not "_fail") were slipping past a naive status-based skip guard and getting silently re-processed by pass 2. Fixed with a shared dead-combo check that catches both generic failures and domain blocks correctly. Also fixes a results-page display bug found while reviewing a live combo: the per-metric "position" bar showed raw distance from norm_min without inverting for lower_is_better metrics, so an excellent cost score (near the good end) rendered as a ~0%, near-empty bar -- looked bad next to its own 0.99 normalized score. Validated live against phi4 (real Ollama calls, not mocked): full-domain phase-parallel run (2,970 combos, estimator-only p2, 1.6s) followed by a real pass-4 run (111 reviewed, 11m, 0 crashes, 0 null ratings). Two tests that relied on the old LLM-driven pass 2 to force deterministic outcomes were updated to test pass 4's verdict-wiring directly instead. All 100 tests pass. Co-Authored-By: Claude Sonnet 5 --- src/physcom/db/repository.py | 86 ++- src/physcom/db/schema.py | 5 + src/physcom/engine/pipeline.py | 607 +++++++++--------- src/physcom/llm/base.py | 13 +- src/physcom/llm/parsing.py | 7 + src/physcom/llm/prompts.py | 42 +- src/physcom/llm/providers/gemini.py | 8 +- src/physcom/llm/providers/mock.py | 4 +- src/physcom/llm/providers/ollama.py | 8 +- src/physcom/seed/transport_example.py | 65 +- src/physcom_web/routes/results.py | 7 +- src/physcom_web/static/style.css | 3 + src/physcom_web/templates/home.html | 32 +- src/physcom_web/templates/results/detail.html | 12 +- src/physcom_web/templates/results/list.html | 34 +- tests/test_pipeline_async.py | 39 +- 16 files changed, 569 insertions(+), 403 deletions(-) diff --git a/src/physcom/db/repository.py b/src/physcom/db/repository.py index 35aeb40..f38d33b 100644 --- a/src/physcom/db/repository.py +++ b/src/physcom/db/repository.py @@ -230,25 +230,38 @@ class Repository: self.conn.commit() return row["id"] - def backfill_lower_is_better(self, domain_name: str, metric_name: str) -> None: - """Set lower_is_better=1 for an existing domain-metric row that still has the default 0.""" - self.conn.execute( - """UPDATE domain_metric_weights SET lower_is_better = 1 - WHERE lower_is_better = 0 - AND domain_id = (SELECT id FROM domains WHERE name = ?) - AND metric_id = (SELECT id FROM metrics WHERE name = ?)""", - (domain_name, metric_name), - ) - self.conn.commit() - - def backfill_metric_unit(self, domain_name: str, metric_name: str, unit: str) -> None: - """Set this domain-metric row's unit — unit is domain-scoped, not global to the metric name.""" - self.conn.execute( - """UPDATE domain_metric_weights SET unit = ? - WHERE domain_id = (SELECT id FROM domains WHERE name = ?) - AND metric_id = (SELECT id FROM metrics WHERE name = ?)""", - (unit, domain_name, metric_name), - ) + def sync_domain_metric_weights(self, domain: Domain) -> None: + """Make domain_metric_weights exactly match domain.metric_bounds on an + already-seeded domain: upserts weight/norm_min/norm_max/unit for every + currently-declared metric, and deletes any row for a metric that's been + removed from the domain (e.g. safety/availability dropped from the + scored set). Safe to call whether the domain was just freshly inserted + or already existed. + """ + row = self.conn.execute( + "SELECT id FROM domains WHERE name = ?", (domain.name,) + ).fetchone() + if not row: + return + domain_id = row["id"] + keep_ids = [] + for mb in domain.metric_bounds: + metric_id = self.ensure_metric(mb.metric_name, unit=mb.unit) + keep_ids.append(metric_id) + self.conn.execute( + """INSERT OR REPLACE INTO domain_metric_weights + (domain_id, metric_id, weight, norm_min, norm_max, lower_is_better, unit) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + (domain_id, metric_id, mb.weight, mb.norm_min, mb.norm_max, + int(mb.lower_is_better), mb.unit), + ) + if keep_ids: + placeholders = ",".join("?" * len(keep_ids)) + self.conn.execute( + f"""DELETE FROM domain_metric_weights + WHERE domain_id = ? AND metric_id NOT IN ({placeholders})""", + (domain_id, *keep_ids), + ) self.conn.commit() def add_domain(self, domain: Domain) -> Domain: @@ -593,15 +606,18 @@ class Repository: llm_review: str | None = None, human_notes: str | None = None, domain_block_reason: str | None = None, + qualitative_rating: str | None = None, commit: bool = True, ) -> None: self.conn.execute( """INSERT OR REPLACE INTO combination_results (combination_id, domain_id, composite_score, novelty_flag, - llm_review, human_notes, pass_reached, domain_block_reason) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + llm_review, human_notes, pass_reached, domain_block_reason, + qualitative_rating) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", (combo_id, domain_id, composite_score, novelty_flag, - llm_review, human_notes, pass_reached, domain_block_reason), + llm_review, human_notes, pass_reached, domain_block_reason, + qualitative_rating), ) if commit: self.conn.commit() @@ -641,6 +657,20 @@ class Repository: ).fetchall() return {r["status"]: r["cnt"] for r in rows} + def count_results_by_rating(self, domain_name: str) -> dict[str, int]: + """Count results by qualitative_rating (LOW/MEDIUM/HIGH) for a domain. + Rows with no rating (not yet pass-4 reviewed, or reviewed before this + existed) are excluded, not bucketed as a pseudo-status.""" + rows = self.conn.execute( + """SELECT cr.qualitative_rating as rating, COUNT(*) as cnt + FROM combination_results cr + JOIN domains d ON cr.domain_id = d.id + WHERE d.name = ? AND cr.qualitative_rating IS NOT NULL + GROUP BY cr.qualitative_rating""", + (domain_name,), + ).fetchall() + return {r["rating"]: r["cnt"] for r in rows} + def get_pipeline_summary(self, domain_name: str) -> dict | None: """Return a summary of results for a domain, or None if no results.""" row = self.conn.execute( @@ -685,8 +715,12 @@ class Repository: ).fetchone() return dict(row) if row else None - def get_all_results(self, domain_name: str, status: str | None = None) -> list[dict]: - """Return all results for a domain, optionally filtered by combo status.""" + def get_all_results( + self, domain_name: str, status: str | None = None, rating: str | None = None + ) -> list[dict]: + """Return all results for a domain, optionally filtered by combo + status and/or qualitative_rating (LOW/MEDIUM/HIGH, independent filters + that combine with AND).""" query = """SELECT cr.*, c.hash, c.status as combo_status, d.name as domain_name FROM combination_results cr JOIN combinations c ON cr.combination_id = c.id @@ -698,6 +732,9 @@ class Repository: elif status: query += " AND c.status = ? AND cr.domain_block_reason IS NULL" params.append(status) + if rating: + query += " AND cr.qualitative_rating = ?" + params.append(rating) query += " ORDER BY cr.composite_score DESC" rows = self.conn.execute(query, params).fetchall() combo_ids = [r["combination_id"] for r in rows] @@ -712,6 +749,7 @@ class Repository: "pass_reached": r["pass_reached"], "domain_id": r["domain_id"], "domain_block_reason": r["domain_block_reason"], + "qualitative_rating": r["qualitative_rating"], } for r in rows ] diff --git a/src/physcom/db/schema.py b/src/physcom/db/schema.py index 0108a19..2e121d2 100644 --- a/src/physcom/db/schema.py +++ b/src/physcom/db/schema.py @@ -91,6 +91,7 @@ CREATE TABLE IF NOT EXISTS combination_results ( human_notes TEXT, pass_reached INTEGER, domain_block_reason TEXT, + qualitative_rating TEXT, UNIQUE(combination_id, domain_id) ); @@ -165,6 +166,10 @@ def _migrate(conn: sqlite3.Connection) -> None: conn.execute( "ALTER TABLE combination_results ADD COLUMN domain_block_reason TEXT" ) + if "qualitative_rating" not in result_cols: + conn.execute( + "ALTER TABLE combination_results ADD COLUMN qualitative_rating TEXT" + ) # Backfill: cost_efficiency is lower-is-better in all domains conn.execute( diff --git a/src/physcom/engine/pipeline.py b/src/physcom/engine/pipeline.py index d551b81..d6aefe4 100644 --- a/src/physcom/engine/pipeline.py +++ b/src/physcom/engine/pipeline.py @@ -12,6 +12,7 @@ from physcom.engine.combinator import generate_combinations from physcom.engine.constraint_resolver import ConstraintResolver, ConstraintResult from physcom.engine.scorer import Scorer from physcom.llm.base import LLMProvider, LLMRateLimitError +from physcom.llm.parsing import parse_rating from physcom.models.combination import Combination, ScoredResult from physcom.models.domain import Domain, MetricBound @@ -385,308 +386,57 @@ class Pipeline: # Prepare metric lookup bounds_by_name = {mb.metric_name: mb for mb in domain.metric_bounds} - # ── Combo-first loop ───────────────────────────────────── - # Deterministic passes (1, 3, and 2 without an LLM) defer commits and - # get flushed periodically + in `finally` below -- a crash there costs - # a cheap recompute, not lost work worth committing per write. Pass 4 - # (and pass 2 with an LLM) commit immediately after each call: those - # are slow and crash-prone (see the QwQ timeout saga), so that result - # is worth protecting the moment it lands. + # ── Phase-parallel: each pass runs to completion across every combo + # before the next pass starts, instead of walking each combo through + # every pass before moving to the next combo. This maximizes + # progress before the expensive/slow phase (pass 4's LLM calls) and + # keeps that phase's cost visible on its own, separate from the + # deterministic passes. It also removes any need for the two options + # to reconcile: pass 2 is estimator-only now (no LLM call in it at + # all -- self.llm is reserved for pass 4), so there's no combo that + # touches an LLM in both pass 2 and pass 4, and nothing here needs a + # live/resumed conversation across passes. + # + # Deterministic passes (1, 2, 3) defer commits and get flushed + # periodically + in `finally` below -- a crash there costs a cheap + # recompute, not lost work worth committing per write. Pass 4 + # commits immediately after each call: those are slow and + # crash-prone (see the QwQ timeout saga), so that result is worth + # protecting the moment it lands. combos_since_commit = 0 + + def _tick_commit() -> None: + nonlocal combos_since_commit + combos_since_commit += 1 + if combos_since_commit >= 200: + self.repo.commit() + combos_since_commit = 0 + try: - for combo in combos: - self._check_cancelled(run_id) - combos_since_commit += 1 - if combos_since_commit >= 200: - self.repo.commit() - combos_since_commit = 0 + if 1 in passes: + for combo in combos: + self._check_cancelled(run_id) + _tick_commit() + self._process_pass1(combo, domain, result, run_id) - # Check existing progress for this combo in this domain - existing_pass = self.repo.get_combo_pass_reached( - combo.id, domain.id - ) or 0 + if 2 in passes: + for combo in combos: + self._check_cancelled(run_id) + _tick_commit() + self._process_pass2(combo, domain, bounds_by_name, result, run_id) - # Load existing result to preserve human review data - existing_result = self.repo.get_existing_result( - combo.id, domain.id - ) - - # ── Pass 1: Constraint Resolution ──────────────── - if 1 in passes and existing_pass < 1: - cr: ConstraintResult = self.resolver.resolve(combo) - if cr.status == "p1_fail": - combo.status = "p1_fail" - combo.block_reason = "; ".join(cr.violations) - self.repo.update_combination_status( - combo.id, "p1_fail", combo.block_reason, commit=False - ) - # Save a result row so failed combos appear in results - self.repo.save_result( - combo.id, - domain.id, - composite_score=0.0, - pass_reached=1, - commit=False, - ) - result.pass1_failed += 1 - self._update_run_counters(run_id, result, current_pass=1) - continue # p1_fail — skip remaining passes - else: - combo.status = "valid" - self.repo.update_combination_status(combo.id, "valid", commit=False) - - # Domain constraint check (per-domain block only). combo.status - # stays "valid" here on purpose: it's domain-agnostic and the - # same combo can be blocked in this domain but valid in another. - # The per-domain block lives on combination_results.domain_block_reason - # (see count_combinations_by_status / get_all_results, which bucket on it). - if domain.constraints: - dc_result = self.resolver.check_domain_constraints( - combo, domain.constraints - ) - if dc_result.status == "p1_fail": - self.repo.save_result( - combo.id, domain.id, - composite_score=0.0, pass_reached=1, - domain_block_reason="; ".join( - dc_result.violations - ), - commit=False, - ) - result.pass1_failed += 1 - self._update_run_counters( - run_id, result, current_pass=1 - ) - continue - - if cr.status == "conditional": - result.pass1_conditional += 1 - else: - result.pass1_valid += 1 - - self._update_run_counters(run_id, result, current_pass=1) - elif 1 in passes: - # Already pass1'd — check if it failed - if combo.status.endswith("_fail"): - result.pass1_failed += 1 - continue - # Check if domain-blocked from a prior run - if existing_result and existing_result["pass_reached"] == 1: - result.pass1_failed += 1 - continue - result.pass1_valid += 1 - else: - # Pass 1 not requested; check if failed from a prior run - if combo.status.endswith("_fail"): - result.pass1_failed += 1 - continue - - # ── Pass 2: Physics Estimation ─────────────────── - raw_metrics: dict[str, float] = {} - if 2 in passes and existing_pass < 2: - description = _describe_combination(combo) - if self.llm: - raw_metrics = self.llm.estimate_physics( - description, domain.metric_bounds - ) - else: - raw_metrics = self._stub_estimate(combo, domain.metric_bounds) - - # Save raw estimates immediately (crash-safe) - estimate_dicts = [] - for mname, rval in raw_metrics.items(): - mb = bounds_by_name.get(mname) - if mb and mb.metric_id: - estimate_dicts.append({ - "metric_id": mb.metric_id, - "raw_value": rval, - "estimation_method": "llm" if self.llm else "stub", - "confidence": 1.0, - }) - # LLM-produced estimates commit immediately (slow/crash- - # prone, worth protecting); stub estimates are instant - # and defer, same as the rest of the deterministic passes. - used_llm = self.llm is not None - if estimate_dicts: - self.repo.save_raw_estimates( - combo.id, domain.id, estimate_dicts, commit=used_llm - ) - - # Check for all-zero estimates → p2_fail - if raw_metrics and all(v == 0.0 for v in raw_metrics.values()): - combo.status = "p2_fail" - combo.block_reason = "All metric estimates are zero" - self.repo.update_combination_status( - combo.id, "p2_fail", combo.block_reason, commit=used_llm - ) - self.repo.save_result( - combo.id, domain.id, - composite_score=0.0, pass_reached=2, - commit=used_llm, - ) - result.pass2_failed += 1 - self._update_run_counters(run_id, result, current_pass=2) - continue - - result.pass2_estimated += 1 - self._update_run_counters(run_id, result, current_pass=2) - elif 2 in passes: - # Already estimated — reload raw values from DB - existing_scores = self.repo.get_combination_scores( - combo.id, domain.id - ) - raw_metrics = { - s["metric_name"]: s["raw_value"] for s in existing_scores - } - result.pass2_estimated += 1 - else: - # Pass 2 not requested, use empty metrics - raw_metrics = {} - - # ── Pass 3: Scoring & Ranking ──────────────────── - if 3 in passes and existing_pass < 3: - sr = self.scorer.score_combination(combo, raw_metrics) - - # Persist per-metric scores with normalized values - score_dicts = [] - for s in sr.scores: - mb = bounds_by_name.get(s.metric_name) - if mb and mb.metric_id: - score_dicts.append({ - "metric_id": mb.metric_id, - "raw_value": s.raw_value, - "normalized_score": s.normalized_score, - "estimation_method": s.estimation_method, - "confidence": s.confidence, - }) - if score_dicts: - self.repo.save_scores(combo.id, domain.id, score_dicts, commit=False) - - # Preserve existing human data - novelty_flag = ( - existing_result["novelty_flag"] if existing_result else None - ) - human_notes = ( - existing_result["human_notes"] if existing_result else None + if 3 in passes: + for combo in combos: + self._check_cancelled(run_id) + _tick_commit() + self._process_pass3( + combo, domain, bounds_by_name, result, score_threshold, run_id ) - if sr.composite_score < score_threshold: - self.repo.save_result( - combo.id, domain.id, - sr.composite_score, pass_reached=3, - novelty_flag=novelty_flag, - human_notes=human_notes, - commit=False, - ) - combo.status = "p3_fail" - combo.block_reason = ( - f"Composite score {sr.composite_score:.4f} " - f"below threshold {score_threshold}" - ) - self.repo.update_combination_status( - combo.id, "p3_fail", combo.block_reason, commit=False - ) - result.pass3_failed += 1 - result.pass3_scored += 1 - self._update_run_counters(run_id, result, current_pass=3) - continue - - self.repo.save_result( - combo.id, - domain.id, - sr.composite_score, - pass_reached=3, - novelty_flag=novelty_flag, - human_notes=human_notes, - commit=False, - ) - self.repo.update_combination_status(combo.id, "scored", commit=False) - - result.pass3_scored += 1 - result.pass3_above_threshold += 1 - - self._update_run_counters(run_id, result, current_pass=3) - elif 3 in passes and existing_pass >= 3: - # Already scored — count it - result.pass3_scored += 1 - if existing_result and existing_result["composite_score"] is not None: - if existing_result["composite_score"] >= score_threshold: - result.pass3_above_threshold += 1 - - # ── Pass 4: LLM Review ─────────────────────────── - if 4 in passes and self.llm: - cur_pass = self.repo.get_combo_pass_reached( - combo.id, domain.id - ) or 0 - if cur_pass < 4: - cur_result = self.repo.get_existing_result( - combo.id, domain.id - ) - if ( - cur_result - and cur_result["composite_score"] is not None - and cur_result["composite_score"] >= score_threshold - ): - description = _describe_combination(combo) - db_scores = self.repo.get_combination_scores( - combo.id, domain.id - ) - score_dict = { - s["metric_name"]: s["normalized_score"] - for s in db_scores - if s["normalized_score"] is not None - } - raw_dict = { - s["metric_name"]: s["raw_value"] - for s in db_scores - if s["raw_value"] is not None - } - review_result: tuple[str, bool] | None = None - try: - review_result = self.llm.review_plausibility( - description, raw_dict, score_dict, domain.metric_bounds - ) - except LLMRateLimitError as exc: - self._wait_for_rate_limit(run_id, exc.retry_after) - try: - review_result = self.llm.review_plausibility( - description, raw_dict, score_dict, domain.metric_bounds - ) - except LLMRateLimitError: - pass # still limited; skip, retry next run - if review_result is not None: - review_text, plausible = review_result - if not plausible: - self.repo.save_result( - combo.id, domain.id, - cur_result["composite_score"], - pass_reached=4, - novelty_flag=cur_result.get("novelty_flag"), - llm_review=review_text, - human_notes=cur_result.get("human_notes"), - ) - combo.status = "p4_fail" - combo.block_reason = "LLM deemed implausible" - self.repo.update_combination_status( - combo.id, "p4_fail", combo.block_reason - ) - result.pass4_failed += 1 - else: - self.repo.save_result( - combo.id, domain.id, - cur_result["composite_score"], - pass_reached=4, - novelty_flag=cur_result.get("novelty_flag"), - llm_review=review_text, - human_notes=cur_result.get("human_notes"), - ) - self.repo.update_combination_status( - combo.id, "llm_reviewed" - ) - result.pass4_reviewed += 1 - self._update_run_counters( - run_id, result, current_pass=4 - ) + if 4 in passes and self.llm: + for combo in combos: + self._check_cancelled(run_id) + self._process_pass4(combo, domain, result, score_threshold, run_id) except CancelledError: if run_id is not None: @@ -716,6 +466,271 @@ class Pipeline: result.top_results = self.repo.get_top_results(domain.name, limit=20) return result + @staticmethod + def _already_dead(combo: Combination, existing_result: dict | None) -> bool: + """True if this combo is dead for every pass after 1 -- either a + generic failure (status ends in _fail) or a domain-specific block. + The domain-block case needs the extra existing_result check: + combo.status stays "valid" on purpose for it (domain-agnostic, + see _process_pass1's own comment on this), so pass_reached==1 with + the block already recorded is what actually marks it dead -- + status alone isn't enough to catch it.""" + if combo.status.endswith("_fail"): + return True + return bool(existing_result and existing_result["pass_reached"] == 1) + + def _process_pass1( + self, combo: Combination, domain: Domain, result: PipelineResult, run_id: int | None + ) -> None: + """Constraint resolution for one combo. All writes deferred (commit=False).""" + existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0 + if existing_pass >= 1: + if combo.status.endswith("_fail"): + result.pass1_failed += 1 + return + existing_result = self.repo.get_existing_result(combo.id, domain.id) + if existing_result and existing_result["pass_reached"] == 1: + result.pass1_failed += 1 + return + result.pass1_valid += 1 + return + + cr: ConstraintResult = self.resolver.resolve(combo) + if cr.status == "p1_fail": + combo.status = "p1_fail" + combo.block_reason = "; ".join(cr.violations) + self.repo.update_combination_status( + combo.id, "p1_fail", combo.block_reason, commit=False + ) + # Save a result row so failed combos appear in results + self.repo.save_result( + combo.id, domain.id, composite_score=0.0, pass_reached=1, commit=False + ) + result.pass1_failed += 1 + self._update_run_counters(run_id, result, current_pass=1) + return + + combo.status = "valid" + self.repo.update_combination_status(combo.id, "valid", commit=False) + + # Domain constraint check (per-domain block only). combo.status stays + # "valid" here on purpose: it's domain-agnostic and the same combo can + # be blocked in this domain but valid in another. The per-domain + # block lives on combination_results.domain_block_reason (see + # count_combinations_by_status / get_all_results, which bucket on it). + if domain.constraints: + dc_result = self.resolver.check_domain_constraints(combo, domain.constraints) + if dc_result.status == "p1_fail": + self.repo.save_result( + combo.id, domain.id, + composite_score=0.0, pass_reached=1, + domain_block_reason="; ".join(dc_result.violations), + commit=False, + ) + result.pass1_failed += 1 + self._update_run_counters(run_id, result, current_pass=1) + return + + if cr.status == "conditional": + result.pass1_conditional += 1 + else: + result.pass1_valid += 1 + self._update_run_counters(run_id, result, current_pass=1) + + def _process_pass2( + self, + combo: Combination, + domain: Domain, + bounds_by_name: dict[str, MetricBound], + result: PipelineResult, + run_id: int | None, + ) -> None: + """Physics estimation for one combo. Estimator-only -- self.llm is + reserved for pass 4, never consulted here. All writes deferred.""" + existing_result = self.repo.get_existing_result(combo.id, domain.id) + if self._already_dead(combo, existing_result): + return + existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0 + if existing_pass >= 2: + result.pass2_estimated += 1 + return + + raw_metrics = self._stub_estimate(combo, domain.metric_bounds) + + estimate_dicts = [] + for mname, rval in raw_metrics.items(): + mb = bounds_by_name.get(mname) + if mb and mb.metric_id: + estimate_dicts.append({ + "metric_id": mb.metric_id, + "raw_value": rval, + "estimation_method": "stub", + "confidence": 1.0, + }) + if estimate_dicts: + self.repo.save_raw_estimates(combo.id, domain.id, estimate_dicts, commit=False) + + # Check for all-zero estimates → p2_fail + if raw_metrics and all(v == 0.0 for v in raw_metrics.values()): + combo.status = "p2_fail" + combo.block_reason = "All metric estimates are zero" + self.repo.update_combination_status( + combo.id, "p2_fail", combo.block_reason, commit=False + ) + self.repo.save_result( + combo.id, domain.id, composite_score=0.0, pass_reached=2, commit=False + ) + result.pass2_failed += 1 + self._update_run_counters(run_id, result, current_pass=2) + return + + result.pass2_estimated += 1 + self._update_run_counters(run_id, result, current_pass=2) + + def _process_pass3( + self, + combo: Combination, + domain: Domain, + bounds_by_name: dict[str, MetricBound], + result: PipelineResult, + score_threshold: float, + run_id: int | None, + ) -> None: + """Scoring for one combo. Reloads raw estimates from the DB (pass 2 + ran as its own separate phase, not in-memory from this iteration). + All writes deferred.""" + existing_result = self.repo.get_existing_result(combo.id, domain.id) + if self._already_dead(combo, existing_result): + return + existing_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0 + if existing_pass >= 3: + result.pass3_scored += 1 + if existing_result and existing_result["composite_score"] is not None: + if existing_result["composite_score"] >= score_threshold: + result.pass3_above_threshold += 1 + return + + existing_scores = self.repo.get_combination_scores(combo.id, domain.id) + raw_metrics = {s["metric_name"]: s["raw_value"] for s in existing_scores} + sr = self.scorer.score_combination(combo, raw_metrics) + + score_dicts = [] + for s in sr.scores: + mb = bounds_by_name.get(s.metric_name) + if mb and mb.metric_id: + score_dicts.append({ + "metric_id": mb.metric_id, + "raw_value": s.raw_value, + "normalized_score": s.normalized_score, + "estimation_method": s.estimation_method, + "confidence": s.confidence, + }) + if score_dicts: + self.repo.save_scores(combo.id, domain.id, score_dicts, commit=False) + + # Preserve existing human data + novelty_flag = existing_result["novelty_flag"] if existing_result else None + human_notes = existing_result["human_notes"] if existing_result else None + + if sr.composite_score < score_threshold: + self.repo.save_result( + combo.id, domain.id, sr.composite_score, pass_reached=3, + novelty_flag=novelty_flag, human_notes=human_notes, commit=False, + ) + combo.status = "p3_fail" + combo.block_reason = ( + f"Composite score {sr.composite_score:.4f} below threshold {score_threshold}" + ) + self.repo.update_combination_status( + combo.id, "p3_fail", combo.block_reason, commit=False + ) + result.pass3_failed += 1 + result.pass3_scored += 1 + self._update_run_counters(run_id, result, current_pass=3) + return + + self.repo.save_result( + combo.id, domain.id, sr.composite_score, pass_reached=3, + novelty_flag=novelty_flag, human_notes=human_notes, commit=False, + ) + self.repo.update_combination_status(combo.id, "scored", commit=False) + result.pass3_scored += 1 + result.pass3_above_threshold += 1 + self._update_run_counters(run_id, result, current_pass=3) + + def _process_pass4( + self, + combo: Combination, + domain: Domain, + result: PipelineResult, + score_threshold: float, + run_id: int | None, + ) -> None: + """LLM plausibility review for one combo. Writes commit immediately + (default commit=True) -- slow, crash-prone calls worth protecting + the moment a result lands.""" + cur_result = self.repo.get_existing_result(combo.id, domain.id) + if self._already_dead(combo, cur_result): + return + cur_pass = self.repo.get_combo_pass_reached(combo.id, domain.id) or 0 + if cur_pass >= 4: + return + if not ( + cur_result + and cur_result["composite_score"] is not None + and cur_result["composite_score"] >= score_threshold + ): + return + + description = _describe_combination(combo) + db_scores = self.repo.get_combination_scores(combo.id, domain.id) + score_dict = { + s["metric_name"]: s["normalized_score"] + for s in db_scores if s["normalized_score"] is not None + } + raw_dict = { + s["metric_name"]: s["raw_value"] + for s in db_scores if s["raw_value"] is not None + } + + review_result: tuple[str, bool] | None = None + try: + review_result = self.llm.review_plausibility( + description, raw_dict, score_dict, domain + ) + except LLMRateLimitError as exc: + self._wait_for_rate_limit(run_id, exc.retry_after) + try: + review_result = self.llm.review_plausibility( + description, raw_dict, score_dict, domain.metric_bounds + ) + except LLMRateLimitError: + return # still limited; skip, retry next run + + if review_result is None: + return + review_text, plausible = review_result + rating = parse_rating(review_text) + if not plausible: + self.repo.save_result( + combo.id, domain.id, cur_result["composite_score"], pass_reached=4, + novelty_flag=cur_result.get("novelty_flag"), llm_review=review_text, + human_notes=cur_result.get("human_notes"), qualitative_rating=rating, + ) + combo.status = "p4_fail" + combo.block_reason = "LLM deemed implausible" + self.repo.update_combination_status(combo.id, "p4_fail", combo.block_reason) + result.pass4_failed += 1 + else: + self.repo.save_result( + combo.id, domain.id, cur_result["composite_score"], pass_reached=4, + novelty_flag=cur_result.get("novelty_flag"), llm_review=review_text, + human_notes=cur_result.get("human_notes"), qualitative_rating=rating, + ) + self.repo.update_combination_status(combo.id, "llm_reviewed") + result.pass4_reviewed += 1 + self._update_run_counters(run_id, result, current_pass=4) + def _wait_for_rate_limit(self, run_id: int | None, retry_after: int) -> None: """Mark run rate_limited, sleep with cancel checks, then resume.""" if run_id is not None: diff --git a/src/physcom/llm/base.py b/src/physcom/llm/base.py index 91fe9cb..1f0e875 100644 --- a/src/physcom/llm/base.py +++ b/src/physcom/llm/base.py @@ -4,7 +4,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from physcom.models.domain import MetricBound +from physcom.models.domain import Domain, MetricBound class LLMRateLimitError(Exception): @@ -40,7 +40,7 @@ class LLMProvider(ABC): combination_description: str, raw_metrics: dict[str, float], normalized_scores: dict[str, float], - metrics: list[MetricBound], + domain: Domain, ) -> tuple[str, bool]: """Given a combination, its raw physical estimates, and their normalized scores, return a (text, is_plausible) tuple: @@ -50,6 +50,11 @@ class LLMProvider(ABC): normalized score) so the review can reason from the actual physics rather than only a compressed 0-1 number, which can look deceptively bad for a metric whose scale was built for a different - kind of vehicle. `metrics` carries each metric's unit for - formatting the raw value meaningfully.""" + kind of vehicle. `domain` carries both each metric's unit (via + domain.metric_bounds, for formatting the raw value meaningfully) + and the domain's own name/description, so the review judges a + metric like range against what THIS domain actually needs rather + than generic real-world expectations for the platform category + (e.g. a short-hop domain shouldn't get judged against typical + long-haul aircraft range).""" ... diff --git a/src/physcom/llm/parsing.py b/src/physcom/llm/parsing.py index d5ea15b..23daf86 100644 --- a/src/physcom/llm/parsing.py +++ b/src/physcom/llm/parsing.py @@ -16,6 +16,13 @@ def parse_verdict(text: str) -> bool: return True +def parse_rating(text: str) -> str | None: + """Extract RATING: LOW/MEDIUM/HIGH from response; None if absent (older + reviews saved before this existed, or a malformed response).""" + m = re.search(r"RATING:\s*(LOW|MEDIUM|HIGH)", text, re.IGNORECASE) + return m.group(1).upper() if m else None + + def parse_metric_json(text: str, metrics: list[MetricBound]) -> dict[str, float]: """Strip markdown fences and parse JSON; fall back to each metric's own norm_min/norm_max midpoint on error — a flat constant like 0.5 is diff --git a/src/physcom/llm/prompts.py b/src/physcom/llm/prompts.py index 7173a7e..4a4b07e 100644 --- a/src/physcom/llm/prompts.py +++ b/src/physcom/llm/prompts.py @@ -116,6 +116,14 @@ You are reviewing a transportation concept for real-world viability — could th actually be built and operated safely. Whether it is new, exciting, or original is NOT the question. +## Domain +This concept is being evaluated for "{domain_name}": {domain_description} +Judge every metric against what THIS domain actually needs, not general +expectations for the platform category. A range far beyond what this domain +requires is a strength or a non-issue, never a weakness -- don't reason about +range, speed, or capacity by comparing to what other vehicles of this type +typically have in general use; compare to what this specific domain calls for. + ## Concept {description} @@ -139,6 +147,14 @@ before treating the score as evidence of a problem. {scores} +Safety and accessibility (infrastructure/regulatory availability) are NOT +among the scores above — neither reduces to a physics formula the way the +metrics above do, so nothing here estimates them numerically. Reason about +both directly from the concept description: does this combination carry a +specific safety hazard, and is the infrastructure/regulatory environment it +needs realistic? Both feed into the RATING below as qualitative judgment +calls, not as scores of their own. + ## What makes something IMPLAUSIBLE Mark IMPLAUSIBLE if either of these is true: - It is physically or engineering-wise impossible given the components @@ -150,15 +166,13 @@ Mark IMPLAUSIBLE if either of these is true: fatiguing a hull over time is a real structural risk, not just "explosives are dangerous in general"; a fuel that's fine in the open becoming concentrated in a sealed tube is a real risk, not just "fuel is - flammable"). A low given safety score is a signal the pipeline already - found something concerning — treat it as evidence, not noise to explain - away. + flammable"). - Or: a real regulatory/infrastructure barrier with no plausible workaround. None of these make something implausible on their own: - being unoriginal or something like it already exists - being expensive, slow, or short-range -- a single mediocre score on one metric that isn't safety-related +- a single mediocre score on one metric Most concepts that reach this review are ordinary and workable; reserve IMPLAUSIBLE for a real, specific problem you can name — but don't require @@ -167,14 +181,26 @@ Reason from the physics and engineering actually described here, not from whether something like it already exists — novelty or lack of it is not evidence either way. +## Overall Rating +Separately from the plausibility verdict, give ONE holistic rating — +LOW, MEDIUM, or HIGH — for how good this combination is overall. This is a +single combined judgment, not a separate score per attribute: weigh the +metric scores above together with your own qualitative read on safety and +accessibility into one rating, the way a person sizing up the whole concept +would, not a checklist of independent numbers. + ## What to write In 2-4 sentences, give your reasoning, then check it against the scores -above: if your reasoning conflicts with a score (e.g. you believe this is -hazardous but its safety score is high), name the metric and say so +above: if your reasoning conflicts with a score (e.g. you believe cost is +a serious problem but its cost score is high), name the metric and say so explicitly — don't silently contradict a given score. -Finish with exactly one line: +Finish with exactly two lines. For the first, pick exactly one: +RATING: LOW +RATING: MEDIUM +RATING: HIGH + +Then, for the second, pick exactly one: VERDICT: PLAUSIBLE -or VERDICT: IMPLAUSIBLE """ diff --git a/src/physcom/llm/providers/gemini.py b/src/physcom/llm/providers/gemini.py index 643b1e4..bf106bb 100644 --- a/src/physcom/llm/providers/gemini.py +++ b/src/physcom/llm/providers/gemini.py @@ -13,7 +13,7 @@ from physcom.llm.prompts import ( format_metrics_for_prompt, format_scores_for_prompt, ) -from physcom.models.domain import MetricBound +from physcom.models.domain import Domain, MetricBound class GeminiLLMProvider(LLMProvider): @@ -51,12 +51,14 @@ class GeminiLLMProvider(LLMProvider): combination_description: str, raw_metrics: dict[str, float], normalized_scores: dict[str, float], - metrics: list[MetricBound], + domain: Domain, ) -> tuple[str, bool]: - scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, metrics) + scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, domain.metric_bounds) prompt = PLAUSIBILITY_REVIEW_PROMPT.format( description=combination_description, scores=scores_str, + domain_name=domain.name, + domain_description=domain.description, ) try: response = self._client.models.generate_content( diff --git a/src/physcom/llm/providers/mock.py b/src/physcom/llm/providers/mock.py index 832858e..f94e8a9 100644 --- a/src/physcom/llm/providers/mock.py +++ b/src/physcom/llm/providers/mock.py @@ -3,7 +3,7 @@ from __future__ import annotations from physcom.llm.base import LLMProvider -from physcom.models.domain import MetricBound +from physcom.models.domain import Domain, MetricBound class MockLLMProvider(LLMProvider): @@ -25,7 +25,7 @@ class MockLLMProvider(LLMProvider): combination_description: str, raw_metrics: dict[str, float], normalized_scores: dict[str, float], - metrics: list[MetricBound], + domain: Domain, ) -> tuple[str, bool]: avg = sum(normalized_scores.values()) / max(len(normalized_scores), 1) if avg > 0.5: diff --git a/src/physcom/llm/providers/ollama.py b/src/physcom/llm/providers/ollama.py index e2ef3a7..517c907 100644 --- a/src/physcom/llm/providers/ollama.py +++ b/src/physcom/llm/providers/ollama.py @@ -14,7 +14,7 @@ from physcom.llm.prompts import ( format_metrics_for_prompt, format_scores_for_prompt, ) -from physcom.models.domain import MetricBound +from physcom.models.domain import Domain, MetricBound class OllamaLLMProvider(LLMProvider): @@ -39,12 +39,14 @@ class OllamaLLMProvider(LLMProvider): combination_description: str, raw_metrics: dict[str, float], normalized_scores: dict[str, float], - metrics: list[MetricBound], + domain: Domain, ) -> tuple[str, bool]: - scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, metrics) + scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, domain.metric_bounds) prompt = PLAUSIBILITY_REVIEW_PROMPT.format( description=combination_description, scores=scores_str, + domain_name=domain.name, + domain_description=domain.description, ) text = self._generate(prompt, json_mode=False).strip() return (text, parse_verdict(text)) diff --git a/src/physcom/seed/transport_example.py b/src/physcom/seed/transport_example.py index a64b64a..a256e80 100644 --- a/src/physcom/seed/transport_example.py +++ b/src/physcom/seed/transport_example.py @@ -728,11 +728,22 @@ URBAN_COMMUTING = Domain( name="urban_commuting", description="Daily travel within a city, 1-50km range", metric_bounds=[ - MetricBound("power_density", weight=0.25, norm_min=1, norm_max=2000, unit="W/kg"), - MetricBound("cost_efficiency", weight=0.25, norm_min=1e-5, norm_max=2e-3, unit="$/m", lower_is_better=True), - MetricBound("safety", weight=0.25, norm_min=0.0, norm_max=1.0, unit="0-1"), - MetricBound("availability", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"), - MetricBound("range_fuel", weight=0.10, norm_min=5000, norm_max=500000, unit="m"), + # safety and availability removed from the scored/weighted metric set: + # both are judgment calls (risk assessment, infrastructure prevalence), + # not physics quantities with a formula, and running them through the + # same log-normalize() built for physical quantities produced + # incoherent results (a safety raw value already declared as "0-1" + # getting re-normalized into a different, unexplainable 0-1 number -- + # see combo 1540's review, where phi4 could only cite the post- + # normalization number with no way to justify it). Safety is now a + # qualitative consideration folded into pass 4's holistic RATING + # instead. Availability needs real per-infrastructure-type research + # this project hasn't done -- not scored anywhere for now rather than + # pretend a quick formula or an equally uninformed LLM guess settles it. + # Weights renormalized to sum to 1.0 across the remaining metrics. + MetricBound("power_density", weight=0.4167, norm_min=1, norm_max=2000, unit="W/kg"), + MetricBound("cost_efficiency", weight=0.4167, norm_min=1e-5, norm_max=2e-3, unit="$/m", lower_is_better=True), + MetricBound("range_fuel", weight=0.1666, norm_min=5000, norm_max=500000, unit="m"), ], constraints=[DomainConstraint("medium", ["ground", "air"])], ) @@ -741,11 +752,12 @@ INTERPLANETARY = Domain( name="interplanetary_travel", description="Travel between planets within a solar system", metric_bounds=[ - MetricBound("power_density", weight=0.30, norm_min=10, norm_max=10000, unit="W/kg"), - MetricBound("range_fuel", weight=0.30, norm_min=1e9, norm_max=1e13, unit="m"), - MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"), - MetricBound("cost_efficiency", weight=0.10, norm_min=1.0, norm_max=1e6, unit="$/m", lower_is_better=True), - MetricBound("range_degradation", weight=0.10, norm_min=8640000, norm_max=3.1536e9, unit="s"), + # safety removed -- see URBAN_COMMUTING comment above. Weights + # renormalized across the remaining metrics. + MetricBound("power_density", weight=0.375, norm_min=10, norm_max=10000, unit="W/kg"), + MetricBound("range_fuel", weight=0.375, norm_min=1e9, norm_max=1e13, unit="m"), + MetricBound("cost_efficiency", weight=0.125, norm_min=1.0, norm_max=1e6, unit="$/m", lower_is_better=True), + MetricBound("range_degradation", weight=0.125, norm_min=8640000, norm_max=3.1536e9, unit="s"), ], constraints=[DomainConstraint("medium", ["space"])], ) @@ -754,11 +766,12 @@ MARITIME_SHIPPING = Domain( name="maritime_shipping", description="Ocean cargo transport between ports, 100-40000km range", metric_bounds=[ - MetricBound("power_density", weight=0.15, norm_min=1, norm_max=1000, unit="W/kg"), - MetricBound("cargo_capacity", weight=0.25, norm_min=1000, norm_max=2e8, unit="kg"), - MetricBound("cost_efficiency", weight=0.25, norm_min=1e-9, norm_max=1e-6, unit="$/(kg\u00b7m)", lower_is_better=True), - MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"), - MetricBound("range_fuel", weight=0.15, norm_min=100000, norm_max=40000000, unit="m"), + # safety removed -- see URBAN_COMMUTING comment above. Weights + # renormalized across the remaining metrics. + MetricBound("power_density", weight=0.1875, norm_min=1, norm_max=1000, unit="W/kg"), + MetricBound("cargo_capacity", weight=0.3125, norm_min=1000, norm_max=2e8, unit="kg"), + MetricBound("cost_efficiency", weight=0.3125, norm_min=1e-9, norm_max=1e-6, unit="$/(kg\u00b7m)", lower_is_better=True), + MetricBound("range_fuel", weight=0.1875, norm_min=100000, norm_max=40000000, unit="m"), ], constraints=[DomainConstraint("medium", ["water"])], ) @@ -767,11 +780,12 @@ LAST_MILE_DELIVERY = Domain( name="last_mile_delivery", description="Short-range package delivery within neighborhoods, 0.5-15km", metric_bounds=[ - MetricBound("power_density", weight=0.25, norm_min=1, norm_max=500, unit="W/kg"), - MetricBound("cost_efficiency", weight=0.30, norm_min=1e-5, norm_max=5e-3, unit="$/m", lower_is_better=True), - MetricBound("cargo_capacity_kg", weight=0.20, norm_min=1, norm_max=500, unit="kg"), - MetricBound("safety", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"), - MetricBound("environmental_impact", weight=0.10, norm_min=0, norm_max=5e-4, unit="kg/m", lower_is_better=True), + # safety removed -- see URBAN_COMMUTING comment above. Weights + # renormalized across the remaining metrics. + MetricBound("power_density", weight=0.2941, norm_min=1, norm_max=500, unit="W/kg"), + MetricBound("cost_efficiency", weight=0.3529, norm_min=1e-5, norm_max=5e-3, unit="$/m", lower_is_better=True), + MetricBound("cargo_capacity_kg", weight=0.2353, norm_min=1, norm_max=500, unit="kg"), + MetricBound("environmental_impact", weight=0.1177, norm_min=0, norm_max=5e-4, unit="kg/m", lower_is_better=True), ], constraints=[DomainConstraint("medium", ["ground", "air"])], ) @@ -839,12 +853,11 @@ def load_transport_seed(repo) -> dict: counts["domains"] += 1 except sqlite3.IntegrityError: pass - # Backfill metric units and lower_is_better on existing DBs. - for mb in domain.metric_bounds: - repo.ensure_metric(mb.metric_name, unit=mb.unit) - repo.backfill_metric_unit(domain.name, mb.metric_name, mb.unit) - if mb.lower_is_better: - repo.backfill_lower_is_better(domain.name, mb.metric_name) + # Sync domain_metric_weights to exactly match this domain's current + # metric_bounds on existing DBs -- upserts weight/norm_min/norm_max/ + # unit for current metrics and removes any that were dropped (e.g. + # safety/availability no longer scored). + repo.sync_domain_metric_weights(domain) # Backfill domain constraints repo.replace_domain_constraints(domain) diff --git a/src/physcom_web/routes/results.py b/src/physcom_web/routes/results.py index 675e9c7..c765d64 100644 --- a/src/physcom_web/routes/results.py +++ b/src/physcom_web/routes/results.py @@ -25,9 +25,11 @@ def results_domain(domain_name: str): return redirect(url_for("results.results_index")) status_filter = request.args.get("status") - results = repo.get_all_results(domain_name, status=status_filter) + rating_filter = request.args.get("rating") + results = repo.get_all_results(domain_name, status=status_filter, rating=rating_filter) # Domain-scoped status counts (only combos that have results in this domain) statuses = repo.count_combinations_by_status(domain_name=domain_name) + ratings = repo.count_results_by_rating(domain_name) return render_template( "results/list.html", @@ -35,7 +37,9 @@ def results_domain(domain_name: str): domain=domain, results=results, status_filter=status_filter, + rating_filter=rating_filter, statuses=statuses, + ratings=ratings, total_results=sum(statuses.values()), ) @@ -101,6 +105,7 @@ def submit_review(domain_name: str, combo_id: int): novelty_flag=novelty_flag, llm_review=existing.get("llm_review") if existing else None, human_notes=human_notes, + qualitative_rating=existing.get("qualitative_rating") if existing else None, ) repo.update_combination_status(combo_id, "reviewed") diff --git a/src/physcom_web/static/style.css b/src/physcom_web/static/style.css index 6580802..a92b5bd 100644 --- a/src/physcom_web/static/style.css +++ b/src/physcom_web/static/style.css @@ -214,6 +214,9 @@ table.compact th, table.compact td { padding: 0.25rem 0.4rem; font-size: 0.83rem .badge-llm_reviewed { background: rgba(107,163,160,0.12); color: var(--accent-teal); border-color: rgba(107,163,160,0.25); } .badge-reviewed { background: rgba(155,142,196,0.12); color: var(--accent-violet); border-color: rgba(155,142,196,0.25); } .badge-pending { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); } +.badge-rating-low { background: rgba(184,92,92,0.12); color: var(--accent-red); border-color: rgba(184,92,92,0.25); } +.badge-rating-medium { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); } +.badge-rating-high { background: rgba(122,171,138,0.12); color: var(--accent-green); border-color: rgba(122,171,138,0.25); } /* ── Buttons ─────────────────────────────────────────────── */ .btn { diff --git a/src/physcom_web/templates/home.html b/src/physcom_web/templates/home.html index 9db8ccb..4d9e4bd 100644 --- a/src/physcom_web/templates/home.html +++ b/src/physcom_web/templates/home.html @@ -50,12 +50,13 @@

Physics Estimation

- Surviving combinations get raw metric estimates — speed, cost, - safety, range — via heuristic stubs or an LLM provider that - reasons about the physical properties of each pairing. + Surviving combinations get raw metric estimates — power + density, cost, range — from a deterministic physics engine + that sizes each combination from its own declared attributes, not + a guess.

- Bicycle + Human Pedalling → speed: 20 km/h, cost: $0.01/km + Bicycle + Human Muscle → power density: 4.4 W/kg, range: 500km
@@ -72,8 +73,8 @@ Combinations are ranked within their domain.

- Domain urban_commuting weights: speed 25%, cost 25%, - safety 25%, availability 15%, range 10% + Domain urban_commuting weights: power density 42%, + cost 42%, range 17%
@@ -85,9 +86,11 @@

LLM Review

- Top-scoring combinations are sent to a language model for plausibility - and novelty assessment — catching physically valid but practically - absurd pairings. + Top-scoring combinations are sent to a language model for a + plausibility verdict plus a holistic LOW/MEDIUM/HIGH rating — + weighing safety and accessibility as qualitative judgment calls + alongside the physics scores, catching physically valid but + practically absurd pairings.

"Train + Solar Sail: structurally valid constraints, but solar radiation @@ -163,14 +166,15 @@

Metrics

- Quantitative axes like speed, cost, safety, and range. Each metric - has a domain-specific weight and normalization range. Some are - inverted — lower cost is better. + Quantitative physics axes like power density, cost, and range. Each + metric has a domain-specific weight and normalization range. Some + are inverted — lower cost is better. Safety and accessibility + are judgment calls, not physics quantities — they're weighed + qualitatively in the LLM review pass instead of scored here.

- speed + power_density cost_efficiency - safety range_fuel
diff --git a/src/physcom_web/templates/results/detail.html b/src/physcom_web/templates/results/detail.html index 46e11e3..a79c2db 100644 --- a/src/physcom_web/templates/results/detail.html +++ b/src/physcom_web/templates/results/detail.html @@ -27,6 +27,9 @@ {% if result %}
Composite Score
{{ "%.4f"|format(result.composite_score) }}
Pass Reached
{{ result.pass_reached }}
+ {% if result.qualitative_rating %} +
Rating
{{ result.qualitative_rating }}
+ {% endif %} {% if result.novelty_flag %}
Novelty
{{ result.novelty_flag }}
{% endif %} @@ -102,11 +105,16 @@ {%- elif s.raw_value >= mb.norm_max -%} at/above max{{ ' (worst)' if mb.lower_is_better else '' }} {%- else -%} - {% set pct = ((s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100) | int %} + {% set raw_pct = (s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100 %} + {# For lower_is_better metrics, raw_pct alone measures distance from norm_min, + not quality -- a value near norm_min (excellent, cost near its floor) would + otherwise render as a near-empty bar. Invert so the bar and percentage always + mean "how good", matching the normalized score's own higher-is-better convention. #} + {% set pct = ((100 - raw_pct) if mb.lower_is_better else raw_pct) | int %}
- ~{{ pct }}%{{ ' ↓' if mb.lower_is_better else '' }} + ~{{ pct }}%{{ ' (lower is better)' if mb.lower_is_better else '' }} {%- endif -%} {%- else -%} — diff --git a/src/physcom_web/templates/results/list.html b/src/physcom_web/templates/results/list.html index 18e51a3..0933330 100644 --- a/src/physcom_web/templates/results/list.html +++ b/src/physcom_web/templates/results/list.html @@ -26,11 +26,11 @@ {% if statuses %}
- Filter: - Status: + All ({{ total_results }}) {% for s, cnt in statuses.items() %} - {{ s }} ({{ cnt }}) @@ -38,9 +38,25 @@
{% endif %} + {% if ratings %} +
+ Rating: + All + {% for rt in ['HIGH', 'MEDIUM', 'LOW'] %} + {% if rt in ratings %} + + {{ rt }} ({{ ratings[rt] }}) + + {% endif %} + {% endfor %} +
+ {% endif %} + {% if not results %} - {% if status_filter %} -

No results with status "{{ status_filter }}" in this domain.

+ {% if status_filter or rating_filter %} +

No results matching that filter in this domain.

{% else %}

No results for this domain yet. Run the pipeline first.

{% endif %} @@ -52,6 +68,7 @@ Score Entities Status + Rating Details @@ -69,6 +86,13 @@ {{ r.combination.status }} {%- endif -%} + + {%- if r.qualitative_rating -%} + {{ r.qualitative_rating }} + {%- else -%} + — + {%- endif -%} + {%- if r.domain_block_reason -%} {{ r.domain_block_reason }} diff --git a/tests/test_pipeline_async.py b/tests/test_pipeline_async.py index 5e588c8..2492e93 100644 --- a/tests/test_pipeline_async.py +++ b/tests/test_pipeline_async.py @@ -335,28 +335,34 @@ def test_p3_fail_below_threshold(seeded_repo): def test_p4_fail_implausible(seeded_repo): - """Combos deemed implausible by LLM should get p4_fail status.""" + """Combos deemed implausible by LLM should get p4_fail status. + + Pass 2 is estimator-only now (never calls the LLM), so there's no way + to force every combo's raw estimates toward a controlled low/high value + the way MockLLMProvider's default_estimates used to. Force the pass-4 + verdict directly instead -- what's under test here is pipeline.py's + wiring of review_plausibility's return value to status/counters, not + MockLLMProvider's avg-based heuristic. + """ from physcom.llm.providers.mock import MockLLMProvider + class AlwaysImplausibleLLM(MockLLMProvider): + def review_plausibility(self, description, raw_metrics, normalized_scores, domain): + return ("Always implausible for testing.", False) + repo = seeded_repo domain = repo.get_domain("urban_commuting") resolver = ConstraintResolver() scorer = Scorer(domain) - # Low estimates → normalized scores avg <= 0.5 → MockLLMProvider returns (text, False) # Use threshold=0.0 so no combo gets p3_fail and all reach pass 4 - mock_llm = MockLLMProvider(default_estimates={ - "power_density": 0.1, "cost_efficiency": 0.1, "safety": 0.1, - "availability": 0.1, "range_fuel": 0.1, - }) - pipeline = Pipeline(repo, resolver, scorer, llm=mock_llm) + pipeline = Pipeline(repo, resolver, scorer, llm=AlwaysImplausibleLLM()) result = pipeline.run( domain, ["platform", "actuator", "energy_storage"], score_threshold=0.0, passes=[1, 2, 3, 4], ) - # With low normalized scores (avg <= 0.5), reviewed combos should be p4_fail assert result.pass4_failed > 0 assert result.pass4_reviewed == 0 @@ -367,20 +373,23 @@ def test_p4_fail_implausible(seeded_repo): def test_p4_pass_plausible(seeded_repo): - """Combos deemed plausible by LLM should get llm_reviewed status.""" + """Combos deemed plausible by LLM should get llm_reviewed status. + + See test_p4_fail_implausible on why the verdict is forced directly + rather than via controlled pass-2 estimates. + """ from physcom.llm.providers.mock import MockLLMProvider + class AlwaysPlausibleLLM(MockLLMProvider): + def review_plausibility(self, description, raw_metrics, normalized_scores, domain): + return ("Always plausible for testing.", True) + repo = seeded_repo domain = repo.get_domain("urban_commuting") resolver = ConstraintResolver() scorer = Scorer(domain) - # High estimates → avg > 0.5 → MockLLMProvider returns (text, True) - mock_llm = MockLLMProvider(default_estimates={ - "power_density": 500.0, "cost_efficiency": 5e-4, "safety": 0.6, - "availability": 0.7, "range_fuel": 200000.0, - }) - pipeline = Pipeline(repo, resolver, scorer, llm=mock_llm) + pipeline = Pipeline(repo, resolver, scorer, llm=AlwaysPlausibleLLM()) result = pipeline.run( domain, ["platform", "actuator", "energy_storage"],