Compare commits
2 Commits
730a23bac3
...
d871635779
| Author | SHA1 | Date | |
|---|---|---|---|
| d871635779 | |||
| 76f460499a |
@@ -230,24 +230,37 @@ class Repository:
|
||||
self.conn.commit()
|
||||
return row["id"]
|
||||
|
||||
def backfill_lower_is_better(self, domain_name: str, metric_name: str) -> None:
|
||||
"""Set lower_is_better=1 for an existing domain-metric row that still has the default 0."""
|
||||
def sync_domain_metric_weights(self, domain: Domain) -> None:
|
||||
"""Make domain_metric_weights exactly match domain.metric_bounds on an
|
||||
already-seeded domain: upserts weight/norm_min/norm_max/unit for every
|
||||
currently-declared metric, and deletes any row for a metric that's been
|
||||
removed from the domain (e.g. safety/availability dropped from the
|
||||
scored set). Safe to call whether the domain was just freshly inserted
|
||||
or already existed.
|
||||
"""
|
||||
row = self.conn.execute(
|
||||
"SELECT id FROM domains WHERE name = ?", (domain.name,)
|
||||
).fetchone()
|
||||
if not row:
|
||||
return
|
||||
domain_id = row["id"]
|
||||
keep_ids = []
|
||||
for mb in domain.metric_bounds:
|
||||
metric_id = self.ensure_metric(mb.metric_name, unit=mb.unit)
|
||||
keep_ids.append(metric_id)
|
||||
self.conn.execute(
|
||||
"""UPDATE domain_metric_weights SET lower_is_better = 1
|
||||
WHERE lower_is_better = 0
|
||||
AND domain_id = (SELECT id FROM domains WHERE name = ?)
|
||||
AND metric_id = (SELECT id FROM metrics WHERE name = ?)""",
|
||||
(domain_name, metric_name),
|
||||
"""INSERT OR REPLACE INTO domain_metric_weights
|
||||
(domain_id, metric_id, weight, norm_min, norm_max, lower_is_better, unit)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||
(domain_id, metric_id, mb.weight, mb.norm_min, mb.norm_max,
|
||||
int(mb.lower_is_better), mb.unit),
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def backfill_metric_unit(self, domain_name: str, metric_name: str, unit: str) -> None:
|
||||
"""Set this domain-metric row's unit — unit is domain-scoped, not global to the metric name."""
|
||||
if keep_ids:
|
||||
placeholders = ",".join("?" * len(keep_ids))
|
||||
self.conn.execute(
|
||||
"""UPDATE domain_metric_weights SET unit = ?
|
||||
WHERE domain_id = (SELECT id FROM domains WHERE name = ?)
|
||||
AND metric_id = (SELECT id FROM metrics WHERE name = ?)""",
|
||||
(unit, domain_name, metric_name),
|
||||
f"""DELETE FROM domain_metric_weights
|
||||
WHERE domain_id = ? AND metric_id NOT IN ({placeholders})""",
|
||||
(domain_id, *keep_ids),
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
@@ -593,15 +606,18 @@ class Repository:
|
||||
llm_review: str | None = None,
|
||||
human_notes: str | None = None,
|
||||
domain_block_reason: str | None = None,
|
||||
qualitative_rating: str | None = None,
|
||||
commit: bool = True,
|
||||
) -> None:
|
||||
self.conn.execute(
|
||||
"""INSERT OR REPLACE INTO combination_results
|
||||
(combination_id, domain_id, composite_score, novelty_flag,
|
||||
llm_review, human_notes, pass_reached, domain_block_reason)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
llm_review, human_notes, pass_reached, domain_block_reason,
|
||||
qualitative_rating)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(combo_id, domain_id, composite_score, novelty_flag,
|
||||
llm_review, human_notes, pass_reached, domain_block_reason),
|
||||
llm_review, human_notes, pass_reached, domain_block_reason,
|
||||
qualitative_rating),
|
||||
)
|
||||
if commit:
|
||||
self.conn.commit()
|
||||
@@ -641,6 +657,20 @@ class Repository:
|
||||
).fetchall()
|
||||
return {r["status"]: r["cnt"] for r in rows}
|
||||
|
||||
def count_results_by_rating(self, domain_name: str) -> dict[str, int]:
|
||||
"""Count results by qualitative_rating (LOW/MEDIUM/HIGH) for a domain.
|
||||
Rows with no rating (not yet pass-4 reviewed, or reviewed before this
|
||||
existed) are excluded, not bucketed as a pseudo-status."""
|
||||
rows = self.conn.execute(
|
||||
"""SELECT cr.qualitative_rating as rating, COUNT(*) as cnt
|
||||
FROM combination_results cr
|
||||
JOIN domains d ON cr.domain_id = d.id
|
||||
WHERE d.name = ? AND cr.qualitative_rating IS NOT NULL
|
||||
GROUP BY cr.qualitative_rating""",
|
||||
(domain_name,),
|
||||
).fetchall()
|
||||
return {r["rating"]: r["cnt"] for r in rows}
|
||||
|
||||
def get_pipeline_summary(self, domain_name: str) -> dict | None:
|
||||
"""Return a summary of results for a domain, or None if no results."""
|
||||
row = self.conn.execute(
|
||||
@@ -685,8 +715,12 @@ class Repository:
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def get_all_results(self, domain_name: str, status: str | None = None) -> list[dict]:
|
||||
"""Return all results for a domain, optionally filtered by combo status."""
|
||||
def get_all_results(
|
||||
self, domain_name: str, status: str | None = None, rating: str | None = None
|
||||
) -> list[dict]:
|
||||
"""Return all results for a domain, optionally filtered by combo
|
||||
status and/or qualitative_rating (LOW/MEDIUM/HIGH, independent filters
|
||||
that combine with AND)."""
|
||||
query = """SELECT cr.*, c.hash, c.status as combo_status, d.name as domain_name
|
||||
FROM combination_results cr
|
||||
JOIN combinations c ON cr.combination_id = c.id
|
||||
@@ -698,6 +732,9 @@ class Repository:
|
||||
elif status:
|
||||
query += " AND c.status = ? AND cr.domain_block_reason IS NULL"
|
||||
params.append(status)
|
||||
if rating:
|
||||
query += " AND cr.qualitative_rating = ?"
|
||||
params.append(rating)
|
||||
query += " ORDER BY cr.composite_score DESC"
|
||||
rows = self.conn.execute(query, params).fetchall()
|
||||
combo_ids = [r["combination_id"] for r in rows]
|
||||
@@ -712,6 +749,7 @@ class Repository:
|
||||
"pass_reached": r["pass_reached"],
|
||||
"domain_id": r["domain_id"],
|
||||
"domain_block_reason": r["domain_block_reason"],
|
||||
"qualitative_rating": r["qualitative_rating"],
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
@@ -91,6 +91,7 @@ CREATE TABLE IF NOT EXISTS combination_results (
|
||||
human_notes TEXT,
|
||||
pass_reached INTEGER,
|
||||
domain_block_reason TEXT,
|
||||
qualitative_rating TEXT,
|
||||
UNIQUE(combination_id, domain_id)
|
||||
);
|
||||
|
||||
@@ -165,6 +166,10 @@ def _migrate(conn: sqlite3.Connection) -> None:
|
||||
conn.execute(
|
||||
"ALTER TABLE combination_results ADD COLUMN domain_block_reason TEXT"
|
||||
)
|
||||
if "qualitative_rating" not in result_cols:
|
||||
conn.execute(
|
||||
"ALTER TABLE combination_results ADD COLUMN qualitative_rating TEXT"
|
||||
)
|
||||
|
||||
# Backfill: cost_efficiency is lower-is-better in all domains
|
||||
conn.execute(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from physcom.models.domain import MetricBound
|
||||
from physcom.models.domain import Domain, MetricBound
|
||||
|
||||
|
||||
class LLMRateLimitError(Exception):
|
||||
@@ -40,7 +40,7 @@ class LLMProvider(ABC):
|
||||
combination_description: str,
|
||||
raw_metrics: dict[str, float],
|
||||
normalized_scores: dict[str, float],
|
||||
metrics: list[MetricBound],
|
||||
domain: Domain,
|
||||
) -> tuple[str, bool]:
|
||||
"""Given a combination, its raw physical estimates, and their
|
||||
normalized scores, return a (text, is_plausible) tuple:
|
||||
@@ -50,6 +50,11 @@ class LLMProvider(ABC):
|
||||
normalized score) so the review can reason from the actual physics
|
||||
rather than only a compressed 0-1 number, which can look
|
||||
deceptively bad for a metric whose scale was built for a different
|
||||
kind of vehicle. `metrics` carries each metric's unit for
|
||||
formatting the raw value meaningfully."""
|
||||
kind of vehicle. `domain` carries both each metric's unit (via
|
||||
domain.metric_bounds, for formatting the raw value meaningfully)
|
||||
and the domain's own name/description, so the review judges a
|
||||
metric like range against what THIS domain actually needs rather
|
||||
than generic real-world expectations for the platform category
|
||||
(e.g. a short-hop domain shouldn't get judged against typical
|
||||
long-haul aircraft range)."""
|
||||
...
|
||||
|
||||
@@ -16,6 +16,13 @@ def parse_verdict(text: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def parse_rating(text: str) -> str | None:
|
||||
"""Extract RATING: LOW/MEDIUM/HIGH from response; None if absent (older
|
||||
reviews saved before this existed, or a malformed response)."""
|
||||
m = re.search(r"RATING:\s*(LOW|MEDIUM|HIGH)", text, re.IGNORECASE)
|
||||
return m.group(1).upper() if m else None
|
||||
|
||||
|
||||
def parse_metric_json(text: str, metrics: list[MetricBound]) -> dict[str, float]:
|
||||
"""Strip markdown fences and parse JSON; fall back to each metric's own
|
||||
norm_min/norm_max midpoint on error — a flat constant like 0.5 is
|
||||
|
||||
@@ -116,6 +116,14 @@ You are reviewing a transportation concept for real-world viability — could th
|
||||
actually be built and operated safely. Whether it is new, exciting, or original
|
||||
is NOT the question.
|
||||
|
||||
## Domain
|
||||
This concept is being evaluated for "{domain_name}": {domain_description}
|
||||
Judge every metric against what THIS domain actually needs, not general
|
||||
expectations for the platform category. A range far beyond what this domain
|
||||
requires is a strength or a non-issue, never a weakness -- don't reason about
|
||||
range, speed, or capacity by comparing to what other vehicles of this type
|
||||
typically have in general use; compare to what this specific domain calls for.
|
||||
|
||||
## Concept
|
||||
{description}
|
||||
|
||||
@@ -139,6 +147,14 @@ before treating the score as evidence of a problem.
|
||||
|
||||
{scores}
|
||||
|
||||
Safety and accessibility (infrastructure/regulatory availability) are NOT
|
||||
among the scores above — neither reduces to a physics formula the way the
|
||||
metrics above do, so nothing here estimates them numerically. Reason about
|
||||
both directly from the concept description: does this combination carry a
|
||||
specific safety hazard, and is the infrastructure/regulatory environment it
|
||||
needs realistic? Both feed into the RATING below as qualitative judgment
|
||||
calls, not as scores of their own.
|
||||
|
||||
## What makes something IMPLAUSIBLE
|
||||
Mark IMPLAUSIBLE if either of these is true:
|
||||
- It is physically or engineering-wise impossible given the components
|
||||
@@ -150,15 +166,13 @@ Mark IMPLAUSIBLE if either of these is true:
|
||||
fatiguing a hull over time is a real structural risk, not just "explosives
|
||||
are dangerous in general"; a fuel that's fine in the open becoming
|
||||
concentrated in a sealed tube is a real risk, not just "fuel is
|
||||
flammable"). A low given safety score is a signal the pipeline already
|
||||
found something concerning — treat it as evidence, not noise to explain
|
||||
away.
|
||||
flammable").
|
||||
- Or: a real regulatory/infrastructure barrier with no plausible workaround.
|
||||
|
||||
None of these make something implausible on their own:
|
||||
- being unoriginal or something like it already exists
|
||||
- being expensive, slow, or short-range
|
||||
- a single mediocre score on one metric that isn't safety-related
|
||||
- a single mediocre score on one metric
|
||||
|
||||
Most concepts that reach this review are ordinary and workable; reserve
|
||||
IMPLAUSIBLE for a real, specific problem you can name — but don't require
|
||||
@@ -167,14 +181,26 @@ Reason from the physics and engineering actually described here, not from
|
||||
whether something like it already exists — novelty or lack of it is not
|
||||
evidence either way.
|
||||
|
||||
## Overall Rating
|
||||
Separately from the plausibility verdict, give ONE holistic rating —
|
||||
LOW, MEDIUM, or HIGH — for how good this combination is overall. This is a
|
||||
single combined judgment, not a separate score per attribute: weigh the
|
||||
metric scores above together with your own qualitative read on safety and
|
||||
accessibility into one rating, the way a person sizing up the whole concept
|
||||
would, not a checklist of independent numbers.
|
||||
|
||||
## What to write
|
||||
In 2-4 sentences, give your reasoning, then check it against the scores
|
||||
above: if your reasoning conflicts with a score (e.g. you believe this is
|
||||
hazardous but its safety score is high), name the metric and say so
|
||||
above: if your reasoning conflicts with a score (e.g. you believe cost is
|
||||
a serious problem but its cost score is high), name the metric and say so
|
||||
explicitly — don't silently contradict a given score.
|
||||
|
||||
Finish with exactly one line:
|
||||
Finish with exactly two lines. For the first, pick exactly one:
|
||||
RATING: LOW
|
||||
RATING: MEDIUM
|
||||
RATING: HIGH
|
||||
|
||||
Then, for the second, pick exactly one:
|
||||
VERDICT: PLAUSIBLE
|
||||
or
|
||||
VERDICT: IMPLAUSIBLE
|
||||
"""
|
||||
|
||||
@@ -13,7 +13,7 @@ from physcom.llm.prompts import (
|
||||
format_metrics_for_prompt,
|
||||
format_scores_for_prompt,
|
||||
)
|
||||
from physcom.models.domain import MetricBound
|
||||
from physcom.models.domain import Domain, MetricBound
|
||||
|
||||
|
||||
class GeminiLLMProvider(LLMProvider):
|
||||
@@ -51,12 +51,14 @@ class GeminiLLMProvider(LLMProvider):
|
||||
combination_description: str,
|
||||
raw_metrics: dict[str, float],
|
||||
normalized_scores: dict[str, float],
|
||||
metrics: list[MetricBound],
|
||||
domain: Domain,
|
||||
) -> tuple[str, bool]:
|
||||
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, metrics)
|
||||
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, domain.metric_bounds)
|
||||
prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
|
||||
description=combination_description,
|
||||
scores=scores_str,
|
||||
domain_name=domain.name,
|
||||
domain_description=domain.description,
|
||||
)
|
||||
try:
|
||||
response = self._client.models.generate_content(
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from physcom.llm.base import LLMProvider
|
||||
from physcom.models.domain import MetricBound
|
||||
from physcom.models.domain import Domain, MetricBound
|
||||
|
||||
|
||||
class MockLLMProvider(LLMProvider):
|
||||
@@ -25,7 +25,7 @@ class MockLLMProvider(LLMProvider):
|
||||
combination_description: str,
|
||||
raw_metrics: dict[str, float],
|
||||
normalized_scores: dict[str, float],
|
||||
metrics: list[MetricBound],
|
||||
domain: Domain,
|
||||
) -> tuple[str, bool]:
|
||||
avg = sum(normalized_scores.values()) / max(len(normalized_scores), 1)
|
||||
if avg > 0.5:
|
||||
|
||||
@@ -14,7 +14,7 @@ from physcom.llm.prompts import (
|
||||
format_metrics_for_prompt,
|
||||
format_scores_for_prompt,
|
||||
)
|
||||
from physcom.models.domain import MetricBound
|
||||
from physcom.models.domain import Domain, MetricBound
|
||||
|
||||
|
||||
class OllamaLLMProvider(LLMProvider):
|
||||
@@ -39,12 +39,14 @@ class OllamaLLMProvider(LLMProvider):
|
||||
combination_description: str,
|
||||
raw_metrics: dict[str, float],
|
||||
normalized_scores: dict[str, float],
|
||||
metrics: list[MetricBound],
|
||||
domain: Domain,
|
||||
) -> tuple[str, bool]:
|
||||
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, metrics)
|
||||
scores_str = format_scores_for_prompt(raw_metrics, normalized_scores, domain.metric_bounds)
|
||||
prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
|
||||
description=combination_description,
|
||||
scores=scores_str,
|
||||
domain_name=domain.name,
|
||||
domain_description=domain.description,
|
||||
)
|
||||
text = self._generate(prompt, json_mode=False).strip()
|
||||
return (text, parse_verdict(text))
|
||||
|
||||
@@ -728,11 +728,22 @@ URBAN_COMMUTING = Domain(
|
||||
name="urban_commuting",
|
||||
description="Daily travel within a city, 1-50km range",
|
||||
metric_bounds=[
|
||||
MetricBound("power_density", weight=0.25, norm_min=1, norm_max=2000, unit="W/kg"),
|
||||
MetricBound("cost_efficiency", weight=0.25, norm_min=1e-5, norm_max=2e-3, unit="$/m", lower_is_better=True),
|
||||
MetricBound("safety", weight=0.25, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
||||
MetricBound("availability", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
||||
MetricBound("range_fuel", weight=0.10, norm_min=5000, norm_max=500000, unit="m"),
|
||||
# safety and availability removed from the scored/weighted metric set:
|
||||
# both are judgment calls (risk assessment, infrastructure prevalence),
|
||||
# not physics quantities with a formula, and running them through the
|
||||
# same log-normalize() built for physical quantities produced
|
||||
# incoherent results (a safety raw value already declared as "0-1"
|
||||
# getting re-normalized into a different, unexplainable 0-1 number --
|
||||
# see combo 1540's review, where phi4 could only cite the post-
|
||||
# normalization number with no way to justify it). Safety is now a
|
||||
# qualitative consideration folded into pass 4's holistic RATING
|
||||
# instead. Availability needs real per-infrastructure-type research
|
||||
# this project hasn't done -- not scored anywhere for now rather than
|
||||
# pretend a quick formula or an equally uninformed LLM guess settles it.
|
||||
# Weights renormalized to sum to 1.0 across the remaining metrics.
|
||||
MetricBound("power_density", weight=0.4167, norm_min=1, norm_max=2000, unit="W/kg"),
|
||||
MetricBound("cost_efficiency", weight=0.4167, norm_min=1e-5, norm_max=2e-3, unit="$/m", lower_is_better=True),
|
||||
MetricBound("range_fuel", weight=0.1666, norm_min=5000, norm_max=500000, unit="m"),
|
||||
],
|
||||
constraints=[DomainConstraint("medium", ["ground", "air"])],
|
||||
)
|
||||
@@ -741,11 +752,12 @@ INTERPLANETARY = Domain(
|
||||
name="interplanetary_travel",
|
||||
description="Travel between planets within a solar system",
|
||||
metric_bounds=[
|
||||
MetricBound("power_density", weight=0.30, norm_min=10, norm_max=10000, unit="W/kg"),
|
||||
MetricBound("range_fuel", weight=0.30, norm_min=1e9, norm_max=1e13, unit="m"),
|
||||
MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
||||
MetricBound("cost_efficiency", weight=0.10, norm_min=1.0, norm_max=1e6, unit="$/m", lower_is_better=True),
|
||||
MetricBound("range_degradation", weight=0.10, norm_min=8640000, norm_max=3.1536e9, unit="s"),
|
||||
# safety removed -- see URBAN_COMMUTING comment above. Weights
|
||||
# renormalized across the remaining metrics.
|
||||
MetricBound("power_density", weight=0.375, norm_min=10, norm_max=10000, unit="W/kg"),
|
||||
MetricBound("range_fuel", weight=0.375, norm_min=1e9, norm_max=1e13, unit="m"),
|
||||
MetricBound("cost_efficiency", weight=0.125, norm_min=1.0, norm_max=1e6, unit="$/m", lower_is_better=True),
|
||||
MetricBound("range_degradation", weight=0.125, norm_min=8640000, norm_max=3.1536e9, unit="s"),
|
||||
],
|
||||
constraints=[DomainConstraint("medium", ["space"])],
|
||||
)
|
||||
@@ -754,11 +766,12 @@ MARITIME_SHIPPING = Domain(
|
||||
name="maritime_shipping",
|
||||
description="Ocean cargo transport between ports, 100-40000km range",
|
||||
metric_bounds=[
|
||||
MetricBound("power_density", weight=0.15, norm_min=1, norm_max=1000, unit="W/kg"),
|
||||
MetricBound("cargo_capacity", weight=0.25, norm_min=1000, norm_max=2e8, unit="kg"),
|
||||
MetricBound("cost_efficiency", weight=0.25, norm_min=1e-9, norm_max=1e-6, unit="$/(kg\u00b7m)", lower_is_better=True),
|
||||
MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
||||
MetricBound("range_fuel", weight=0.15, norm_min=100000, norm_max=40000000, unit="m"),
|
||||
# safety removed -- see URBAN_COMMUTING comment above. Weights
|
||||
# renormalized across the remaining metrics.
|
||||
MetricBound("power_density", weight=0.1875, norm_min=1, norm_max=1000, unit="W/kg"),
|
||||
MetricBound("cargo_capacity", weight=0.3125, norm_min=1000, norm_max=2e8, unit="kg"),
|
||||
MetricBound("cost_efficiency", weight=0.3125, norm_min=1e-9, norm_max=1e-6, unit="$/(kg\u00b7m)", lower_is_better=True),
|
||||
MetricBound("range_fuel", weight=0.1875, norm_min=100000, norm_max=40000000, unit="m"),
|
||||
],
|
||||
constraints=[DomainConstraint("medium", ["water"])],
|
||||
)
|
||||
@@ -767,11 +780,12 @@ LAST_MILE_DELIVERY = Domain(
|
||||
name="last_mile_delivery",
|
||||
description="Short-range package delivery within neighborhoods, 0.5-15km",
|
||||
metric_bounds=[
|
||||
MetricBound("power_density", weight=0.25, norm_min=1, norm_max=500, unit="W/kg"),
|
||||
MetricBound("cost_efficiency", weight=0.30, norm_min=1e-5, norm_max=5e-3, unit="$/m", lower_is_better=True),
|
||||
MetricBound("cargo_capacity_kg", weight=0.20, norm_min=1, norm_max=500, unit="kg"),
|
||||
MetricBound("safety", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"),
|
||||
MetricBound("environmental_impact", weight=0.10, norm_min=0, norm_max=5e-4, unit="kg/m", lower_is_better=True),
|
||||
# safety removed -- see URBAN_COMMUTING comment above. Weights
|
||||
# renormalized across the remaining metrics.
|
||||
MetricBound("power_density", weight=0.2941, norm_min=1, norm_max=500, unit="W/kg"),
|
||||
MetricBound("cost_efficiency", weight=0.3529, norm_min=1e-5, norm_max=5e-3, unit="$/m", lower_is_better=True),
|
||||
MetricBound("cargo_capacity_kg", weight=0.2353, norm_min=1, norm_max=500, unit="kg"),
|
||||
MetricBound("environmental_impact", weight=0.1177, norm_min=0, norm_max=5e-4, unit="kg/m", lower_is_better=True),
|
||||
],
|
||||
constraints=[DomainConstraint("medium", ["ground", "air"])],
|
||||
)
|
||||
@@ -839,12 +853,11 @@ def load_transport_seed(repo) -> dict:
|
||||
counts["domains"] += 1
|
||||
except sqlite3.IntegrityError:
|
||||
pass
|
||||
# Backfill metric units and lower_is_better on existing DBs.
|
||||
for mb in domain.metric_bounds:
|
||||
repo.ensure_metric(mb.metric_name, unit=mb.unit)
|
||||
repo.backfill_metric_unit(domain.name, mb.metric_name, mb.unit)
|
||||
if mb.lower_is_better:
|
||||
repo.backfill_lower_is_better(domain.name, mb.metric_name)
|
||||
# Sync domain_metric_weights to exactly match this domain's current
|
||||
# metric_bounds on existing DBs -- upserts weight/norm_min/norm_max/
|
||||
# unit for current metrics and removes any that were dropped (e.g.
|
||||
# safety/availability no longer scored).
|
||||
repo.sync_domain_metric_weights(domain)
|
||||
# Backfill domain constraints
|
||||
repo.replace_domain_constraints(domain)
|
||||
|
||||
|
||||
@@ -4,11 +4,25 @@ from __future__ import annotations
|
||||
|
||||
from flask import Blueprint, flash, redirect, render_template, request, url_for
|
||||
|
||||
from physcom.engine.constraint_resolver import ConstraintResolver
|
||||
from physcom.engine.pipeline import Pipeline
|
||||
from physcom.engine.scorer import Scorer
|
||||
from physcom_web.app import get_repo
|
||||
|
||||
bp = Blueprint("results", __name__, url_prefix="/results")
|
||||
|
||||
|
||||
def _run_evaluate(repo, domain, combo, platform_mass=None, actuator_mass=None, storage_mass=None):
|
||||
"""Purely exploratory -- never writes to the DB. Returns None if this
|
||||
combo has no free mass allocation to explore (see
|
||||
Pipeline.evaluate_allocation's docstring)."""
|
||||
pipeline = Pipeline(repo, ConstraintResolver(), Scorer(domain))
|
||||
return pipeline.evaluate_allocation(
|
||||
combo, domain,
|
||||
platform_mass=platform_mass, actuator_mass=actuator_mass, storage_mass=storage_mass,
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/")
|
||||
def results_index():
|
||||
repo = get_repo()
|
||||
@@ -25,9 +39,11 @@ def results_domain(domain_name: str):
|
||||
return redirect(url_for("results.results_index"))
|
||||
|
||||
status_filter = request.args.get("status")
|
||||
results = repo.get_all_results(domain_name, status=status_filter)
|
||||
rating_filter = request.args.get("rating")
|
||||
results = repo.get_all_results(domain_name, status=status_filter, rating=rating_filter)
|
||||
# Domain-scoped status counts (only combos that have results in this domain)
|
||||
statuses = repo.count_combinations_by_status(domain_name=domain_name)
|
||||
ratings = repo.count_results_by_rating(domain_name)
|
||||
|
||||
return render_template(
|
||||
"results/list.html",
|
||||
@@ -35,7 +51,9 @@ def results_domain(domain_name: str):
|
||||
domain=domain,
|
||||
results=results,
|
||||
status_filter=status_filter,
|
||||
rating_filter=rating_filter,
|
||||
statuses=statuses,
|
||||
ratings=ratings,
|
||||
total_results=sum(statuses.values()),
|
||||
)
|
||||
|
||||
@@ -58,6 +76,7 @@ def result_detail(domain_name: str, combo_id: int):
|
||||
flash("No results for this combination in this domain.", "error")
|
||||
return redirect(url_for("results.results_domain", domain_name=domain_name))
|
||||
scores = repo.get_combination_scores(combo_id, domain.id)
|
||||
explore_result = _run_evaluate(repo, domain, combo)
|
||||
|
||||
return render_template(
|
||||
"results/detail.html",
|
||||
@@ -65,6 +84,40 @@ def result_detail(domain_name: str, combo_id: int):
|
||||
combo=combo,
|
||||
result=result,
|
||||
scores=scores,
|
||||
explore_result=explore_result,
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/<domain_name>/<int:combo_id>/explore", methods=["POST"])
|
||||
def explore(domain_name: str, combo_id: int):
|
||||
"""Live, purely exploratory re-evaluation for an explicit platform/
|
||||
actuator/storage mass choice -- never touches stored data. Returns an
|
||||
HTMX partial."""
|
||||
repo = get_repo()
|
||||
domain = repo.get_domain(domain_name)
|
||||
combo = repo.get_combination(combo_id) if domain else None
|
||||
if not domain or not combo:
|
||||
return "", 404
|
||||
|
||||
def _mass(field: str) -> float | None:
|
||||
raw = request.form.get(field)
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return float(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
explore_result = _run_evaluate(
|
||||
repo, domain, combo,
|
||||
platform_mass=_mass("platform_mass"),
|
||||
actuator_mass=_mass("actuator_mass"),
|
||||
storage_mass=_mass("storage_mass"),
|
||||
)
|
||||
return render_template(
|
||||
"results/_explore_result.html",
|
||||
domain=domain,
|
||||
explore_result=explore_result,
|
||||
)
|
||||
|
||||
|
||||
@@ -101,6 +154,7 @@ def submit_review(domain_name: str, combo_id: int):
|
||||
novelty_flag=novelty_flag,
|
||||
llm_review=existing.get("llm_review") if existing else None,
|
||||
human_notes=human_notes,
|
||||
qualitative_rating=existing.get("qualitative_rating") if existing else None,
|
||||
)
|
||||
repo.update_combination_status(combo_id, "reviewed")
|
||||
|
||||
|
||||
@@ -214,6 +214,9 @@ table.compact th, table.compact td { padding: 0.25rem 0.4rem; font-size: 0.83rem
|
||||
.badge-llm_reviewed { background: rgba(107,163,160,0.12); color: var(--accent-teal); border-color: rgba(107,163,160,0.25); }
|
||||
.badge-reviewed { background: rgba(155,142,196,0.12); color: var(--accent-violet); border-color: rgba(155,142,196,0.25); }
|
||||
.badge-pending { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); }
|
||||
.badge-rating-low { background: rgba(184,92,92,0.12); color: var(--accent-red); border-color: rgba(184,92,92,0.25); }
|
||||
.badge-rating-medium { background: rgba(184,147,92,0.12); color: var(--accent-amber); border-color: rgba(184,147,92,0.25); }
|
||||
.badge-rating-high { background: rgba(122,171,138,0.12); color: var(--accent-green); border-color: rgba(122,171,138,0.25); }
|
||||
|
||||
/* ── Buttons ─────────────────────────────────────────────── */
|
||||
.btn {
|
||||
@@ -461,6 +464,52 @@ dd { font-size: 0.9rem; color: var(--text-primary); }
|
||||
margin-left: 0.3rem;
|
||||
}
|
||||
|
||||
/* ── Mass allocation bar (optimizer) ───────────────────────── */
|
||||
.mass-bar-container {
|
||||
display: flex;
|
||||
width: 100%;
|
||||
height: 18px;
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
border: 1px solid var(--border-subtle);
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
.mass-bar-seg { height: 100%; }
|
||||
.mass-bar-platform { background: var(--accent-blue); }
|
||||
.mass-bar-actuator { background: var(--accent-gold); }
|
||||
.mass-bar-storage { background: var(--accent-teal); }
|
||||
.mass-bar-legend {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.25rem 1rem;
|
||||
font-size: 0.8rem;
|
||||
color: var(--text-muted);
|
||||
margin-top: 0.4rem;
|
||||
align-items: center;
|
||||
}
|
||||
.mass-swatch {
|
||||
display: inline-block;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
border-radius: 2px;
|
||||
margin-right: 0.35rem;
|
||||
vertical-align: middle;
|
||||
}
|
||||
.optimize-summary { margin-bottom: 0.25rem; }
|
||||
.optimize-score { display: flex; flex-direction: column; gap: 0.1rem; }
|
||||
|
||||
/* ── Importance sliders (optimizer) ────────────────────────── */
|
||||
.weight-slider-row {
|
||||
display: grid;
|
||||
grid-template-columns: 140px 1fr 48px;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
.weight-slider-row label { font-size: 0.85rem; color: var(--text-muted); }
|
||||
.weight-slider-row output { font-size: 0.85rem; text-align: right; font-variant-numeric: tabular-nums; }
|
||||
.weight-slider-row input[type="range"] { width: 100%; }
|
||||
|
||||
/* ── Select dropdown dark styling ────────────────────────── */
|
||||
select option {
|
||||
background: var(--bg-surface);
|
||||
|
||||
@@ -50,12 +50,13 @@
|
||||
<div class="step-body">
|
||||
<h3>Physics Estimation</h3>
|
||||
<p>
|
||||
Surviving combinations get raw metric estimates — speed, cost,
|
||||
safety, range — via heuristic stubs or an LLM provider that
|
||||
reasons about the physical properties of each pairing.
|
||||
Surviving combinations get raw metric estimates — power
|
||||
density, cost, range — from a deterministic physics engine
|
||||
that sizes each combination from its own declared attributes, not
|
||||
a guess.
|
||||
</p>
|
||||
<div class="step-example">
|
||||
Bicycle + Human Pedalling → speed: 20 km/h, cost: $0.01/km
|
||||
Bicycle + Human Muscle → power density: 4.4 W/kg, range: 500km
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -72,8 +73,8 @@
|
||||
Combinations are ranked within their domain.
|
||||
</p>
|
||||
<div class="step-example">
|
||||
Domain <code>urban_commuting</code> weights: speed 25%, cost 25%,
|
||||
safety 25%, availability 15%, range 10%
|
||||
Domain <code>urban_commuting</code> weights: power density 42%,
|
||||
cost 42%, range 17%
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -85,9 +86,11 @@
|
||||
<div class="step-body">
|
||||
<h3>LLM Review</h3>
|
||||
<p>
|
||||
Top-scoring combinations are sent to a language model for plausibility
|
||||
and novelty assessment — catching physically valid but practically
|
||||
absurd pairings.
|
||||
Top-scoring combinations are sent to a language model for a
|
||||
plausibility verdict plus a holistic LOW/MEDIUM/HIGH rating —
|
||||
weighing safety and accessibility as qualitative judgment calls
|
||||
alongside the physics scores, catching physically valid but
|
||||
practically absurd pairings.
|
||||
</p>
|
||||
<div class="step-example">
|
||||
"Train + Solar Sail: structurally valid constraints, but solar radiation
|
||||
@@ -163,14 +166,15 @@
|
||||
<div class="card concept-card">
|
||||
<h3>Metrics</h3>
|
||||
<p>
|
||||
Quantitative axes like speed, cost, safety, and range. Each metric
|
||||
has a domain-specific weight and normalization range. Some are
|
||||
inverted — lower cost is better.
|
||||
Quantitative physics axes like power density, cost, and range. Each
|
||||
metric has a domain-specific weight and normalization range. Some
|
||||
are inverted — lower cost is better. Safety and accessibility
|
||||
are judgment calls, not physics quantities — they're weighed
|
||||
qualitatively in the LLM review pass instead of scored here.
|
||||
</p>
|
||||
<div class="concept-examples">
|
||||
<span class="badge">speed</span>
|
||||
<span class="badge">power_density</span>
|
||||
<span class="badge">cost_efficiency</span>
|
||||
<span class="badge">safety</span>
|
||||
<span class="badge">range_fuel</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
55
src/physcom_web/templates/results/_explore_result.html
Normal file
55
src/physcom_web/templates/results/_explore_result.html
Normal file
@@ -0,0 +1,55 @@
|
||||
{% if explore_result is none %}
|
||||
<p class="empty">No free mass allocation to explore for this combination — its
|
||||
actuator's mass isn't a design choice (a physiological or footprint-derived
|
||||
quantity), or the platform has no declared mass ceiling to bound the sliders.</p>
|
||||
{% else %}
|
||||
{% set r = explore_result %}
|
||||
<div class="optimize-summary">
|
||||
<div class="optimize-score">
|
||||
<span class="score-cell" style="font-size:1.4rem">{{ "%.4f"|format(r.composite_score) }}</span>
|
||||
<span class="subtitle">composite score at this build</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if r.exceeds_platform_envelope %}
|
||||
<p class="badge badge-p1_fail" style="display:inline-block;margin-bottom:0.75rem">
|
||||
⚠ total mass {{ "%.1f"|format(r.total_mass) }}kg exceeds this platform's declared ceiling
|
||||
({{ "%.1f"|format(r.platform_max) }}kg) — not a build this platform category could carry
|
||||
</p>
|
||||
{% endif %}
|
||||
{% if r.insufficient_structure %}
|
||||
<p class="badge badge-p1_fail" style="display:inline-block;margin-bottom:0.75rem">
|
||||
⚠ platform mass {{ "%.1f"|format(r.platform_mass) }}kg is too little structure to carry
|
||||
{{ "%.1f"|format(r.actuator_mass + r.storage_mass) }}kg of actuator+storage
|
||||
</p>
|
||||
{% endif %}
|
||||
|
||||
<div class="mass-bar-container" title="platform {{ '%.1f'|format(r.platform_mass) }}kg / actuator {{ '%.1f'|format(r.actuator_mass) }}kg / storage {{ '%.1f'|format(r.storage_mass) }}kg">
|
||||
{% set total = r.total_mass %}
|
||||
<div class="mass-bar-seg mass-bar-platform" style="width: {{ (r.platform_mass / total * 100)|round(1) }}%"></div>
|
||||
<div class="mass-bar-seg mass-bar-actuator" style="width: {{ (r.actuator_mass / total * 100)|round(1) }}%"></div>
|
||||
<div class="mass-bar-seg mass-bar-storage" style="width: {{ (r.storage_mass / total * 100)|round(1) }}%"></div>
|
||||
</div>
|
||||
<div class="mass-bar-legend">
|
||||
<span><span class="mass-swatch mass-bar-platform"></span>platform {{ "%.1f"|format(r.platform_mass) }}kg</span>
|
||||
<span><span class="mass-swatch mass-bar-actuator"></span>actuator {{ "%.1f"|format(r.actuator_mass) }}kg</span>
|
||||
<span><span class="mass-swatch mass-bar-storage"></span>storage {{ "%.1f"|format(r.storage_mass) }}kg</span>
|
||||
<span class="subtitle">{{ "%.1f"|format(r.total_mass) }}kg total</span>
|
||||
</div>
|
||||
|
||||
<table class="compact" style="margin-top:0.75rem">
|
||||
<thead><tr><th>Metric</th><th>Raw Value</th><th>Normalized</th><th>Weight</th></tr></thead>
|
||||
<tbody>
|
||||
{% for mb in domain.metric_bounds %}
|
||||
{% set val = r.raw_metrics.get(mb.metric_name) %}
|
||||
{% set n = r.normalized_scores.get(mb.metric_name) %}
|
||||
<tr>
|
||||
<td>{{ mb.metric_name }}</td>
|
||||
<td class="score-cell">{{ val|qty(mb.unit) if val is not none else '—' }}</td>
|
||||
<td class="score-cell">{{ "%.4f"|format(n) if n is not none else '—' }}</td>
|
||||
<td>{{ "%.0f%%"|format(mb.weight * 100) }}{{ ' ↓' if mb.lower_is_better else '' }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% endif %}
|
||||
@@ -27,6 +27,9 @@
|
||||
{% if result %}
|
||||
<dt>Composite Score</dt><dd class="score-cell">{{ "%.4f"|format(result.composite_score) }}</dd>
|
||||
<dt>Pass Reached</dt><dd>{{ result.pass_reached }}</dd>
|
||||
{% if result.qualitative_rating %}
|
||||
<dt>Rating</dt><dd><span class="badge badge-rating-{{ result.qualitative_rating|lower }}">{{ result.qualitative_rating }}</span></dd>
|
||||
{% endif %}
|
||||
{% if result.novelty_flag %}
|
||||
<dt>Novelty</dt><dd>{{ result.novelty_flag }}</dd>
|
||||
{% endif %}
|
||||
@@ -102,11 +105,16 @@
|
||||
{%- elif s.raw_value >= mb.norm_max -%}
|
||||
<span class="badge badge-{{ 'p1_fail' if mb.lower_is_better else 'valid' }}">at/above max{{ ' (worst)' if mb.lower_is_better else '' }}</span>
|
||||
{%- else -%}
|
||||
{% set pct = ((s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100) | int %}
|
||||
{% set raw_pct = (s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100 %}
|
||||
{# For lower_is_better metrics, raw_pct alone measures distance from norm_min,
|
||||
not quality -- a value near norm_min (excellent, cost near its floor) would
|
||||
otherwise render as a near-empty bar. Invert so the bar and percentage always
|
||||
mean "how good", matching the normalized score's own higher-is-better convention. #}
|
||||
{% set pct = ((100 - raw_pct) if mb.lower_is_better else raw_pct) | int %}
|
||||
<div class="metric-bar-container">
|
||||
<div class="metric-bar" style="width: {{ pct }}%"></div>
|
||||
</div>
|
||||
<span class="metric-bar-label">~{{ pct }}%{{ ' ↓' if mb.lower_is_better else '' }}</span>
|
||||
<span class="metric-bar-label">~{{ pct }}%{{ ' (lower is better)' if mb.lower_is_better else '' }}</span>
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
—
|
||||
@@ -121,6 +129,51 @@
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if explore_result is not none %}
|
||||
<h2>Explore: Scale the Build</h2>
|
||||
<p class="subtitle">
|
||||
Purely exploratory — nothing here is saved. Drag a slider to pick a
|
||||
platform weight class, motor size, or battery size directly, and see how
|
||||
power density, range, and the resulting score respond. Sliders open on
|
||||
the saved build above, which is already the score-optimized allocation
|
||||
for this domain (subject to the platform's physical performance floor),
|
||||
so the starting point is the best build already found, not an arbitrary
|
||||
or merely functional one.
|
||||
</p>
|
||||
<div class="card">
|
||||
{% set r = explore_result %}
|
||||
<form id="explore-form"
|
||||
hx-post="{{ url_for('results.explore', domain_name=domain.name, combo_id=combo.id) }}"
|
||||
hx-trigger="input changed delay:200ms"
|
||||
hx-target="#explore-result" hx-swap="innerHTML">
|
||||
<div class="weight-slider-row">
|
||||
<label for="platform_mass">platform (weight class)</label>
|
||||
<input type="range" min="{{ r.platform_min }}" max="{{ r.platform_max }}" step="0.1"
|
||||
id="platform_mass" name="platform_mass" value="{{ r.platform_mass }}"
|
||||
oninput="document.getElementById('out_platform_mass').textContent = (+this.value).toFixed(1) + 'kg'">
|
||||
<output id="out_platform_mass">{{ "%.1f"|format(r.platform_mass) }}kg</output>
|
||||
</div>
|
||||
<div class="weight-slider-row">
|
||||
<label for="actuator_mass">actuator (motor size)</label>
|
||||
<input type="range" min="{{ r.actuator_min }}" max="{{ r.actuator_slider_max }}" step="0.1"
|
||||
id="actuator_mass" name="actuator_mass" value="{{ r.actuator_mass }}"
|
||||
oninput="document.getElementById('out_actuator_mass').textContent = (+this.value).toFixed(1) + 'kg'">
|
||||
<output id="out_actuator_mass">{{ "%.1f"|format(r.actuator_mass) }}kg</output>
|
||||
</div>
|
||||
<div class="weight-slider-row">
|
||||
<label for="storage_mass">storage (battery/tank size)</label>
|
||||
<input type="range" min="{{ r.storage_min }}" max="{{ r.storage_slider_max }}" step="0.1"
|
||||
id="storage_mass" name="storage_mass" value="{{ r.storage_mass }}"
|
||||
oninput="document.getElementById('out_storage_mass').textContent = (+this.value).toFixed(1) + 'kg'">
|
||||
<output id="out_storage_mass">{{ "%.1f"|format(r.storage_mass) }}kg</output>
|
||||
</div>
|
||||
</form>
|
||||
<div id="explore-result">
|
||||
{% include "results/_explore_result.html" %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<h2>Human Review</h2>
|
||||
<div id="review-section">
|
||||
{% include "results/_review_form.html" %}
|
||||
|
||||
@@ -26,11 +26,11 @@
|
||||
|
||||
{% if statuses %}
|
||||
<div class="filter-row">
|
||||
<span>Filter:</span>
|
||||
<a href="{{ url_for('results.results_domain', domain_name=domain.name) }}"
|
||||
<span>Status:</span>
|
||||
<a href="{{ url_for('results.results_domain', domain_name=domain.name, rating=rating_filter) }}"
|
||||
class="btn btn-sm {{ '' if status_filter else 'btn-primary' }}">All ({{ total_results }})</a>
|
||||
{% for s, cnt in statuses.items() %}
|
||||
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=s) }}"
|
||||
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=s, rating=rating_filter) }}"
|
||||
class="btn btn-sm {{ 'btn-primary' if status_filter == s else '' }}">
|
||||
{{ s }} ({{ cnt }})
|
||||
</a>
|
||||
@@ -38,9 +38,25 @@
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if ratings %}
|
||||
<div class="filter-row">
|
||||
<span>Rating:</span>
|
||||
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=status_filter) }}"
|
||||
class="btn btn-sm {{ '' if not rating_filter else 'btn-primary' }}">All</a>
|
||||
{% for rt in ['HIGH', 'MEDIUM', 'LOW'] %}
|
||||
{% if rt in ratings %}
|
||||
<a href="{{ url_for('results.results_domain', domain_name=domain.name, status=status_filter, rating=rt) }}"
|
||||
class="btn btn-sm {{ 'btn-primary' if rating_filter == rt else '' }}">
|
||||
{{ rt }} ({{ ratings[rt] }})
|
||||
</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if not results %}
|
||||
{% if status_filter %}
|
||||
<p class="empty">No results with status "{{ status_filter }}" in this domain.</p>
|
||||
{% if status_filter or rating_filter %}
|
||||
<p class="empty">No results matching that filter in this domain.</p>
|
||||
{% else %}
|
||||
<p class="empty">No results for this domain yet. <a href="{{ url_for('pipeline.pipeline_form') }}">Run the pipeline</a> first.</p>
|
||||
{% endif %}
|
||||
@@ -52,6 +68,7 @@
|
||||
<th>Score</th>
|
||||
<th>Entities</th>
|
||||
<th>Status</th>
|
||||
<th>Rating</th>
|
||||
<th>Details</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
@@ -69,6 +86,13 @@
|
||||
<span class="badge badge-{{ r.combination.status }}">{{ r.combination.status }}</span>
|
||||
{%- endif -%}
|
||||
</td>
|
||||
<td>
|
||||
{%- if r.qualitative_rating -%}
|
||||
<span class="badge badge-rating-{{ r.qualitative_rating|lower }}">{{ r.qualitative_rating }}</span>
|
||||
{%- else -%}
|
||||
—
|
||||
{%- endif -%}
|
||||
</td>
|
||||
<td class="block-reason-cell">
|
||||
{%- if r.domain_block_reason -%}
|
||||
{{ r.domain_block_reason }}
|
||||
|
||||
@@ -69,6 +69,12 @@ def test_blocked_combos_not_scored(seeded_repo):
|
||||
score_threshold=0.0, passes=[1, 2, 3, 5],
|
||||
)
|
||||
|
||||
# Estimated count should be less than total (blocked ones filtered)
|
||||
# Estimated count should be less than total (blocked ones filtered).
|
||||
# Not necessarily equal to pass1_valid + pass1_conditional: a combo can
|
||||
# pass pass 1's entity-declared-floor checks but still turn out
|
||||
# structurally infeasible once pass 2 solves the domain-specific
|
||||
# actuator/storage requirement (e.g. an engine too big to fit its own
|
||||
# platform's declared mass ceiling) -- that's a legitimate per-domain
|
||||
# block, not a bug (see Pipeline._decide_masses' `feasible` return).
|
||||
assert result.pass2_estimated < result.total_generated
|
||||
assert result.pass2_estimated == result.pass1_valid + result.pass1_conditional
|
||||
assert result.pass2_estimated <= result.pass1_valid + result.pass1_conditional
|
||||
|
||||
@@ -335,28 +335,34 @@ def test_p3_fail_below_threshold(seeded_repo):
|
||||
|
||||
|
||||
def test_p4_fail_implausible(seeded_repo):
|
||||
"""Combos deemed implausible by LLM should get p4_fail status."""
|
||||
"""Combos deemed implausible by LLM should get p4_fail status.
|
||||
|
||||
Pass 2 is estimator-only now (never calls the LLM), so there's no way
|
||||
to force every combo's raw estimates toward a controlled low/high value
|
||||
the way MockLLMProvider's default_estimates used to. Force the pass-4
|
||||
verdict directly instead -- what's under test here is pipeline.py's
|
||||
wiring of review_plausibility's return value to status/counters, not
|
||||
MockLLMProvider's avg-based heuristic.
|
||||
"""
|
||||
from physcom.llm.providers.mock import MockLLMProvider
|
||||
|
||||
class AlwaysImplausibleLLM(MockLLMProvider):
|
||||
def review_plausibility(self, description, raw_metrics, normalized_scores, domain):
|
||||
return ("Always implausible for testing.", False)
|
||||
|
||||
repo = seeded_repo
|
||||
domain = repo.get_domain("urban_commuting")
|
||||
|
||||
resolver = ConstraintResolver()
|
||||
scorer = Scorer(domain)
|
||||
# Low estimates → normalized scores avg <= 0.5 → MockLLMProvider returns (text, False)
|
||||
# Use threshold=0.0 so no combo gets p3_fail and all reach pass 4
|
||||
mock_llm = MockLLMProvider(default_estimates={
|
||||
"power_density": 0.1, "cost_efficiency": 0.1, "safety": 0.1,
|
||||
"availability": 0.1, "range_fuel": 0.1,
|
||||
})
|
||||
pipeline = Pipeline(repo, resolver, scorer, llm=mock_llm)
|
||||
pipeline = Pipeline(repo, resolver, scorer, llm=AlwaysImplausibleLLM())
|
||||
|
||||
result = pipeline.run(
|
||||
domain, ["platform", "actuator", "energy_storage"],
|
||||
score_threshold=0.0, passes=[1, 2, 3, 4],
|
||||
)
|
||||
|
||||
# With low normalized scores (avg <= 0.5), reviewed combos should be p4_fail
|
||||
assert result.pass4_failed > 0
|
||||
assert result.pass4_reviewed == 0
|
||||
|
||||
@@ -367,20 +373,23 @@ def test_p4_fail_implausible(seeded_repo):
|
||||
|
||||
|
||||
def test_p4_pass_plausible(seeded_repo):
|
||||
"""Combos deemed plausible by LLM should get llm_reviewed status."""
|
||||
"""Combos deemed plausible by LLM should get llm_reviewed status.
|
||||
|
||||
See test_p4_fail_implausible on why the verdict is forced directly
|
||||
rather than via controlled pass-2 estimates.
|
||||
"""
|
||||
from physcom.llm.providers.mock import MockLLMProvider
|
||||
|
||||
class AlwaysPlausibleLLM(MockLLMProvider):
|
||||
def review_plausibility(self, description, raw_metrics, normalized_scores, domain):
|
||||
return ("Always plausible for testing.", True)
|
||||
|
||||
repo = seeded_repo
|
||||
domain = repo.get_domain("urban_commuting")
|
||||
|
||||
resolver = ConstraintResolver()
|
||||
scorer = Scorer(domain)
|
||||
# High estimates → avg > 0.5 → MockLLMProvider returns (text, True)
|
||||
mock_llm = MockLLMProvider(default_estimates={
|
||||
"power_density": 500.0, "cost_efficiency": 5e-4, "safety": 0.6,
|
||||
"availability": 0.7, "range_fuel": 200000.0,
|
||||
})
|
||||
pipeline = Pipeline(repo, resolver, scorer, llm=mock_llm)
|
||||
pipeline = Pipeline(repo, resolver, scorer, llm=AlwaysPlausibleLLM())
|
||||
|
||||
result = pipeline.run(
|
||||
domain, ["platform", "actuator", "energy_storage"],
|
||||
|
||||
Reference in New Issue
Block a user