physicalCombinatorics/src/physcom/llm/prompts.py

"""Prompt templates for LLM-assisted passes."""

from __future__ import annotations

from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from physcom.models.domain import MetricBound


def format_metrics_for_prompt(metrics: list["MetricBound"]) -> str:
    """Render each metric with its unit and expected range, so the model
    anchors on the right order of magnitude instead of a generic decimal."""
    lines = []
    for mb in metrics:
        unit = mb.unit or "dimensionless"
        lines.append(
            f"- {mb.metric_name} ({unit}): typical range {mb.norm_min:g} to {mb.norm_max:g}"
        )
    return "\n".join(lines)


def format_scores_for_prompt(
    raw_metrics: dict[str, float],
    normalized_scores: dict[str, float],
    metrics: list["MetricBound"],
) -> str:
    """Render each metric with BOTH its raw physical value and its
    normalized score, so the reviewing pass can reason from the actual
    physics instead of only ever seeing a compressed 0-1 number.

    A real, correct estimate can still look damning once log-normalized
    against a scale built for a different kind of vehicle (a cyclist's
    real ~5 W/kg reads as "0.159" next to a car's 2000 W/kg ceiling) --
    a reviewer that only sees the 0.159 has no way to notice that. See
    the labeled-set calibration note on PLAUSIBILITY_REVIEW_PROMPT below.
    """
    lines = []
    for mb in metrics:
        normed = normalized_scores.get(mb.metric_name)
        if normed is None:
            continue
        raw = raw_metrics.get(mb.metric_name)
        unit = mb.unit or "dimensionless"
        raw_str = f"{raw:g} {unit}" if raw is not None else "unknown"
        lines.append(
            f"- {mb.metric_name}: raw estimate {raw_str} — normalized score {normed:.3f}"
        )
    return "\n".join(lines)


PHYSICS_ESTIMATION_PROMPT = """\
You are a physics estimation assistant. Given the following transportation concept, \
estimate the requested metrics using order-of-magnitude physics reasoning.

## Concept
{description}

## Metrics to estimate
Each metric's unit and the typical range values fall in for this domain are given —
match that magnitude, don't guess a generically "reasonable-looking" decimal.
{metrics}

## Instructions
- Use real-world physics to estimate each metric, in the exact unit given.
- For "safety" specifically: consider hazards that arise from THIS combination's
  specific interactions — a fuel that's safe in an open vehicle can be far more
  dangerous inside a sealed tube or enclosed structure, a stable actuator on a
  fragile platform can be a real risk even if neither is risky alone. Don't just
  rate how safe the platform or actuator would be in isolation.
- If the concept is implausible, still provide your best estimate.
- Return ONLY valid JSON mapping metric names to numeric values, e.g.
  {{"some_metric": <number>, "another_metric": <number>}} — no explanatory text.
"""

# ponytail: pass 4 used to see only pass 2's normalized scores, not the raw
# physical numbers or any reasoning behind them. Fixed the raw-value half of
# that gap: format_scores_for_prompt() now shows both, since a correct raw
# estimate can look damning once log-normalized against a scale built for a
# different kind of vehicle (a cyclist's real ~5 W/kg reads as "0.159" next
# to a car's 2000 W/kg ceiling) -- gemma2:27b did exactly this on a real
# bicycle combo, citing "extremely low power density (0.159)" as grounds for
# IMPLAUSIBLE while never reasoning from the actual (correct) 5 W/kg. The
# reasoning-text half of the gap is still open: estimate_physics() doesn't
# return a per-metric rationale, so pass 4 still can't see WHY pass 2 landed
# on a number, only what the number is. Upgrade path if the raw value alone
# isn't enough in practice: have estimate_physics() also return a short
# per-metric reason, persist it alongside raw_value (new nullable column),
# and feed it into this prompt. Deferred because it needs a schema/interface
# change across LLMProvider + both providers + pipeline + scorer +
# repository, and more generated tokens per combo.
#
# If we plan to LLM-review every p2 pass then maybe p2 and p4 should be combined.
#
# Calibration history: the original wording asked the model to weigh "novelty"
# and "genuinely interesting innovation or nonsense" as part of the verdict,
# which measurably biased weaker/harsher models toward IMPLAUSIBLE on ordinary,
# working concepts just for being unoriginal (gemma2:27b scored 4/8 on a
# labeled test set, wrongly rejecting an ordinary commuter car). Rewritten to:
# separate "interesting" from "viable" entirely, state explicitly that scores
# are 0-1 where higher is always better (models were misreading a normalized
# 0.85 as a bad sign), require a *specific, named* mechanism for IMPLAUSIBLE
# rather than a vague "seems risky," and lower the bar from "must be proven
# physically impossible" to "a well-reasoned specific danger is enough" (the
# original wording let a careful reasoner argue its way out of flagging a
# genuinely hazardous combo on a technicality). Deliberately does NOT ask the
# model to weigh whether the concept or something like it already exists in
# the real world — that's a bias on physical/safety judgment, not a physics
# argument, and it papers over rather than fixes weak per-metric reasoning.
# Post-fix labeled-set accuracy: phi4 8/8, qwq 8/8, gemma2:27b 4/8->7/8,
# qwen2.5:7b 5/8->4/8 (a real capability ceiling on this model for this
# judgment task, not a prompt regression -- left as-is rather than chasing
# further prompt-specific patches for one weak model).
PLAUSIBILITY_REVIEW_PROMPT = """\
You are reviewing a transportation concept for real-world viability — could this
actually be built and operated safely. Whether it is new, exciting, or original
is NOT the question.

## Concept
{description}

## Metric Scores
Each metric below is given as its raw estimated physical value (in the unit
shown) AND a normalized score from 0-1, where HIGHER IS ALWAYS BETTER for
every metric listed regardless of what it measures (this already accounts
for things like "lower cost is better" — you don't need to invert anything).
A score of 1.0 means excellent, not "pegged" or "maxed out badly."

Reason from the RAW value first — it's the actual physics. The normalized
score is a summary, not a fact on its own: a real, correct estimate can
still normalize to a low-looking number simply because the domain's scale
was built for a different, more demanding kind of vehicle (a cyclist's real
~5 W/kg legitimately normalizes to ~0.16 next to a car engine's 2000 W/kg
ceiling — that low score doesn't mean the estimate is bad or the concept is
weak, it means human power is small next to a car engine, which everyone
already knows). If a normalized score looks alarming, check whether the raw
value is actually reasonable for what this component fundamentally is
before treating the score as evidence of a problem.

{scores}

## What makes something IMPLAUSIBLE
Mark IMPLAUSIBLE if either of these is true:
- It is physically or engineering-wise impossible given the components
  described, OR
- Combining these SPECIFIC components creates a serious, specific danger
  that goes beyond what either component already carries on its own —
  this does NOT require proof of outright impossibility, a well-reasoned,
  specific, serious danger is enough (e.g. repeated explosive recoil
  fatiguing a hull over time is a real structural risk, not just "explosives
  are dangerous in general"; a fuel that's fine in the open becoming
  concentrated in a sealed tube is a real risk, not just "fuel is
  flammable"). A low given safety score is a signal the pipeline already
  found something concerning — treat it as evidence, not noise to explain
  away.
- Or: a real regulatory/infrastructure barrier with no plausible workaround.

None of these make something implausible on their own:
- being unoriginal or something like it already exists
- being expensive, slow, or short-range
- a single mediocre score on one metric that isn't safety-related

Most concepts that reach this review are ordinary and workable; reserve
IMPLAUSIBLE for a real, specific problem you can name — but don't require
airtight proof of impossibility when the danger is already clear and specific.
Reason from the physics and engineering actually described here, not from
whether something like it already exists — novelty or lack of it is not
evidence either way.

## What to write
In 2-4 sentences, give your reasoning, then check it against the scores
above: if your reasoning conflicts with a score (e.g. you believe this is
hazardous but its safety score is high), name the metric and say so
explicitly — don't silently contradict a given score.

Finish with exactly one line:
VERDICT: PLAUSIBLE
or
VERDICT: IMPLAUSIBLE
"""