andrew/physicalCombinatorics
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

close guardrail gaps and fix the scoring pipeline top to bottom

Browse Source 
Constraint resolver: aggregate mass/footprint across a combo instead of
pairwise-only checks, treat medium/atmosphere as agreement not supply/demand,
reduce multi-provider checks by best/sum instead of AND-ing every provider,
fail closed on unrecognized mutex values, add a propulsion-viability
(thrust-to-weight) rule. Seed data updated to match (nuclear/solar-sail
footprint floors, water-medium exclusions, explicit ground/gravity providers).

Domain metric units were stored globally per metric name instead of
per-domain, silently corrupting cost_efficiency for every domain but the
first one seeded — fixed with a schema migration.

Stub estimator's cost_efficiency/safety/availability/reliability were a
backwards formula and flat constants; replaced with heuristics grounded in
each entity's thrust_profile/energy_form/infrastructure.

LLM estimate_physics() now receives each metric's unit and expected range
instead of a bare name, fixing wildly miscalibrated estimates traced back to
the prompt's own hardcoded example anchoring the model to the wrong order of
magnitude. Sharpened the safety-estimation and plausibility-review prompts.
Deduped provider parsing logic into llm/parsing.py.

Web pipeline form can now pick an LLM provider per run instead of only via
server env var.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Andrew Simonson
2026-07-26 00:13:16 -05:00
parent 63295ab80e
commit 434df718d7
18 changed files with 836 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/physcom/llm/base.py
View File

@@ -4,6 +4,8 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from physcom.models.domain import MetricBound
class LLMRateLimitError(Exception):
"""Raised by a provider when the API rate limit is exceeded.
@@ -22,10 +24,14 @@ class LLMProvider(ABC):
@abstractmethod
def estimate_physics(
self, combination_description: str, metrics: list[str]
self, combination_description: str, metrics: list[MetricBound]
) -> dict[str, float]:
"""Given a natural-language description of a combination,
estimate raw metric values. Returns {metric_name: estimated_value}."""
estimate raw metric values. `metrics` carries each metric's unit and
expected norm_min/norm_max so the estimate lands in the right
magnitude — a bare metric name gives no hint that "cost_efficiency"
means dollars per meter in the 1e-5 range, not a 0-1 score.
Returns {metric_name: estimated_value}."""
...
@abstractmethod

30
src/physcom/llm/parsing.py Normal file
View File

@@ -0,0 +1,30 @@
"""Shared response-parsing helpers for LLM providers."""
from __future__ import annotations
import json
import re
from physcom.models.domain import MetricBound
def parse_verdict(text: str) -> bool:
"""Extract VERDICT: PLAUSIBLE/IMPLAUSIBLE from response; default to True."""
m = re.search(r"VERDICT:\s*(PLAUSIBLE|IMPLAUSIBLE)", text, re.IGNORECASE)
if m:
return m.group(1).upper() == "PLAUSIBLE"
return True
def parse_metric_json(text: str, metrics: list[MetricBound]) -> dict[str, float]:
"""Strip markdown fences and parse JSON; fall back to each metric's own
norm_min/norm_max midpoint on error — a flat constant like 0.5 is
guaranteed wrong-magnitude for at least some metrics regardless of unit.
"""
names = {mb.metric_name for mb in metrics}
text = re.sub(r"```(?:json)?\s*", "", text).strip().rstrip("`").strip()
try:
data = json.loads(text)
return {k: float(v) for k, v in data.items() if k in names}
except (json.JSONDecodeError, ValueError, TypeError):
return {mb.metric_name: (mb.norm_min + mb.norm_max) / 2 for mb in metrics}

50
src/physcom/llm/prompts.py
View File

@@ -1,5 +1,25 @@
"""Prompt templates for LLM-assisted passes."""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from physcom.models.domain import MetricBound
def format_metrics_for_prompt(metrics: list["MetricBound"]) -> str:
"""Render each metric with its unit and expected range, so the model
anchors on the right order of magnitude instead of a generic decimal."""
lines = []
for mb in metrics:
unit = mb.unit or "dimensionless"
lines.append(
f"- {mb.metric_name} ({unit}): typical range {mb.norm_min:g} to {mb.norm_max:g}"
)
return "\n".join(lines)
PHYSICS_ESTIMATION_PROMPT = """\
You are a physics estimation assistant. Given the following transportation concept, \
estimate the requested metrics using order-of-magnitude physics reasoning.
@@ -8,15 +28,35 @@ estimate the requested metrics using order-of-magnitude physics reasoning.
{description}
## Metrics to estimate
Each metric's unit and the typical range values fall in for this domain are given —
match that magnitude, don't guess a generically "reasonable-looking" decimal.
{metrics}
## Instructions
- Use real-world physics to estimate each metric.
- Use real-world physics to estimate each metric, in the exact unit given.
- For "safety" specifically: consider hazards that arise from THIS combination's
specific interactions — a fuel that's safe in an open vehicle can be far more
dangerous inside a sealed tube or enclosed structure, a stable actuator on a
fragile platform can be a real risk even if neither is risky alone. Don't just
rate how safe the platform or actuator would be in isolation.
- If the concept is implausible, still provide your best estimate.
- Return ONLY valid JSON mapping metric names to numeric values.
- Example: {{"power_density": 500.0, "cost_efficiency": 0.15, "safety": 0.7}}
- Return ONLY valid JSON mapping metric names to numeric values, e.g.
{{"some_metric": <number>, "another_metric": <number>}} — no explanatory text.
"""
# ponytail: pass 4 only sees pass 2's raw numbers, not its reasoning. Sharpened
# prompts on both sides closed most of the gap (a bad safety estimate went from
# 0.95 to 0.80 on the same combo once pass 2 was told to consider combination-
# specific hazards), but pass 4 still doesn't reliably call out a contradiction
# by name when one remains — qwen2.5:7b doesn't follow that meta-instruction
# consistently. Upgrade path if this isn't good enough in practice: have
# estimate_physics() also return a short per-metric reason, persist it
# alongside raw_value (new nullable column), and feed it into this prompt so
# pass 4 has something concrete to agree or disagree with. Deferred because it
# needs a schema/interface change across LLMProvider + both providers +
# pipeline + scorer + repository, and more generated tokens per combo.
#
# If we plan to LLM-review every p2 pass then maybe p2 and p4 should be combined.
PLAUSIBILITY_REVIEW_PROMPT = """\
You are reviewing a novel transportation concept for social and practical viability.
@@ -32,6 +72,10 @@ Review this concept for:
2. Practical barriers — what engineering or regulatory obstacles exist?
3. Novelty — does anything similar already exist?
4. Overall plausibility — is this a genuinely interesting innovation or nonsense?
5. Consistency — if your assessment conflicts with any score above (e.g. you
consider this hazardous but its safety score is high), say so explicitly by
naming the metric and the discrepancy. Don't silently contradict a given
score in your reasoning without calling out that you're doing so.
Provide a concise 2-4 sentence assessment, then on a final line write exactly:
VERDICT: PLAUSIBLE

34
src/physcom/llm/providers/gemini.py
View File

@@ -2,12 +2,17 @@
from __future__ import annotations
import json
import re
import math
from physcom.llm.base import LLMProvider, LLMRateLimitError
from physcom.llm.prompts import PHYSICS_ESTIMATION_PROMPT, PLAUSIBILITY_REVIEW_PROMPT
from physcom.llm.parsing import parse_metric_json, parse_verdict
from physcom.llm.prompts import (
PHYSICS_ESTIMATION_PROMPT,
PLAUSIBILITY_REVIEW_PROMPT,
format_metrics_for_prompt,
)
from physcom.models.domain import MetricBound
class GeminiLLMProvider(LLMProvider):
@@ -24,11 +29,11 @@ class GeminiLLMProvider(LLMProvider):
self._model = model
def estimate_physics(
self, combination_description: str, metrics: list[str]
self, combination_description: str, metrics: list[MetricBound]
) -> dict[str, float]:
prompt = PHYSICS_ESTIMATION_PROMPT.format(
description=combination_description,
metrics=", ".join(metrics),
metrics=format_metrics_for_prompt(metrics),
)
try:
response = self._client.models.generate_content(
@@ -38,7 +43,7 @@ class GeminiLLMProvider(LLMProvider):
if "429" in str(exc) or "RESOURCE_EXHAUSTED" in str(exc):
raise LLMRateLimitError(str(exc), self._parse_retry_after(exc)) from exc
raise
return self._parse_json(response.text, metrics)
return parse_metric_json(response.text, metrics)
def review_plausibility(
self, combination_description: str, scores: dict[str, float]
@@ -57,26 +62,9 @@ class GeminiLLMProvider(LLMProvider):
raise LLMRateLimitError(str(exc), self._parse_retry_after(exc)) from exc
raise
text = response.text.strip()
plausible = self._parse_verdict(text)
return (text, plausible)
def _parse_verdict(self, text: str) -> bool:
"""Extract VERDICT: PLAUSIBLE/IMPLAUSIBLE from response; default to True."""
m = re.search(r"VERDICT:\s*(PLAUSIBLE|IMPLAUSIBLE)", text, re.IGNORECASE)
if m:
return m.group(1).upper() == "PLAUSIBLE"
return True
return (text, parse_verdict(text))
def _parse_retry_after(self, exc: Exception) -> int:
"""Extract retry delay from the error message, with a safe default."""
m = re.search(r"retry in (\d+(?:\.\d+)?)", str(exc))
return math.ceil(float(m.group(1))) + 5 if m else 65
def _parse_json(self, text: str, metrics: list[str]) -> dict[str, float]:
"""Strip markdown fences and parse JSON; fall back to 0.5 per metric on error."""
text = re.sub(r"```(?:json)?\s*", "", text).strip().rstrip("`").strip()
try:
data = json.loads(text)
return {k: float(v) for k, v in data.items() if k in metrics}
except (json.JSONDecodeError, ValueError, TypeError):
return {m: 0.5 for m in metrics}

7
src/physcom/llm/providers/mock.py
View File

@@ -3,6 +3,7 @@
from __future__ import annotations
from physcom.llm.base import LLMProvider
from physcom.models.domain import MetricBound
class MockLLMProvider(LLMProvider):
@@ -12,11 +13,11 @@ class MockLLMProvider(LLMProvider):
self._defaults = default_estimates or {}
def estimate_physics(
self, combination_description: str, metrics: list[str]
self, combination_description: str, metrics: list[MetricBound]
) -> dict[str, float]:
result = {}
for metric in metrics:
result[metric] = self._defaults.get(metric, 0.5)
for mb in metrics:
result[mb.metric_name] = self._defaults.get(mb.metric_name, 0.5)
return result
def review_plausibility(

33
src/physcom/llm/providers/ollama.py
View File

@@ -3,12 +3,17 @@
from __future__ import annotations
import json
import re
import urllib.error
import urllib.request
from physcom.llm.base import LLMProvider
from physcom.llm.prompts import PHYSICS_ESTIMATION_PROMPT, PLAUSIBILITY_REVIEW_PROMPT
from physcom.llm.parsing import parse_metric_json, parse_verdict
from physcom.llm.prompts import (
PHYSICS_ESTIMATION_PROMPT,
PLAUSIBILITY_REVIEW_PROMPT,
format_metrics_for_prompt,
)
from physcom.models.domain import MetricBound
class OllamaLLMProvider(LLMProvider):
@@ -19,14 +24,14 @@ class OllamaLLMProvider(LLMProvider):
self._host = host.rstrip("/")
def estimate_physics(
self, combination_description: str, metrics: list[str]
self, combination_description: str, metrics: list[MetricBound]
) -> dict[str, float]:
prompt = PHYSICS_ESTIMATION_PROMPT.format(
description=combination_description,
metrics=", ".join(metrics),
metrics=format_metrics_for_prompt(metrics),
)
text = self._generate(prompt, json_mode=True)
return self._parse_json(text, metrics)
return parse_metric_json(text, metrics)
def review_plausibility(
self, combination_description: str, scores: dict[str, float]
@@ -37,7 +42,7 @@ class OllamaLLMProvider(LLMProvider):
scores=scores_str,
)
text = self._generate(prompt, json_mode=False).strip()
return (text, self._parse_verdict(text))
return (text, parse_verdict(text))
def _generate(self, prompt: str, json_mode: bool) -> str:
payload = {"model": self._model, "prompt": prompt, "stream": False}
@@ -55,19 +60,3 @@ class OllamaLLMProvider(LLMProvider):
raise ConnectionError(
f"Could not reach Ollama at {self._host} (is `ollama serve` running?)"
) from exc
def _parse_verdict(self, text: str) -> bool:
"""Extract VERDICT: PLAUSIBLE/IMPLAUSIBLE from response; default to True."""
m = re.search(r"VERDICT:\s*(PLAUSIBLE|IMPLAUSIBLE)", text, re.IGNORECASE)
if m:
return m.group(1).upper() == "PLAUSIBLE"
return True
def _parse_json(self, text: str, metrics: list[str]) -> dict[str, float]:
"""Strip markdown fences and parse JSON; fall back to 0.5 per metric on error."""
text = re.sub(r"```(?:json)?\s*", "", text).strip().rstrip("`").strip()
try:
data = json.loads(text)
return {k: float(v) for k, v in data.items() if k in metrics}
except (json.JSONDecodeError, ValueError, TypeError):
return {m: 0.5 for m in metrics}

28
src/physcom/llm/registry.py
View File

@@ -7,32 +7,40 @@ import os
from physcom.llm.base import LLMProvider
def build_llm_provider() -> LLMProvider | None:
"""Return an LLMProvider based on env vars, or None if not configured.
def build_llm_provider(
provider: str | None = None,
model: str | None = None,
host: str | None = None,
) -> LLMProvider | None:
"""Return an LLMProvider, or None if not configured.
Explicit args (e.g. from a per-request web form) override env vars;
passing nothing falls back to the env-var-only behavior below.
LLM_PROVIDER — provider name ('gemini', 'ollama'; more can be added)
GEMINI_API_KEY — required when LLM_PROVIDER=gemini
GEMINI_API_KEY — required when provider is 'gemini' (server env only,
never accepted as a request param)
GEMINI_MODEL — optional Gemini model name (default: gemini-2.0-flash)
OLLAMA_MODEL — optional Ollama model name (default: qwen2.5:7b)
OLLAMA_HOST — optional Ollama server URL (default: http://localhost:11434)
"""
provider = os.environ.get("LLM_PROVIDER", "").lower().strip()
provider = (provider or os.environ.get("LLM_PROVIDER", "")).lower().strip()
if not provider:
if not provider or provider == "stub":
return None
if provider == "gemini":
api_key = os.environ.get("GEMINI_API_KEY", "")
if not api_key:
raise ValueError("LLM_PROVIDER=gemini requires GEMINI_API_KEY to be set")
model = os.environ.get("GEMINI_MODEL", "gemini-2.0-flash")
raise ValueError("Gemini requires GEMINI_API_KEY to be set in the server environment")
model = model or os.environ.get("GEMINI_MODEL", "gemini-2.0-flash")
from physcom.llm.providers.gemini import GeminiLLMProvider
return GeminiLLMProvider(api_key=api_key, model=model)
if provider == "ollama":
model = os.environ.get("OLLAMA_MODEL", "qwen2.5:7b")
host = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
model = model or os.environ.get("OLLAMA_MODEL", "qwen2.5:7b")
host = host or os.environ.get("OLLAMA_HOST", "http://localhost:11434")
from physcom.llm.providers.ollama import OllamaLLMProvider
return OllamaLLMProvider(model=model, host=host)
raise ValueError(f"Unknown LLM_PROVIDER: {provider!r}. Supported: gemini, ollama")
raise ValueError(f"Unknown LLM provider: {provider!r}. Supported: gemini, ollama, stub")