andrew/physicalCombinatorics
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QoL and metric value inverter

Browse Source
This commit is contained in:
Andrew Simonson
2026-03-04 11:10:45 -06:00
parent 8dfe3607b1
commit f57ac7d6dc
30 changed files with 556 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/physcom/cli.py
View File

@@ -146,13 +146,16 @@ def run(ctx, domain_name, passes, threshold, dimensions):
click.echo(f" Total combinations: {result.total_generated}")
click.echo(f" Pass 1 — valid: {result.pass1_valid}, "
f"conditional: {result.pass1_conditional}, "
f"blocked: {result.pass1_blocked}")
f"failed: {result.pass1_failed}")
if 2 in pass_list:
click.echo(f" Pass 2 — estimated: {result.pass2_estimated}")
click.echo(f" Pass 2 — estimated: {result.pass2_estimated}, "
f"failed: {result.pass2_failed}")
if 3 in pass_list:
click.echo(f" Pass 3 — above threshold: {result.pass3_above_threshold}")
click.echo(f" Pass 3 — above threshold: {result.pass3_above_threshold}, "
f"failed: {result.pass3_failed}")
if 4 in pass_list:
click.echo(f" Pass 4 — LLM reviewed: {result.pass4_reviewed}")
click.echo(f" Pass 4 — LLM reviewed: {result.pass4_reviewed}, "
f"failed: {result.pass4_failed}")
@main.command()

66
src/physcom/db/repository.py
View File

@@ -193,6 +193,17 @@ class Repository:
self.conn.commit()
return row["id"]
def backfill_lower_is_better(self, domain_name: str, metric_name: str) -> None:
"""Set lower_is_better=1 for an existing domain-metric row that still has the default 0."""
self.conn.execute(
"""UPDATE domain_metric_weights SET lower_is_better = 1
WHERE lower_is_better = 0
AND domain_id = (SELECT id FROM domains WHERE name = ?)
AND metric_id = (SELECT id FROM metrics WHERE name = ?)""",
(domain_name, metric_name),
)
self.conn.commit()
def add_domain(self, domain: Domain) -> Domain:
cur = self.conn.execute(
"INSERT INTO domains (name, description) VALUES (?, ?)",
@@ -204,9 +215,10 @@ class Repository:
mb.metric_id = metric_id
self.conn.execute(
"""INSERT INTO domain_metric_weights
(domain_id, metric_id, weight, norm_min, norm_max)
VALUES (?, ?, ?, ?, ?)""",
(domain.id, metric_id, mb.weight, mb.norm_min, mb.norm_max),
(domain_id, metric_id, weight, norm_min, norm_max, lower_is_better)
VALUES (?, ?, ?, ?, ?, ?)""",
(domain.id, metric_id, mb.weight, mb.norm_min, mb.norm_max,
int(mb.lower_is_better)),
)
self.conn.commit()
return domain
@@ -216,7 +228,8 @@ class Repository:
if not row:
return None
weights = self.conn.execute(
"""SELECT m.name, m.unit, dmw.weight, dmw.norm_min, dmw.norm_max, dmw.metric_id
"""SELECT m.name, m.unit, dmw.weight, dmw.norm_min, dmw.norm_max,
dmw.metric_id, dmw.lower_is_better
FROM domain_metric_weights dmw
JOIN metrics m ON dmw.metric_id = m.id
WHERE dmw.domain_id = ?""",
@@ -231,6 +244,7 @@ class Repository:
metric_name=w["name"], weight=w["weight"],
norm_min=w["norm_min"], norm_max=w["norm_max"],
metric_id=w["metric_id"], unit=w["unit"] or "",
lower_is_better=bool(w["lower_is_better"]),
)
for w in weights
],
@@ -245,7 +259,8 @@ class Repository:
if not row:
return None
weights = self.conn.execute(
"""SELECT m.name, m.unit, dmw.weight, dmw.norm_min, dmw.norm_max, dmw.metric_id
"""SELECT m.name, m.unit, dmw.weight, dmw.norm_min, dmw.norm_max,
dmw.metric_id, dmw.lower_is_better
FROM domain_metric_weights dmw
JOIN metrics m ON dmw.metric_id = m.id
WHERE dmw.domain_id = ?""",
@@ -260,6 +275,7 @@ class Repository:
metric_name=w["name"], weight=w["weight"],
norm_min=w["norm_min"], norm_max=w["norm_max"],
metric_id=w["metric_id"], unit=w["unit"] or "",
lower_is_better=bool(w["lower_is_better"]),
)
for w in weights
],
@@ -277,21 +293,23 @@ class Repository:
mb.metric_id = metric_id
self.conn.execute(
"""INSERT OR REPLACE INTO domain_metric_weights
(domain_id, metric_id, weight, norm_min, norm_max)
VALUES (?, ?, ?, ?, ?)""",
(domain_id, metric_id, mb.weight, mb.norm_min, mb.norm_max),
(domain_id, metric_id, weight, norm_min, norm_max, lower_is_better)
VALUES (?, ?, ?, ?, ?, ?)""",
(domain_id, metric_id, mb.weight, mb.norm_min, mb.norm_max,
int(mb.lower_is_better)),
)
self.conn.commit()
return mb
def update_metric_bound(
self, domain_id: int, metric_id: int, weight: float, norm_min: float, norm_max: float, unit: str
self, domain_id: int, metric_id: int, weight: float, norm_min: float, norm_max: float,
unit: str, lower_is_better: bool = False,
) -> None:
self.conn.execute(
"""UPDATE domain_metric_weights
SET weight = ?, norm_min = ?, norm_max = ?
SET weight = ?, norm_min = ?, norm_max = ?, lower_is_better = ?
WHERE domain_id = ? AND metric_id = ?""",
(weight, norm_min, norm_max, domain_id, metric_id),
(weight, norm_min, norm_max, int(lower_is_better), domain_id, metric_id),
)
if unit:
self.conn.execute(
@@ -330,6 +348,21 @@ class Repository:
self.conn.execute("DELETE FROM combination_scores WHERE domain_id = ?", (domain.id,))
self.conn.execute("DELETE FROM combination_results WHERE domain_id = ?", (domain.id,))
self.conn.execute("DELETE FROM pipeline_runs WHERE domain_id = ?", (domain.id,))
# Delete orphaned combos (no results left in any domain) and all their
# related rows — scores, entity links — so FK constraints don't block.
orphan_sql = """SELECT c.id FROM combinations c
WHERE c.id NOT IN (
SELECT DISTINCT combination_id FROM combination_results
)"""
self.conn.execute(
f"DELETE FROM combination_scores WHERE combination_id IN ({orphan_sql})"
)
self.conn.execute(
f"DELETE FROM combination_entities WHERE combination_id IN ({orphan_sql})"
)
self.conn.execute(
f"DELETE FROM combinations WHERE id IN ({orphan_sql})"
)
self.conn.commit()
return count
@@ -371,12 +404,15 @@ class Repository:
self, combo_id: int, status: str, block_reason: str | None = None
) -> None:
# Don't downgrade from higher pass states — preserves human/LLM review data
if status in ("scored", "llm_reviewed"):
if status in ("scored", "llm_reviewed") or status.endswith("_fail"):
row = self.conn.execute(
"SELECT status FROM combinations WHERE id = ?", (combo_id,)
).fetchone()
if row:
cur = row["status"]
# Fail statuses should not overwrite llm_reviewed or reviewed
if status.endswith("_fail") and cur in ("llm_reviewed", "reviewed"):
return
if status == "scored" and cur in ("llm_reviewed", "reviewed"):
return
if status == "llm_reviewed" and cur == "reviewed":
@@ -550,12 +586,12 @@ class Repository:
).fetchone()
if not row or row["total"] == 0:
return None
blocked = self.conn.execute(
failed = self.conn.execute(
"""SELECT COUNT(*) as cnt
FROM combinations c
JOIN combination_results cr ON cr.combination_id = c.id
JOIN domains d ON cr.domain_id = d.id
WHERE c.status = 'blocked' AND d.name = ?""",
WHERE c.status LIKE '%\\_fail' ESCAPE '\\' AND d.name = ?""",
(domain_name,),
).fetchone()
return {
@@ -564,7 +600,7 @@ class Repository:
"max_score": row["max_score"],
"min_score": row["min_score"],
"last_pass": row["last_pass"],
"blocked": blocked["cnt"] if blocked else 0,
"failed": failed["cnt"] if failed else 0,
}
def get_result(self, combo_id: int, domain_id: int) -> dict | None:

31
src/physcom/db/schema.py
View File

@@ -44,12 +44,13 @@ CREATE TABLE IF NOT EXISTS metrics (
);
CREATE TABLE IF NOT EXISTS domain_metric_weights (
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain_id INTEGER NOT NULL REFERENCES domains(id),
metric_id INTEGER NOT NULL REFERENCES metrics(id),
weight REAL NOT NULL,
norm_min REAL,
norm_max REAL,
id INTEGER PRIMARY KEY AUTOINCREMENT,
domain_id INTEGER NOT NULL REFERENCES domains(id),
metric_id INTEGER NOT NULL REFERENCES metrics(id),
weight REAL NOT NULL,
norm_min REAL,
norm_max REAL,
lower_is_better INTEGER NOT NULL DEFAULT 0,
UNIQUE(domain_id, metric_id)
);
@@ -117,6 +118,23 @@ CREATE INDEX IF NOT EXISTS idx_pipeline_runs_domain ON pipeline_runs(domain_id);
"""
def _migrate(conn: sqlite3.Connection) -> None:
"""Apply incremental migrations for existing databases."""
cols = {r[1] for r in conn.execute("PRAGMA table_info(domain_metric_weights)").fetchall()}
if "lower_is_better" not in cols:
conn.execute(
"ALTER TABLE domain_metric_weights ADD COLUMN lower_is_better INTEGER NOT NULL DEFAULT 0"
)
# Backfill: cost_efficiency is lower-is-better in all domains
conn.execute(
"""UPDATE domain_metric_weights SET lower_is_better = 1
WHERE lower_is_better = 0
AND metric_id IN (SELECT id FROM metrics WHERE name = 'cost_efficiency')"""
)
conn.commit()
def init_db(db_path: str | Path) -> sqlite3.Connection:
"""Create/open the database and ensure all tables exist."""
db_path = Path(db_path)
@@ -125,5 +143,6 @@ def init_db(db_path: str | Path) -> sqlite3.Connection:
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA foreign_keys=ON")
conn.executescript(DDL)
_migrate(conn)
conn.commit()
return conn

2
src/physcom/engine/constraint_resolver.py
View File

@@ -46,7 +46,7 @@ class ConstraintResolver:
self._check_unmet_requirements(all_deps, result)
if result.violations:
result.status = "blocked"
result.status = "p1_fail"
elif result.warnings:
result.status = "conditional"

119
src/physcom/engine/pipeline.py
View File

@@ -21,12 +21,15 @@ class PipelineResult:
total_generated: int = 0
pass1_valid: int = 0
pass1_blocked: int = 0
pass1_failed: int = 0
pass1_conditional: int = 0
pass2_estimated: int = 0
pass2_failed: int = 0
pass3_scored: int = 0
pass3_above_threshold: int = 0
pass3_failed: int = 0
pass4_reviewed: int = 0
pass4_failed: int = 0
pass5_human_reviewed: int = 0
top_results: list[dict] = field(default_factory=list)
@@ -77,7 +80,7 @@ class Pipeline:
run_id,
combos_pass1=result.pass1_valid
+ result.pass1_conditional
+ result.pass1_blocked,
+ result.pass1_failed,
combos_pass2=result.pass2_estimated,
combos_pass3=result.pass3_scored,
combos_pass4=result.pass4_reviewed,
@@ -142,22 +145,22 @@ class Pipeline:
# ── Pass 1: Constraint Resolution ────────────────
if 1 in passes and existing_pass < 1:
cr: ConstraintResult = self.resolver.resolve(combo)
if cr.status == "blocked":
combo.status = "blocked"
if cr.status == "p1_fail":
combo.status = "p1_fail"
combo.block_reason = "; ".join(cr.violations)
self.repo.update_combination_status(
combo.id, "blocked", combo.block_reason
combo.id, "p1_fail", combo.block_reason
)
# Save a result row so blocked combos appear in results
# Save a result row so failed combos appear in results
self.repo.save_result(
combo.id,
domain.id,
composite_score=0.0,
pass_reached=1,
)
result.pass1_blocked += 1
result.pass1_failed += 1
self._update_run_counters(run_id, result, current_pass=1)
continue # blocked — skip remaining passes
continue # p1_fail — skip remaining passes
else:
combo.status = "valid"
self.repo.update_combination_status(combo.id, "valid")
@@ -168,16 +171,16 @@ class Pipeline:
self._update_run_counters(run_id, result, current_pass=1)
elif 1 in passes:
# Already pass1'd — check if it was blocked
if combo.status == "blocked":
result.pass1_blocked += 1
# Already pass1'd — check if it failed
if combo.status.endswith("_fail"):
result.pass1_failed += 1
continue
else:
result.pass1_valid += 1
else:
# Pass 1 not requested; check if blocked from a prior run
if combo.status == "blocked":
result.pass1_blocked += 1
# Pass 1 not requested; check if failed from a prior run
if combo.status.endswith("_fail"):
result.pass1_failed += 1
continue
# ── Pass 2: Physics Estimation ───────────────────
@@ -207,6 +210,21 @@ class Pipeline:
combo.id, domain.id, estimate_dicts
)
# Check for all-zero estimates → p2_fail
if raw_metrics and all(v == 0.0 for v in raw_metrics.values()):
combo.status = "p2_fail"
combo.block_reason = "All metric estimates are zero"
self.repo.update_combination_status(
combo.id, "p2_fail", combo.block_reason
)
self.repo.save_result(
combo.id, domain.id,
composite_score=0.0, pass_reached=2,
)
result.pass2_failed += 1
self._update_run_counters(run_id, result, current_pass=2)
continue
result.pass2_estimated += 1
self._update_run_counters(run_id, result, current_pass=2)
elif 2 in passes:
@@ -249,6 +267,26 @@ class Pipeline:
existing_result["human_notes"] if existing_result else None
)
if sr.composite_score < score_threshold:
self.repo.save_result(
combo.id, domain.id,
sr.composite_score, pass_reached=3,
novelty_flag=novelty_flag,
human_notes=human_notes,
)
combo.status = "p3_fail"
combo.block_reason = (
f"Composite score {sr.composite_score:.4f} "
f"below threshold {score_threshold}"
)
self.repo.update_combination_status(
combo.id, "p3_fail", combo.block_reason
)
result.pass3_failed += 1
result.pass3_scored += 1
self._update_run_counters(run_id, result, current_pass=3)
continue
self.repo.save_result(
combo.id,
domain.id,
@@ -260,8 +298,7 @@ class Pipeline:
self.repo.update_combination_status(combo.id, "scored")
result.pass3_scored += 1
if sr.composite_score >= score_threshold:
result.pass3_above_threshold += 1
result.pass3_above_threshold += 1
self._update_run_counters(run_id, result, current_pass=3)
elif 3 in passes and existing_pass >= 3:
@@ -294,33 +331,49 @@ class Pipeline:
for s in db_scores
if s["normalized_score"] is not None
}
review: str | None = None
review_result: tuple[str, bool] | None = None
try:
review = self.llm.review_plausibility(
review_result = self.llm.review_plausibility(
description, score_dict
)
except LLMRateLimitError as exc:
self._wait_for_rate_limit(run_id, exc.retry_after)
try:
review = self.llm.review_plausibility(
review_result = self.llm.review_plausibility(
description, score_dict
)
except LLMRateLimitError:
pass # still limited; skip, retry next run
if review is not None:
self.repo.save_result(
combo.id,
domain.id,
cur_result["composite_score"],
pass_reached=4,
novelty_flag=cur_result.get("novelty_flag"),
llm_review=review,
human_notes=cur_result.get("human_notes"),
)
self.repo.update_combination_status(
combo.id, "llm_reviewed"
)
result.pass4_reviewed += 1
if review_result is not None:
review_text, plausible = review_result
if not plausible:
self.repo.save_result(
combo.id, domain.id,
cur_result["composite_score"],
pass_reached=4,
novelty_flag=cur_result.get("novelty_flag"),
llm_review=review_text,
human_notes=cur_result.get("human_notes"),
)
combo.status = "p4_fail"
combo.block_reason = "LLM deemed implausible"
self.repo.update_combination_status(
combo.id, "p4_fail", combo.block_reason
)
result.pass4_failed += 1
else:
self.repo.save_result(
combo.id, domain.id,
cur_result["composite_score"],
pass_reached=4,
novelty_flag=cur_result.get("novelty_flag"),
llm_review=review_text,
human_notes=cur_result.get("human_notes"),
)
self.repo.update_combination_status(
combo.id, "llm_reviewed"
)
result.pass4_reviewed += 1
self._update_run_counters(
run_id, result, current_pass=4
)

2
src/physcom/engine/scorer.py
View File

@@ -69,6 +69,8 @@ class Scorer:
for mb in self.domain.metric_bounds:
raw = raw_metrics.get(mb.metric_name, 0.0)
normed = normalize(raw, mb.norm_min, mb.norm_max)
if mb.lower_is_better:
normed = 1.0 - normed
scores.append(Score(
metric_name=mb.metric_name,
raw_value=raw,

6
src/physcom/llm/base.py
View File

@@ -31,7 +31,7 @@ class LLMProvider(ABC):
@abstractmethod
def review_plausibility(
self, combination_description: str, scores: dict[str, float]
) -> str:
"""Given a combination and its scores, return a natural-language
plausibility and novelty assessment."""
) -> tuple[str, bool]:
"""Given a combination and its scores, return a (text, is_plausible)
tuple: natural-language assessment and whether the concept is plausible."""
...

5
src/physcom/llm/prompts.py
View File

@@ -33,5 +33,8 @@ Review this concept for:
3. Novelty — does anything similar already exist?
4. Overall plausibility — is this a genuinely interesting innovation or nonsense?
Provide a concise 2-4 sentence assessment.
Provide a concise 2-4 sentence assessment, then on a final line write exactly:
VERDICT: PLAUSIBLE
or
VERDICT: IMPLAUSIBLE
"""

13
src/physcom/llm/providers/gemini.py
View File

@@ -42,7 +42,7 @@ class GeminiLLMProvider(LLMProvider):
def review_plausibility(
self, combination_description: str, scores: dict[str, float]
) -> str:
) -> tuple[str, bool]:
scores_str = "\n".join(f"- {k}: {v:.3f}" for k, v in scores.items())
prompt = PLAUSIBILITY_REVIEW_PROMPT.format(
description=combination_description,
@@ -56,7 +56,16 @@ class GeminiLLMProvider(LLMProvider):
if "429" in str(exc) or "RESOURCE_EXHAUSTED" in str(exc):
raise LLMRateLimitError(str(exc), self._parse_retry_after(exc)) from exc
raise
return response.text.strip()
text = response.text.strip()
plausible = self._parse_verdict(text)
return (text, plausible)
def _parse_verdict(self, text: str) -> bool:
"""Extract VERDICT: PLAUSIBLE/IMPLAUSIBLE from response; default to True."""
m = re.search(r"VERDICT:\s*(PLAUSIBLE|IMPLAUSIBLE)", text, re.IGNORECASE)
if m:
return m.group(1).upper() == "PLAUSIBLE"
return True
def _parse_retry_after(self, exc: Exception) -> int:
"""Extract retry delay from the error message, with a safe default."""

6
src/physcom/llm/providers/mock.py
View File

@@ -21,8 +21,8 @@ class MockLLMProvider(LLMProvider):
def review_plausibility(
self, combination_description: str, scores: dict[str, float]
) -> str:
) -> tuple[str, bool]:
avg = sum(scores.values()) / max(len(scores), 1)
if avg > 0.5:
return "This concept appears plausible and worth further investigation."
return "This concept has significant feasibility challenges."
return ("This concept appears plausible and worth further investigation.", True)
return ("This concept has significant feasibility challenges.", False)

2
src/physcom/models/combination.py
View File

@@ -39,7 +39,7 @@ class Combination:
"""A generated combination of entities (one per dimension)."""
entities: list[Entity] = field(default_factory=list)
status: str = "pending" # pending → valid/blocked → scored → reviewed
status: str = "pending" # pending → valid/p1_fail/p2_fail/p3_fail/p4_fail → scored → reviewed
block_reason: str | None = None
hash: str | None = None
id: int | None = None

5
src/physcom/models/domain.py
View File

@@ -11,9 +11,10 @@ class MetricBound:
metric_name: str
weight: float # 0.0–1.0
norm_min: float # Below this → score 0
norm_max: float # Above this → score 1
norm_min: float # Below this → score 0 (or 1 if lower_is_better)
norm_max: float # Above this → score 1 (or 0 if lower_is_better)
unit: str = ""
lower_is_better: bool = False # Invert scale (e.g., cost: lower = better)
metric_id: int | None = None

8
src/physcom/seed/transport_example.py
View File

@@ -254,7 +254,7 @@ URBAN_COMMUTING = Domain(
description="Daily travel within a city, 1-50km range",
metric_bounds=[
MetricBound("speed", weight=0.25, norm_min=5, norm_max=120, unit="km/h"),
MetricBound("cost_efficiency", weight=0.25, norm_min=0.01, norm_max=2.0, unit="$/km"),
MetricBound("cost_efficiency", weight=0.25, norm_min=0.01, norm_max=2.0, unit="$/km", lower_is_better=True),
MetricBound("safety", weight=0.25, norm_min=0.0, norm_max=1.0, unit="0-1"),
MetricBound("availability", weight=0.15, norm_min=0.0, norm_max=1.0, unit="0-1"),
MetricBound("range_fuel", weight=0.10, norm_min=5, norm_max=500, unit="km"),
@@ -268,7 +268,7 @@ INTERPLANETARY = Domain(
MetricBound("speed", weight=0.30, norm_min=1000, norm_max=300000, unit="km/s"),
MetricBound("range_fuel", weight=0.30, norm_min=1e6, norm_max=1e10, unit="km"),
MetricBound("safety", weight=0.20, norm_min=0.0, norm_max=1.0, unit="0-1"),
MetricBound("cost_efficiency", weight=0.10, norm_min=1e3, norm_max=1e9, unit="$/km"),
MetricBound("cost_efficiency", weight=0.10, norm_min=1e3, norm_max=1e9, unit="$/km", lower_is_better=True),
MetricBound("range_degradation", weight=0.10, norm_min=100, norm_max=36500, unit="days"),
],
)
@@ -302,8 +302,10 @@ def load_transport_seed(repo) -> dict:
counts["domains"] += 1
except sqlite3.IntegrityError:
pass
# Backfill metric units on existing DBs (ensure_metric is idempotent).
# Backfill metric units and lower_is_better on existing DBs.
for mb in domain.metric_bounds:
repo.ensure_metric(mb.metric_name, unit=mb.unit)
if mb.lower_is_better:
repo.backfill_lower_is_better(domain.name, mb.metric_name)
return counts