Add async pipeline with progress monitoring, resumability, and result transparency

Pipeline engine rewritten with combo-first loop: each combination is processed
through all requested passes before moving to the next, with incremental DB
saves after every step (crash-safe). Blocked combos now get result rows so they
appear in the results page with constraint violation reasons.

New pipeline_runs table tracks run lifecycle (pending/running/completed/failed/
cancelled). Web route launches pipeline in a background thread with its own DB
connection. HTMX polling partial shows live progress with per-pass breakdown.

Also: status guard prevents reviewed->scored downgrade, save_combination loads
existing status on dedup for correct resume, per-metric scores show domain
bounds + units + position bars, ensure_metric backfills units on existing rows.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/physcom_web/templates/domains/list.html

@@ -7,14 +7,13 @@
 {% if not domains %}
 <p class="empty">No domains found. Seed data via CLI first.</p>
 {% else %}
-<div class="card-grid">
   {% for d in domains %}
   <div class="card">
     <h2>{{ d.name }}</h2>
     <p>{{ d.description }}</p>
     <table>
       <thead>
-        <tr><th>Metric</th><th>Weight</th><th>Min</th><th>Max</th></tr>
+        <tr><th>Metric</th><th>Weight</th><th>Norm Min</th><th>Norm Max</th></tr>
       </thead>
       <tbody>
         {% for mb in d.metric_bounds %}
@@ -29,6 +28,5 @@
     </table>
   </div>
   {% endfor %}
-</div>
 {% endif %}
 {% endblock %}

src/physcom_web/templates/entities/detail.html

@@ -3,21 +3,29 @@
 
 {% block content %}
 <div class="page-header">
-  <h1>{{ entity.name }}</h1>
-  <div>
-    <a href="{{ url_for('entities.entity_edit', entity_id=entity.id) }}" class="btn">Edit</a>
-    <form method="post" action="{{ url_for('entities.entity_delete', entity_id=entity.id) }}" class="inline-form"
-          onsubmit="return confirm('Delete this entity?')">
-      <button type="submit" class="btn btn-danger">Delete</button>
-    </form>
-  </div>
+  <h1>{{ entity.name }} <span class="subtitle">{{ entity.dimension }}</span></h1>
+  <form method="post" action="{{ url_for('entities.entity_delete', entity_id=entity.id) }}" class="inline-form"
+        onsubmit="return confirm('Delete this entity and all its dependencies?')">
+    <button type="submit" class="btn btn-danger">Delete Entity</button>
+  </form>
 </div>
 
 <div class="card">
-  <dl>
-    <dt>Dimension</dt><dd>{{ entity.dimension }}</dd>
-    <dt>Description</dt><dd>{{ entity.description or '—' }}</dd>
-  </dl>
+  <form method="post" action="{{ url_for('entities.entity_detail', entity_id=entity.id) }}">
+    <div class="form-row">
+      <div class="form-group" style="flex:1">
+        <label for="name">Name</label>
+        <input type="text" id="name" name="name" value="{{ entity.name }}" required>
+      </div>
+      <div class="form-group" style="flex:2">
+        <label for="description">Description</label>
+        <input type="text" id="description" name="description" value="{{ entity.description }}">
+      </div>
+      <div class="form-group" style="align-self:end">
+        <button type="submit" class="btn btn-primary">Save</button>
+      </div>
+    </div>
+  </form>
 </div>
 
 <h2>Dependencies</h2>

src/physcom_web/templates/pipeline/_run_status.html

@@ -0,0 +1,78 @@
+{# HTMX partial: live status for a single pipeline run #}
+<div class="run-status run-status-{{ run.status }}"
+     {% if run.status == 'running' or run.status == 'pending' %}
+     hx-get="{{ url_for('pipeline.run_status', run_id=run.id) }}"
+     hx-trigger="every 2s"
+     hx-swap="outerHTML"
+     {% endif %}>
+
+  <div class="run-status-header">
+    <span class="badge badge-{{ run.status }}">{{ run.status }}</span>
+    <span class="run-status-label">Run #{{ run.id }}</span>
+    {% if run.current_pass %}
+    <span class="subtitle">Processing pass {{ run.current_pass }}</span>
+    {% endif %}
+  </div>
+
+  {% if run.total_combos and run.total_combos > 0 %}
+  {% set done = run.combos_pass1 or 0 %}
+  {% set pct = (done / run.total_combos * 100) | int %}
+  <div class="progress-bar-container">
+    <div class="progress-bar" style="width: {{ pct }}%"></div>
+  </div>
+  <div class="run-status-counters">
+    <span>{{ done }} / {{ run.total_combos }} combos processed</span>
+  </div>
+  <table class="compact" style="margin-top:0.35rem">
+    <thead>
+      <tr><th>Pass</th><th>Result</th></tr>
+    </thead>
+    <tbody>
+      {% if (run.combos_pass1 or 0) > 0 %}
+      {% set valid = (run.combos_pass1 or 0) - (run.total_combos - (run.combos_pass2 or 0)) if (run.combos_pass2 or 0) > 0 else (run.combos_pass1 or 0) %}
+      <tr>
+        <td>1 — Constraints</td>
+        <td>{{ run.combos_pass1 or 0 }} checked
+          {%- if (run.combos_pass2 or 0) > 0 and (run.combos_pass1 or 0) > (run.combos_pass2 or 0) %},
+            <span class="badge badge-blocked">{{ (run.combos_pass1 or 0) - (run.combos_pass2 or 0) }} blocked</span>
+          {%- endif -%}
+        </td>
+      </tr>
+      {% endif %}
+      {% if (run.combos_pass2 or 0) > 0 %}
+      <tr>
+        <td>2 — Estimation</td>
+        <td>{{ run.combos_pass2 or 0 }} estimated</td>
+      </tr>
+      {% endif %}
+      {% if (run.combos_pass3 or 0) > 0 %}
+      <tr>
+        <td>3 — Scoring</td>
+        <td>{{ run.combos_pass3 or 0 }} scored</td>
+      </tr>
+      {% endif %}
+      {% if (run.combos_pass4 or 0) > 0 %}
+      <tr>
+        <td>4 — LLM Review</td>
+        <td>{{ run.combos_pass4 or 0 }} reviewed</td>
+      </tr>
+      {% endif %}
+    </tbody>
+  </table>
+  {% endif %}
+
+  {% if run.error_message %}
+  <div class="flash flash-error" style="margin-top:0.5rem">{{ run.error_message }}</div>
+  {% endif %}
+
+  <div class="run-status-actions">
+    {% if run.status == 'running' %}
+    <form method="post" action="{{ url_for('pipeline.run_cancel', run_id=run.id) }}" class="inline-form">
+      <button type="submit" class="btn btn-danger btn-sm">Cancel</button>
+    </form>
+    {% endif %}
+    {% if run.status == 'completed' %}
+    <a href="{{ url_for('results.results_index') }}" class="btn btn-sm">View results</a>
+    {% endif %}
+  </div>
+</div>

src/physcom_web/templates/pipeline/run.html

@@ -8,6 +8,7 @@
   <form method="post" action="{{ url_for('pipeline.pipeline_run') }}">
     <div class="form-group">
       <label for="domain">Domain</label>
+      <p class="form-hint">The evaluation context that defines which metrics matter and how they're weighted.</p>
       <select name="domain" id="domain" required>
         <option value="">— select —</option>
         {% for d in domains %}
@@ -18,30 +19,45 @@
 
     <fieldset>
       <legend>Passes</legend>
-      <div class="checkbox-row">
-        {% for p in [1, 2, 3, 4, 5] %}
+      <p class="form-hint">Each pass progressively filters and enriches combinations. Later passes depend on earlier ones.</p>
+      <div class="checkbox-col">
         <label>
-          <input type="checkbox" name="passes" value="{{ p }}"
-                 {{ 'checked' if p <= 3 }}>
-          Pass {{ p }}
-          {% if p == 1 %}(Constraints)
-          {% elif p == 2 %}(Estimation)
-          {% elif p == 3 %}(Scoring)
-          {% elif p == 4 %}(LLM Review)
-          {% elif p == 5 %}(Human Review)
-          {% endif %}
+          <input type="checkbox" name="passes" value="1" checked>
+          <strong>Pass 1 — Constraint Resolution</strong>
+          <span class="form-hint">Checks requires/provides/excludes compatibility between entities. Blocks impossible combinations.</span>
+        </label>
+        <label>
+          <input type="checkbox" name="passes" value="2" checked>
+          <strong>Pass 2 — Physics Estimation</strong>
+          <span class="form-hint">Estimates raw metric values (speed, cost, etc.) using heuristics or an LLM. Without an LLM provider, uses a force/mass stub.</span>
+        </label>
+        <label>
+          <input type="checkbox" name="passes" value="3" checked>
+          <strong>Pass 3 — Scoring &amp; Ranking</strong>
+          <span class="form-hint">Normalizes estimates against domain bounds and computes a weighted geometric mean composite score.</span>
+        </label>
+        <label>
+          <input type="checkbox" name="passes" value="4">
+          <strong>Pass 4 — LLM Review</strong>
+          <span class="form-hint">Sends top combinations to an LLM for a plausibility and novelty assessment. Requires an LLM provider to be configured.</span>
+        </label>
+        <label>
+          <input type="checkbox" name="passes" value="5">
+          <strong>Pass 5 — Human Review</strong>
+          <span class="form-hint">Marks results as ready for human review on the Results page.</span>
         </label>
-        {% endfor %}
       </div>
     </fieldset>
 
     <div class="form-group">
       <label for="threshold">Score Threshold</label>
+      <p class="form-hint">Minimum composite score (0–1) for a combination to pass scoring. Lower values keep more results; higher values are more selective.</p>
       <input type="number" name="threshold" id="threshold" value="0.1" step="0.01" min="0" max="1">
     </div>
 
     <fieldset>
       <legend>Dimensions</legend>
+      <p class="form-hint">Which entity dimensions to combine. The pipeline generates the Cartesian product of all entities in the selected dimensions.</p>
       <div class="checkbox-row">
         {% for d in dimensions %}
         <label>
@@ -57,4 +73,76 @@
     </div>
   </form>
 </div>
+
+{% set active_runs = runs | selectattr('status', 'in', ['pending', 'running']) | list %}
+{% if active_runs %}
+<h2>Active Runs</h2>
+{% for run in active_runs %}
+<div class="card"
+     hx-get="{{ url_for('pipeline.run_status', run_id=run.id) }}"
+     hx-trigger="every 2s"
+     hx-swap="innerHTML">
+  {% include "pipeline/_run_status.html" %}
+</div>
+{% endfor %}
+{% endif %}
+
+{% if runs %}
+<h2>Run History</h2>
+<table>
+  <thead>
+    <tr>
+      <th>ID</th>
+      <th>Domain</th>
+      <th>Status</th>
+      <th>Total</th>
+      <th>P1 Checked</th>
+      <th>P1 Blocked</th>
+      <th>P2 Estimated</th>
+      <th>P3 Scored</th>
+      <th>P4 Reviewed</th>
+      <th>Started</th>
+    </tr>
+  </thead>
+  <tbody>
+    {% for run in runs %}
+    {% set blocked = (run.combos_pass1 or 0) - (run.combos_pass2 or 0) if (run.combos_pass2 or 0) > 0 and (run.combos_pass1 or 0) > (run.combos_pass2 or 0) else 0 %}
+    <tr>
+      <td>{{ run.id }}</td>
+      <td>{{ run.domain_name }}</td>
+      <td><span class="badge badge-{{ run.status }}">{{ run.status }}</span></td>
+      <td>{{ run.total_combos or '—' }}</td>
+      <td>{{ run.combos_pass1 or '—' }}</td>
+      <td>{% if blocked %}<span class="badge badge-blocked">{{ blocked }}</span>{% else %}—{% endif %}</td>
+      <td>{{ run.combos_pass2 or '—' }}</td>
+      <td>{{ run.combos_pass3 or '—' }}</td>
+      <td>{{ run.combos_pass4 or '—' }}</td>
+      <td>{{ run.started_at or run.created_at }}</td>
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
+{% endif %}
+
+{% if summaries.values()|select|list %}
+<h2>Domain Summaries</h2>
+{% for d in domains %}
+{% set s = summaries[d.name] %}
+{% if s %}
+<div class="card">
+  <h3>{{ d.name }} <span class="subtitle">{{ d.description }}</span></h3>
+  <dl class="summary-dl">
+    <dt>Results</dt><dd>{{ s.total_results }} scored combinations</dd>
+    <dt>Blocked</dt><dd>{{ s.blocked }} combinations</dd>
+    <dt>Score range</dt><dd class="score-cell">{{ "%.4f"|format(s.min_score) }} — {{ "%.4f"|format(s.max_score) }}</dd>
+    <dt>Avg score</dt><dd class="score-cell">{{ "%.4f"|format(s.avg_score) }}</dd>
+    <dt>Last pass</dt><dd>{{ s.last_pass }}</dd>
+  </dl>
+  <div style="margin-top:0.5rem">
+    <a href="{{ url_for('results.results_domain', domain_name=d.name) }}" class="btn btn-sm">View results</a>
+  </div>
+</div>
+{% endif %}
+{% endfor %}
+{% endif %}
 {% endblock %}

src/physcom_web/templates/results/detail.html

@@ -55,19 +55,55 @@
 
 {% if scores %}
 <h2>Per-Metric Scores</h2>
+{% set bounds = {} %}
+{% for mb in domain.metric_bounds %}
+  {% set _ = bounds.update({mb.metric_name: mb}) %}
+{% endfor %}
 <div class="card">
   <table>
     <thead>
-      <tr><th>Metric</th><th>Raw Value</th><th>Normalized</th><th>Method</th><th>Confidence</th></tr>
+      <tr>
+        <th>Metric</th>
+        <th>Raw Value</th>
+        <th>Domain Range</th>
+        <th>Position</th>
+        <th>Normalized</th>
+        <th>Weight</th>
+      </tr>
     </thead>
     <tbody>
       {% for s in scores %}
+      {% set mb = bounds.get(s.metric_name) %}
       <tr>
         <td>{{ s.metric_name }}</td>
-        <td>{{ "%.2f"|format(s.raw_value) if s.raw_value is not none else '—' }}</td>
+        {% set unit = s.metric_unit or '' %}
+        <td class="score-cell">{{ "%.2f"|format(s.raw_value) if s.raw_value is not none else '—' }}{{ ' ' + unit if unit and s.raw_value is not none else '' }}</td>
+        <td>
+          {%- if mb -%}
+            {{ "%.2f"|format(mb.norm_min) }} — {{ "%.2f"|format(mb.norm_max) }}{{ ' ' + unit if unit else '' }}
+          {%- else -%}
+            —
+          {%- endif -%}
+        </td>
+        <td>
+          {%- if mb and s.raw_value is not none -%}
+            {%- if s.raw_value <= mb.norm_min -%}
+              <span class="badge badge-blocked">at/below min</span>
+            {%- elif s.raw_value >= mb.norm_max -%}
+              <span class="badge badge-valid">at/above max</span>
+            {%- else -%}
+              {% set pct = ((s.raw_value - mb.norm_min) / (mb.norm_max - mb.norm_min) * 100) | int %}
+              <div class="metric-bar-container">
+                <div class="metric-bar" style="width: {{ pct }}%"></div>
+              </div>
+              <span class="metric-bar-label">~{{ pct }}%</span>
+            {%- endif -%}
+          {%- else -%}
+            —
+          {%- endif -%}
+        </td>
         <td class="score-cell">{{ "%.4f"|format(s.normalized_score) if s.normalized_score is not none else '—' }}</td>
-        <td>{{ s.estimation_method or '—' }}</td>
-        <td>{{ "%.2f"|format(s.confidence) if s.confidence is not none else '—' }}</td>
+        <td>{{ "%.0f%%"|format(mb.weight * 100) if mb else '—' }}</td>
       </tr>
       {% endfor %}
     </tbody>

src/physcom_web/templates/results/list.html

@@ -21,7 +21,7 @@
   <div class="filter-row">
     <span>Filter:</span>
     <a href="{{ url_for('results.results_domain', domain_name=domain.name) }}"
-       class="btn btn-sm {{ '' if status_filter else 'btn-primary' }}">All</a>
+       class="btn btn-sm {{ '' if status_filter else 'btn-primary' }}">All ({{ total_results }})</a>
     {% for s, cnt in statuses.items() %}
     <a href="{{ url_for('results.results_domain', domain_name=domain.name, status=s) }}"
        class="btn btn-sm {{ 'btn-primary' if status_filter == s else '' }}">
@@ -32,7 +32,11 @@
   {% endif %}
 
   {% if not results %}
-  <p class="empty">No results yet. <a href="{{ url_for('pipeline.pipeline_form') }}">Run the pipeline</a> first.</p>
+    {% if status_filter %}
+    <p class="empty">No results with status "{{ status_filter }}" in this domain.</p>
+    {% else %}
+    <p class="empty">No results for this domain yet. <a href="{{ url_for('pipeline.pipeline_form') }}">Run the pipeline</a> first.</p>
+    {% endif %}
   {% else %}
   <table>
     <thead>
@@ -41,7 +45,7 @@
         <th>Score</th>
         <th>Entities</th>
         <th>Status</th>
-        <th>Novelty</th>
+        <th>Details</th>
         <th></th>
       </tr>
     </thead>
@@ -49,10 +53,18 @@
       {% for r in results %}
       <tr>
         <td>{{ loop.index }}</td>
-        <td class="score-cell">{{ "%.4f"|format(r.composite_score) }}</td>
+        <td class="score-cell">{{ "%.4f"|format(r.composite_score) if r.composite_score else '—' }}</td>
         <td>{{ r.combination.entities|map(attribute='name')|join(' + ') }}</td>
         <td><span class="badge badge-{{ r.combination.status }}">{{ r.combination.status }}</span></td>
-        <td>{{ r.novelty_flag or '—' }}</td>
+        <td class="block-reason-cell">
+          {%- if r.combination.status == 'blocked' and r.combination.block_reason -%}
+            {{ r.combination.block_reason }}
+          {%- elif r.novelty_flag -%}
+            {{ r.novelty_flag }}
+          {%- else -%}
+            —
+          {%- endif -%}
+        </td>
         <td>
           <a href="{{ url_for('results.result_detail', domain_name=domain.name, combo_id=r.combination.id) }}"
              class="btn btn-sm">View</a>