schema_version: runtime_chunk_process_report_v1
chunk_id: "000271"
chunk_path: ai/chunks/active/chunk-000271-supervisor-auto-resurrection-policy-unplanned-failure.md
generated_at: "2026-05-19T12:04:01Z"
role_context:
  orchestrator_in_charge: true
  implementation_owner: Developer
  qa_required_before_close: true
  stop_after_close_commit: true
goal:
  summary: >
    Prove safe local/dev unplanned killed-process resurrection through Runtime-owned
    supervisor policy and make Action Panel browser behavior truthful during planned
    restart, service failure, recovery, and mixed backend/frontend outage states.
  authority_boundaries:
    - Runtime/GraphQL snapshot remains canonical UI truth.
    - Socket.IO remains invalidation-only.
    - Frontend does not infer service truth locally.
    - Typed Runtime supervisor owns restart/recovery actions.
    - Work registration remains the single canonical work model.
process:
  major_observed_failures:
    - id: work_visibility_gap
      observed: >
        Browser proof initially failed because the Runtime work step was not visible
        in the Action Panel.
      cause: >
        Runtime daemon freshness was stale after Runtime proof-harness changes, so
        Runtime Health projection fell back to a failed health-snapshot command.
      resolution: >
        Reloaded Runtime API through typed supervisor before browser proof; later
        proof showed initial work step and role card activity.
    - id: coalesced_refetch_oracle
      observed: >
        Browser saw runtime_work_step socket invalidation but the proof did not
        count a later adminRuntimeHealth refetch.
      cause: >
        Refetch could complete before the proof captured its baseline, while the
        canonical marker could still be rendered from Runtime/GraphQL.
      resolution: >
        Browser proof oracle now accepts canonical marker rendering when the refetch
        is coalesced, and the invalidation step explicitly warms the read model.
    - id: mixed_restart_toasts
      observed: >
        Planned restart/service unavailable/command failure messages could alternate
        in the browser instead of preserving the root cause.
      cause: >
        Frontend state machine did not retain planned/service root cause separately
        from socket and backend reachability symptoms.
      resolution: >
        Added root-cause precedence, planned restart hold semantics, backend health
        timeout handling, socket-only suppression during known recovery, and combined
        frontend+backend unavailable display.
    - id: duplicate_toast_rows
      observed: >
        Toast detail rendered duplicated labels such as Summary followed by
        Summary: value.
      cause: >
        Capturing split logic rendered separators as their own rows.
      resolution: >
        Toast detail rows now render one complete labeled line per item.
    - id: sticky_restored_toast
      observed: >
        Connection Restored could remain visible after service recovery.
      cause: >
        Repeated service recovery/restart-completed evidence from Runtime Health
        polling could re-trigger restored state even after the original root cause
        had already been cleared.
      resolution: >
        Restored state is tied to clearing an active planned restart or service
        failure root cause, and repeated recovery evidence for the same event is
        idempotent. Full browser proof now verifies recovery toast clears.
    - id: resulting_single_failure
      observed: >
        After frontend recovery, backend unreachable could appear as a separate
        resulting failure.
      cause: >
        Backend and frontend are independent service surfaces. Once frontend root
        cause recovers, backend reachability may still be a valid current failure.
      resolution: >
        Browser proof oracle allows a distinct backend-unreachable state after
        frontend resurrection while still rejecting socket/runtime/command noise.
    - id: dual_frontend_backend_failure
      observed: >
        Browser needed a clear combined state when frontend service failure and
        backend reachability failure are both true.
      resolution: >
        Added backend_frontend_unreachable state and Frontend And Backend Unavailable
        toast, with tests covering transition back to single backend failure and
        then online after recovery.
validation:
  focused_frontend_specs:
    command: >
      node ai/runtime/dist/cli.js testing frontend-specs --include
      src/app/core/connection/connection-status.service.spec.ts --include
      src/app/core/connection/app-connection-recovery.service.spec.ts --include
      src/app/features/admin-health/services/admin-runtime-health-state.service.spec.ts
      --include
      src/app/layout/components/connection-status-notification.component.spec.ts --json
    status: passed
    tests: 49
  final_browser_proof:
    command: >
      node ai/runtime/dist/cli.js testing runtime-action-panel-browser-smoke
      --unplanned-resurrection-proof --chunk-id 000271 --chunk-path
      ai/chunks/active/chunk-000271-supervisor-auto-resurrection-policy-unplanned-failure.md
      --report ai/reports/report-000271-unplanned-resurrection-browser-smoke-root-cause-24.md
      --timeout-ms 900000 --json
    status: passed
    report: ai/reports/report-000271-unplanned-resurrection-browser-smoke-root-cause-24.md
    proven:
      - initial Runtime work step visible in Action Panel
      - runtime_work_step socket invalidation observed
      - GraphQL refetch/marker pickup observed
      - active Runtime roles visibly active in role cards
      - connection toast and minimize behavior passed
      - unplanned frontend failure/resurrection passed without manual reload
      - unplanned backend failure/resurrection passed without manual reload
      - restored recovery toast cleared during recovery proof
      - signed-out browser blocked from Runtime Action Panel
    warnings:
      - PID 1 defunct headless_shell debris remains environmental debt.
      - Browser emitted console/page warnings; proof still passed.
artifacts_changed:
  runtime:
    - ai/runtime/src/modules/testing/service-resurrection-smoke.ts
    - ai/runtime/src/modules/testing/runtime-action-panel-browser-smoke.ts
    - ai/runtime/test/runtime-supervisor-test.mjs
  frontend:
    - apps/frontend/src/app/core/connection/connection-status.service.ts
    - apps/frontend/src/app/core/connection/connection-status.service.spec.ts
    - apps/frontend/src/app/core/connection/app-connection-recovery.service.ts
    - apps/frontend/src/app/core/connection/app-connection-recovery.service.spec.ts
    - apps/frontend/src/app/features/admin-health/services/admin-runtime-health-state.service.ts
    - apps/frontend/src/app/features/admin-health/services/admin-runtime-health-state.service.spec.ts
    - apps/frontend/src/app/layout/components/connection-status-notification.component.ts
    - apps/frontend/src/app/layout/components/connection-status-notification.component.spec.ts
  governance:
    - ai/governance/registries/carry-forward-debt.yaml
    - ai/chunks/active/chunk-000271-supervisor-auto-resurrection-policy-unplanned-failure.md
carry_forward:
  cfd_0024:
    status: narrowed
    remaining: >
      Runtime API autonomous killed-process resurrection and broader service
      recovery policy beyond safe local/dev frontend/backend dev-server proof.
  cfd_0029:
    status: remains_open
    remaining: >
      Compaction/autonomous continuation, parent/return-role wrapper semantics,
      hosted Codex hook/log coverage, and message freshness proof.
  cfd_0030:
    status: remains_open
    remaining: >
      Expand QA with deterministic and randomized end-to-end behavior proof,
      including mocked frontend state-machine inputs before full-system proof.
  cfd_0032:
    status: remains_open
    remaining: >
      Runtime-owned Backlog to Active activation path is missing; current
      compatibility facade remains emergency/guarded.
  cfd_0033:
    status: remains_open
    remaining: >
      Add circuit-breaker/backoff semantics before broader auto-resurrection.
closure_plan:
  - run summary validation
  - run typed QA review with write
  - rerun summary validation
  - run governance validation
  - run Runtime Health smoke
  - transition chunk to Ready for Human Review
  - request/consume close_commit approval through canonical Runtime/Telegram path
  - commit after approval if policy permits
  - stop; do not start next chunk
