Catalyst / admin/Synapse-Cortex 14.8 GB / 57.8 GB 40.0 GB free
Help Sign in

admin / Synapse-Cortex

public

Self Hosted ITSM Tool with RBAC/Tenanting and MFA

Code Issues Pull requests Pipelines Packages Security Insights Wiki Settings
Synapse-Cortex / Synapse-Cortexv2 / app / routers / remediation.py 26714 B · main
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
import uuid
from dataclasses import asdict
from datetime import datetime, timezone
from typing import Optional

from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.orm import Session

from .. import models, schemas
from ..ai.executor import get_live_executor, get_simulated_executor, sudo_password
from ..ai.guardrails import PlanStep, graph_uses_freeform_command, resolve_placeholders, run_guardrails
from ..ai.investigator import InvestigatorError, get_anthropic_client, investigate
from ..ai.vault import VaultError, decrypt_secret
from ..audit import log_audit
from ..database import get_db
from ..deps import get_session_user

router = APIRouter(prefix="/api/v1/tickets/{ticket_id}/remediation", tags=["remediation"])

# Fixed mask standing in for the real credential secret everywhere a
# command is displayed or persisted (proposed plan, Actions Taken text,
# execution log) - the real value is only ever substituted in-memory,
# immediately before dispatch to the executor, and never stored or
# returned via any API response. See app/ai/vault.py for the same
# decrypt-only-immediately-before-use discipline applied to the secret
# itself.
MASKED_SECRET = "********"


def _ticket_credential_context(ticket: models.Ticket) -> dict:
    """What's available to resolve {{username}}/{{ip_address}}/{{password}}
    placeholders for this ticket - passed to run_guardrails() so a playbook
    that references data the ticket doesn't have yet is blocked with a
    clear reason instead of sending the literal placeholder text."""
    return {
        "has_username": bool(ticket.credential_ref and ticket.credential_ref.username),
        "has_ip": bool(ticket.asset and ticket.asset.ip_address),
        "has_password": bool(ticket.credential_ref),
    }


def _display_steps(steps: list[PlanStep], ticket: models.Ticket) -> list[PlanStep]:
    """Resolves {{username}}/{{ip_address}} to their real values (safe to
    show or store) and {{password}} to a fixed mask - used for the
    proposed plan and Actions Taken text. Never used for the command
    actually dispatched to the executor (see _execute_plan, which does its
    own resolution with the real secret)."""
    username = ticket.credential_ref.username if ticket.credential_ref else None
    ip_address = ticket.asset.ip_address if ticket.asset else None
    password = MASKED_SECRET if ticket.credential_ref else None
    return [
        PlanStep(
            node_id=s.node_id,
            label=s.label,
            command=resolve_placeholders(s.command, username=username, ip_address=ip_address, password=password),
        )
        for s in steps
    ]


def _get_tenant_ticket(db: Session, ticket_id: uuid.UUID, tenant_id: uuid.UUID) -> models.Ticket:
    ticket = (
        db.query(models.Ticket)
        .filter(models.Ticket.id == ticket_id, models.Ticket.tenant_id == tenant_id)
        .first()
    )
    if not ticket:
        raise HTTPException(status.HTTP_404_NOT_FOUND)
    return ticket


def _get_tenant_run(db: Session, run_id: uuid.UUID, ticket_id: uuid.UUID, tenant_id: uuid.UUID) -> models.RemediationRun:
    run = (
        db.query(models.RemediationRun)
        .filter(
            models.RemediationRun.id == run_id,
            models.RemediationRun.ticket_id == ticket_id,
            models.RemediationRun.tenant_id == tenant_id,
        )
        .first()
    )
    if not run:
        raise HTTPException(status.HTTP_404_NOT_FOUND)
    return run


def _post_action(db: Session, *, tenant_id: uuid.UUID, ticket_id: uuid.UUID, user_id: Optional[uuid.UUID], body: str) -> None:
    db.add(models.TicketAction(tenant_id=tenant_id, ticket_id=ticket_id, user_id=user_id, body=body))
    db.flush()


def _format_plan(steps: list[PlanStep]) -> str:
    if not steps:
        return "(no commands)"
    return "\n".join(f"{i + 1}. {s.label or s.command}: {s.command}" for i, s in enumerate(steps))


def _format_results(execution_log: list[dict]) -> str:
    """Renders the per-step command output for the permanent Actions Taken
    log, so the full result of a run is viewable in the ticket history
    forever - not just in the (latest-run-only) AI panel. Commands/outputs
    here are already secret-scrubbed by _execute_plan."""
    if not execution_log:
        return "(no output)"
    blocks = []
    for step in execution_log:
        label = step.get("label") or step.get("node_id") or "step"
        blocks.append(f"[{label}]  $ {step.get('command', '')}  (exit {step.get('exit_status')})")
        output = (step.get("output") or "").strip()
        if output:
            blocks.append(output)
    return "\n".join(blocks)


def _run_plan(
    db: Session,
    *,
    run: models.RemediationRun,
    ticket: models.Ticket,
    steps: list[PlanStep],
    user_id: Optional[uuid.UUID],
    live: bool,
    terminal: bool,
) -> None:
    """Decrypts the ticket's linked credential, dispatches each command
    through the SSH executor, and records the outcome on the run plus a
    permanent Actions Taken entry. Never invoked unless guardrails have
    already passed for these exact steps.

    ``live``     - when True, use the real paramiko executor (actually fixing
                   the client over SSH), on every deployment. When False, always
                   simulate (the Simulate/rehearsal an operator reviews before
                   approving a live run).
    ``terminal`` - when True, the run reaches a terminal SUCCEEDED/FAILED
                   state. When False (dry run) the run stays PENDING_APPROVAL
                   so the operator can review the simulated output and then
                   approve a real execution on the same run."""
    # Live runs use the real SSH executor on every deployment; dry runs always
    # simulate. There is no "demo live" - the safe path is the Simulate button.
    executor = get_live_executor() if live else get_simulated_executor()
    use_live = live

    def _fail(message: str) -> None:
        run.outcome_summary = message
        if terminal:
            run.status = models.RemediationRunStatus.FAILED
        _post_action(db, tenant_id=run.tenant_id, ticket_id=ticket.id, user_id=user_id, body=message)

    if not ticket.credential_ref_id or not ticket.credential_ref:
        _fail("No credential is linked to this ticket; execution could not proceed. Link one and retry.")
        return

    tenant = db.get(models.Tenant, run.tenant_id)
    try:
        secret = decrypt_secret(tenant, ticket.credential_ref.secret_encrypted)
    except VaultError as exc:
        _fail(f"Credential could not be decrypted: {exc}")
        return

    username = ticket.credential_ref.username
    ip_address = ticket.asset.ip_address if ticket.asset else None
    host = ticket.credential_ref.host

    # The live executor decides per host whether to elevate with sudo (non-root
    # account) or run the command directly (root account). We pass the raw
    # command and store back whatever it actually ran. sudo_pw is only needed
    # here to scrub it from stored text.
    sudo_pw = sudo_password(secret)

    execution_log: list[dict] = []
    failed = False
    for step in steps:
        real_command = resolve_placeholders(step.command, username=username, ip_address=ip_address, password=secret)
        result = executor.run(
            host=host,
            port=ticket.credential_ref.port,
            username=username,
            secret=secret,
            command=real_command,
        )
        # The executor returns the command it actually ran (sudo-wrapped or not).
        # Defense in depth: scrub the secret AND the sudo password out of both
        # the stored command and the output before persisting/displaying.
        shown_command = result.command
        safe_output = result.output
        for sensitive in (secret, sudo_pw):
            if sensitive:
                shown_command = shown_command.replace(sensitive, MASKED_SECRET)
                safe_output = safe_output.replace(sensitive, MASKED_SECRET)
        execution_log.append(
            {
                "node_id": step.node_id,
                "label": step.label,
                "command": shown_command,
                "output": safe_output,
                "exit_status": result.exit_status,
                "duration_ms": result.duration_ms,
            }
        )
        if result.exit_status != 0:
            failed = True
            break

    run.execution_log = execution_log
    ran = len(execution_log)
    total = len(steps)
    if terminal:
        run.status = models.RemediationRunStatus.FAILED if failed else models.RemediationRunStatus.SUCCEEDED
        where = f" on client {host}" if use_live else " (simulated)"
        verb = "failed" if failed else "completed successfully"
        run.outcome_summary = (
            f"Playbook '{run.playbook_name_snapshot}' {verb}{where} after {ran} of {total} step(s)."
        )
        mode_label = f"LIVE execution on client {host}" if use_live else "simulated execution"
    else:
        # Dry run: leave the run PENDING_APPROVAL for the operator to review.
        run.outcome_summary = (
            f"Simulation complete - {ran} of {total} step(s). Review the simulated output below, then "
            f"Execute Live to run it on the client for real over SSH."
        )
        mode_label = "dry run (simulation)"

    # Fold the full per-step output into the permanent Actions Taken log too,
    # so the operator can view the results of every run in the ticket history
    # (the AI panel only ever shows the latest run's execution log).
    _post_action(
        db,
        tenant_id=run.tenant_id,
        ticket_id=ticket.id,
        user_id=user_id,
        body=(
            f"{run.outcome_summary}\nMode: {mode_label}\nApplied playbook: {run.playbook_name_snapshot}\n\n"
            f"Results:\n{_format_results(execution_log)}"
        ),
    )


@router.get("/runs", response_model=list[schemas.RemediationRunOut])
def list_runs(
    ticket_id: uuid.UUID,
    db: Session = Depends(get_db),
    user: Optional[models.User] = Depends(get_session_user),
):
    if not user:
        raise HTTPException(status.HTTP_401_UNAUTHORIZED)
    _get_tenant_ticket(db, ticket_id, user.tenant_id)
    return (
        db.query(models.RemediationRun)
        .filter(models.RemediationRun.ticket_id == ticket_id, models.RemediationRun.tenant_id == user.tenant_id)
        .order_by(models.RemediationRun.created_at.desc())
        .all()
    )


@router.post("/investigate", response_model=schemas.RemediationRunOut, status_code=status.HTTP_201_CREATED)
def investigate_ticket(
    ticket_id: uuid.UUID,
    db: Session = Depends(get_db),
    user: Optional[models.User] = Depends(get_session_user),
    client=Depends(get_anthropic_client),
):
    if not user:
        raise HTTPException(status.HTTP_401_UNAUTHORIZED)

    tenant = db.get(models.Tenant, user.tenant_id)
    if not tenant.ai_enabled:
        raise HTTPException(status.HTTP_403_FORBIDDEN, detail="The AI Remediation Module is disabled for this tenant")

    ticket = _get_tenant_ticket(db, ticket_id, user.tenant_id)

    enabled_playbooks = (
        db.query(models.Playbook)
        .filter(models.Playbook.tenant_id == user.tenant_id, models.Playbook.enabled.is_(True))
        .all()
    )
    if not enabled_playbooks:
        raise HTTPException(status.HTTP_409_CONFLICT, detail="No enabled playbooks are configured for this tenant")

    ticket_context = {
        "title": ticket.title,
        "description": ticket.description,
        "priority": ticket.priority.value,
        "ticket_type": ticket.ticket_type.value,
    }
    asset_context = None
    if ticket.asset:
        asset_context = {
            "os": ticket.asset.os,
            "asset_type": ticket.asset.asset_type_name,
            "status": ticket.asset.status.value,
        }
        # Operator-authored client/operational context, when present, is the
        # single most important signal for whether a given fix is safe to run
        # on this box - surface it explicitly to the investigator.
        if ticket.asset.client_context:
            asset_context["client_context"] = ticket.asset.client_context
    playbook_payload = [
        {
            "id": pb.id,
            "name": pb.name,
            "allowed_target_os": pb.allowed_target_os,
            "required_approval_level": pb.required_approval_level.value,
            "freeform": graph_uses_freeform_command(pb.graph_json),
        }
        for pb in enabled_playbooks
    ]

    try:
        result = investigate(
            client=client, ticket_context=ticket_context, asset_context=asset_context, playbooks=playbook_payload
        )
    except InvestigatorError as exc:
        raise HTTPException(status.HTTP_502_BAD_GATEWAY, detail=str(exc))

    run = models.RemediationRun(
        tenant_id=user.tenant_id,
        ticket_id=ticket.id,
        ai_summary=result.rationale,
        created_by_id=user.id,
    )

    if result.selected_playbook_id is None:
        run.status = models.RemediationRunStatus.FAILED
        run.outcome_summary = "AI investigation found no suitable playbook."
        db.add(run)
        db.flush()
        _post_action(
            db,
            tenant_id=user.tenant_id,
            ticket_id=ticket.id,
            user_id=user.id,
            body=f"AI investigation found no suitable playbook. Rationale: {result.rationale}",
        )
        log_audit(db, tenant_id=user.tenant_id, user_id=user.id, action="remediation_investigate", detail={"selected": None})
        db.commit()
        db.refresh(run)
        return run

    playbook = next(pb for pb in enabled_playbooks if pb.id == result.selected_playbook_id)
    run.playbook_id = playbook.id
    run.playbook_name_snapshot = playbook.name
    run.guardrails_snapshot = {
        "allowed_target_os": playbook.allowed_target_os,
        "required_approval_level": playbook.required_approval_level.value,
        "forbidden_commands": playbook.forbidden_commands,
        "acknowledged_dangerous_commands": playbook.acknowledged_dangerous_commands,
    }
    run.graph_snapshot = playbook.graph_json
    db.add(run)
    db.flush()

    # A command-runner playbook: the AI proposes the actual command, which is
    # substituted into {{command}} and then subjected to the full guardrail
    # check (so the non-overridable baseline still blocks a dangerous command).
    is_freeform = graph_uses_freeform_command(playbook.graph_json)
    if is_freeform:
        run.suggested_command = result.suggested_command

    guardrail_result = run_guardrails(
        graph_json=playbook.graph_json,
        allowed_target_os=playbook.allowed_target_os,
        forbidden_commands=playbook.forbidden_commands,
        asset_os=ticket.asset.os if ticket.asset else None,
        freeform_command=result.suggested_command if is_freeform else None,
        acknowledged_dangerous_commands=playbook.acknowledged_dangerous_commands,
        **_ticket_credential_context(ticket),
    )

    if not guardrail_result.ok:
        run.status = models.RemediationRunStatus.BLOCKED
        run.outcome_summary = guardrail_result.reason
        _post_action(
            db,
            tenant_id=user.tenant_id,
            ticket_id=ticket.id,
            user_id=user.id,
            body=(
                f"AI selected playbook '{playbook.name}' but execution was blocked by guardrails: "
                f"{guardrail_result.reason}"
            ),
        )
        log_audit(
            db, tenant_id=user.tenant_id, user_id=user.id, action="remediation_investigate", detail={"selected": playbook.name, "blocked": True}
        )
        db.commit()
        db.refresh(run)
        return run

    steps = guardrail_result.steps or []
    display_steps = _display_steps(steps, ticket)
    run.proposed_plan = [asdict(s) for s in display_steps]
    _post_action(
        db,
        tenant_id=user.tenant_id,
        ticket_id=ticket.id,
        user_id=user.id,
        body=(
            f"AI investigation selected playbook '{playbook.name}'. Rationale: {result.rationale}\n"
            f"Proposed plan:\n{_format_plan(display_steps)}"
        ),
    )
    log_audit(
        db, tenant_id=user.tenant_id, user_id=user.id, action="remediation_investigate", detail={"selected": playbook.name, "blocked": False}
    )

    # A run that includes an acknowledged review-eligible command (e.g. a
    # reboot) ALWAYS goes to a human, even on an Auto-Approve playbook: the
    # acknowledgement unblocks it, it never makes it run unattended.
    if guardrail_result.requires_review:
        run.outcome_summary = guardrail_result.review_reason
        _post_action(
            db,
            tenant_id=user.tenant_id,
            ticket_id=ticket.id,
            user_id=user.id,
            body=(
                f"Playbook '{playbook.name}' requires human review before running: "
                f"{guardrail_result.review_reason} It is awaiting an explicit Approve & Execute."
            ),
        )

    # LIVE execution on the client NEVER happens here: it always requires an
    # explicit human "Execute Live" click AND a prior simulation of the same
    # run. So every run lands in PENDING_APPROVAL. An Auto-Approve playbook
    # (no acknowledged/review-required command, not a free-form runner) still
    # gets a hands-off head start: its plan is auto-run through the SIMULATED
    # executor now, so the operator opens the ticket to a completed rehearsal
    # and can go straight to Execute Live. Written as an allow-list so a future
    # edit can only make the auto-simulation MORE restrictive.
    run.status = models.RemediationRunStatus.PENDING_APPROVAL
    may_auto_simulate = (
        playbook.required_approval_level == models.ApprovalLevel.AUTO_APPROVE
        and not guardrail_result.requires_review
        and not is_freeform
    )
    if may_auto_simulate:
        _run_plan(db, run=run, ticket=ticket, steps=steps, user_id=user.id, live=False, terminal=False)

    db.commit()
    db.refresh(run)
    return run


@router.post("/runs/{run_id}/approve", response_model=schemas.RemediationRunOut)
def approve_run(
    ticket_id: uuid.UUID,
    run_id: uuid.UUID,
    payload: schemas.RemediationApproveRequest = schemas.RemediationApproveRequest(),
    db: Session = Depends(get_db),
    user: Optional[models.User] = Depends(get_session_user),
):
    if not user:
        raise HTTPException(status.HTTP_401_UNAUTHORIZED)

    ticket = _get_tenant_ticket(db, ticket_id, user.tenant_id)
    run = _get_tenant_run(db, run_id, ticket_id, user.tenant_id)

    if run.status != models.RemediationRunStatus.PENDING_APPROVAL:
        raise HTTPException(status.HTTP_409_CONFLICT, detail="This run is not awaiting approval")

    # Simulate-before-live: live execution on the client is only allowed once
    # this exact run has been rehearsed with a simulation (Dry-run). A pending
    # run only ever gets an execution_log from a dry run (a live run is
    # terminal), so its presence is the proof a simulation was reviewed first.
    if not run.execution_log:
        raise HTTPException(
            status.HTTP_400_BAD_REQUEST,
            detail="Run a simulation (Dry-run) first, review the result, then Execute Live on the client.",
        )

    snapshot = run.guardrails_snapshot or {}

    # For a command-runner run, the human may have edited the AI's suggested
    # command in the approval card. Whatever they approve (their edit, or the
    # AI's suggestion if unchanged) is re-run through the FULL guardrail engine
    # here - so the non-overridable baseline blocks a dangerous edited command
    # too, and the run remembers exactly what was approved.
    is_freeform = graph_uses_freeform_command(run.graph_snapshot or {})
    final_command = None
    if is_freeform:
        final_command = (payload.command if payload.command is not None else run.suggested_command) or ""
        run.suggested_command = final_command

    guardrail_result = run_guardrails(
        graph_json=run.graph_snapshot or {},
        allowed_target_os=snapshot.get("allowed_target_os", []),
        forbidden_commands=snapshot.get("forbidden_commands", []),
        asset_os=ticket.asset.os if ticket.asset else None,
        freeform_command=final_command if is_freeform else None,
        acknowledged_dangerous_commands=snapshot.get("acknowledged_dangerous_commands", []),
        **_ticket_credential_context(ticket),
    )

    if not guardrail_result.ok:
        run.status = models.RemediationRunStatus.BLOCKED
        run.outcome_summary = guardrail_result.reason
        _post_action(
            db,
            tenant_id=user.tenant_id,
            ticket_id=ticket.id,
            user_id=user.id,
            body=f"Approval blocked by guardrails: {run.outcome_summary}",
        )
        log_audit(
            db,
            tenant_id=user.tenant_id,
            user_id=user.id,
            action="remediation_execute_live",
            detail={"run_id": str(run.id), "playbook": run.playbook_name_snapshot, "blocked": True},
        )
        db.commit()
        db.refresh(run)
        return run

    # Record the exact approved plan (with the final command resolved for
    # display/masking) so the ticket history reflects what was actually run.
    if is_freeform:
        run.proposed_plan = [asdict(s) for s in _display_steps(guardrail_result.steps or [], ticket)]

    run.approved_by_id = user.id
    run.approved_at = datetime.now(timezone.utc)
    run.status = models.RemediationRunStatus.EXECUTING
    _post_action(
        db,
        tenant_id=user.tenant_id,
        ticket_id=ticket.id,
        user_id=user.id,
        body=f"Live execution approved by {user.full_name} after review of the simulation - running on the client (live SSH).",
    )
    log_audit(
        db,
        tenant_id=user.tenant_id,
        user_id=user.id,
        action="remediation_execute_live",
        detail={
            "run_id": str(run.id),
            "ticket_id": str(ticket.id),
            "playbook": run.playbook_name_snapshot,
            "steps": len(guardrail_result.steps or []),
            "live": True,
            "confirmed": True,
        },
    )

    # The operator's explicit go-ahead, gated above on a prior simulation: run
    # for real on the client over SSH.
    _run_plan(db, run=run, ticket=ticket, steps=guardrail_result.steps or [], user_id=user.id, live=True, terminal=True)

    db.commit()
    db.refresh(run)
    return run


@router.post("/runs/{run_id}/dry-run", response_model=schemas.RemediationRunOut)
def dry_run(
    ticket_id: uuid.UUID,
    run_id: uuid.UUID,
    payload: schemas.RemediationApproveRequest = schemas.RemediationApproveRequest(),
    db: Session = Depends(get_db),
    user: Optional[models.User] = Depends(get_session_user),
):
    """Rehearse a pending run WITHOUT touching the client: every step runs
    through the SIMULATED executor, the results are stored on the run, and it
    stays PENDING_APPROVAL so the operator can review the simulated output and
    then Approve & Execute for real. A guardrail failure (e.g. an edited
    free-form command that trips the baseline) is reported without changing the
    run's state, so the operator can adjust and re-run."""
    if not user:
        raise HTTPException(status.HTTP_401_UNAUTHORIZED)

    ticket = _get_tenant_ticket(db, ticket_id, user.tenant_id)
    run = _get_tenant_run(db, run_id, ticket_id, user.tenant_id)

    if run.status != models.RemediationRunStatus.PENDING_APPROVAL:
        raise HTTPException(status.HTTP_409_CONFLICT, detail="This run is not awaiting approval")

    snapshot = run.guardrails_snapshot or {}
    is_freeform = graph_uses_freeform_command(run.graph_snapshot or {})
    final_command = None
    if is_freeform:
        final_command = (payload.command if payload.command is not None else run.suggested_command) or ""
        run.suggested_command = final_command

    guardrail_result = run_guardrails(
        graph_json=run.graph_snapshot or {},
        allowed_target_os=snapshot.get("allowed_target_os", []),
        forbidden_commands=snapshot.get("forbidden_commands", []),
        asset_os=ticket.asset.os if ticket.asset else None,
        freeform_command=final_command if is_freeform else None,
        acknowledged_dangerous_commands=snapshot.get("acknowledged_dangerous_commands", []),
        **_ticket_credential_context(ticket),
    )
    if not guardrail_result.ok:
        # Non-destructive: don't terminate the run, just report why it can't run
        # so the operator can edit the command / fix the ticket and dry-run again.
        raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=guardrail_result.reason)

    if is_freeform:
        run.proposed_plan = [asdict(s) for s in _display_steps(guardrail_result.steps or [], ticket)]

    _post_action(
        db,
        tenant_id=user.tenant_id,
        ticket_id=ticket.id,
        user_id=user.id,
        body=f"Dry run (simulation) performed by {user.full_name}.",
    )
    log_audit(
        db,
        tenant_id=user.tenant_id,
        user_id=user.id,
        action="remediation_dry_run",
        detail={"run_id": str(run.id), "ticket_id": str(ticket.id), "playbook": run.playbook_name_snapshot, "simulated": True},
    )

    _run_plan(db, run=run, ticket=ticket, steps=guardrail_result.steps or [], user_id=user.id, live=False, terminal=False)

    db.commit()
    db.refresh(run)
    return run


@router.post("/runs/{run_id}/reject", response_model=schemas.RemediationRunOut)
def reject_run(
    ticket_id: uuid.UUID,
    run_id: uuid.UUID,
    db: Session = Depends(get_db),
    user: Optional[models.User] = Depends(get_session_user),
):
    if not user:
        raise HTTPException(status.HTTP_401_UNAUTHORIZED)

    ticket = _get_tenant_ticket(db, ticket_id, user.tenant_id)
    run = _get_tenant_run(db, run_id, ticket_id, user.tenant_id)

    if run.status != models.RemediationRunStatus.PENDING_APPROVAL:
        raise HTTPException(status.HTTP_409_CONFLICT, detail="This run is not awaiting approval")

    run.status = models.RemediationRunStatus.REJECTED
    _post_action(
        db,
        tenant_id=user.tenant_id,
        ticket_id=ticket.id,
        user_id=user.id,
        body=f"Remediation run rejected by {user.full_name}.",
    )
    log_audit(
        db,
        tenant_id=user.tenant_id,
        user_id=user.id,
        action="remediation_reject",
        detail={"run_id": str(run.id), "ticket_id": str(ticket.id), "playbook": run.playbook_name_snapshot},
    )

    db.commit()
    db.refresh(run)
    return run