AI Pilot Success Criteria & Evaluation Template — Interactive Pilot Workbook

{ "Title": "AI Pilot Success Criteria & Evaluation Workbook", "IntroductionHtml": "

Use this interactive workbook to capture a complete, operationally focused AI pilot plan. The fields below guide you from a clear business objective to measurable success criteria, data readiness checks, safety and governance constraints, monitoring and roll‑out triggers. Save this record to preserve institutional knowledge, share with stakeholders, and execute measurable pilots that can move to production.

", "SubmitLabel": "Save Pilot", "SuccessMessage": "Pilot saved. You can copy, edit, or export this pilot definition for review and execution.", "DataType": "ai_pilot_template", "SchemaVersion": 1, "Fields": [ { "Key": "pilot_name", "FieldType": "text", "Label": "Pilot name", "HelpText": "A short, descriptive name (e.g., 'CNC Spindle Vibration Anomaly Detection - Line 2').", "Required": true }, { "Key": "pilot_owner", "FieldType": "text", "Label": "Pilot owner / sponsor", "HelpText": "Person or role accountable for the pilot (name and email or team).", "Required": true }, { "Key": "plant_location", "FieldType": "text", "Label": "Plant / location / line", "HelpText": "Where the pilot will run (site, line, machine group).", "Required": false }, { "Key": "business_objective", "FieldType": "textarea", "Label": "Business objective (why this pilot matters)", "HelpText": "Describe the operational problem and the expected business outcome (reduce downtime, improve first‑pass yield, reduce cycle time, improve scheduling, etc.). Connect to specific stakeholders and customers.", "Required": true }, { "Key": "baseline_metric_description", "FieldType": "textarea", "Label": "Baseline metric (what you measure today)", "HelpText": "Describe the current metric, how it is measured, its current value and time window (e.g., mean downtime per week = 12 hrs measured from CMMS).", "Required": true }, { "Key": "baseline_value", "FieldType": "text", "Label": "Baseline value & units", "HelpText": "Enter the numeric baseline and units (e.g., '12 hrs/week' or '92% FPY').", "Required": true }, { "Key": "expected_lift", "FieldType": "textarea", "Label": "Expected lift / target", "HelpText": "Quantify the pilot target (e.g., reduce downtime by 25% vs baseline, increase FPY from 92% to 96%). Include timeframe for expected improvement.", "Required": true }, { "Key": "kpis", "FieldType": "textarea", "Label": "Primary and secondary KPIs", "HelpText": "List KPIs you will track (primary KPI must map to the business objective). Examples: MTTR, MTBF, OEE, FPY, false positive rate, lead time, schedule adherence.", "Required": true }, { "Key": "timeframe_start", "FieldType": "text", "Label": "Planned start (approx.)", "HelpText": "Month or date when pilot begins.", "Required": false }, { "Key": "timeframe_end", "FieldType": "text", "Label": "Planned end / review date", "HelpText": "Planned completion or formal review date.", "Required": false }, { "Key": "data_sources", "FieldType": "textarea", "Label": "Data sources (systems, sensors, tables)", "HelpText": "List where data comes from (PLC tags, historian, MES, CMMS, inspection records, images, operator logs) and sample rates. Include owners and access notes.", "Required": true }, { "Key": "data_quality_checks", "FieldType": "textarea", "Label": "Data quality checks (what you'll validate)", "HelpText": "Examples: missing value rate, time synchronization, outliers, sensor drift, timestamp alignment, class imbalance for labels. Describe acceptance criteria for data readiness.", "Required": true }, { "Key": "data_readiness_score", "FieldType": "scale", "Label": "Data readiness (1 = poor, 5 = production ready)", "HelpText": "Quick readiness assessment after performing quality checks.", "Min": 1, "Max": 5, "Required": true }, { "Key": "labeling_needs", "FieldType": "textarea", "Label": "Labeling needs and plan", "HelpText": "Describe what labels are required (anomaly/no‑anomaly, defect types), how labels will be created (operator annotation, historical events, automated heuristics), and labeling quality checks.", "Required": false }, { "Key": "sample_size_estimate", "FieldType": "number", "Label": "Estimated training / validation sample size (approx.)", "HelpText": "Enter number of labeled events or images needed, or note 'TBD'.", "Required": false }, { "Key": "model_acceptance_thresholds", "FieldType": "textarea", "Label": "Model acceptance thresholds", "HelpText": "Specify numeric thresholds for acceptance (e.g., Precision >= 0.85, Recall >= 0.70, F1 >= 0.77; or MAPE <= 10%). Tie thresholds to operational cost or benefit when possible.", "Required": true }, { "Key": "operator_safety_constraints", "FieldType": "textarea", "Label": "Operator & safety constraints", "HelpText": "Identify any safety rules, no‑go actions, or regulatory constraints. Describe how the model's outputs will (or will not) affect operator actions and controls.", "Required": true }, { "Key": "human_in_loop_plan", "FieldType": "textarea", "Label": "Human‑in‑the‑loop / decision flow", "HelpText": "Describe how operators will interact with model outputs (alerts, suggestions, confirmations), required training, and standard work changes.", "Required": true }, { "Key": "monitoring_plan", "FieldType": "textarea", "Label": "Monitoring & validation plan", "HelpText": "How will you monitor model performance in the pilot? Include metrics, sampling cadence, drift detection, and who reviews results.", "Required": true }, { "Key": "alert_triggers", "FieldType": "textarea", "Label": "Alert triggers and severity", "HelpText": "Define concrete trigger thresholds for alerts (e.g., anomaly score > 0.8 triggers inspection) and associated response steps.", "Required": false }, { "Key": "rollback_criteria", "FieldType": "textarea", "Label": "Rollback or stop criteria", "HelpText": "When should the pilot be paused or rolled back to manual operation? Tie to safety, unacceptable false positive/negative rates, or negative operational impact.", "Required": true }, { "Key": "commercialization_checklist", "FieldType": "checkbox", "Label": "Commercialization checklist (preconditions to move from pilot to production)", "HelpText": "Select items that must be satisfied before considering production rollout.", "Options": [ { "Value": "kpi_met", "Label": "Primary KPI target met and validated" }, { "Value": "stable_performance", "Label": "Model performance stable across time windows" }, { "Value": "data_pipeline", "Label": "Reliable production data pipeline available" }, { "Value": "operator_training", "Label": "Operator training & SOPs updated" }, { "Value": "safety_review", "Label": "Safety & regulatory review complete" }, { "Value": "monitoring_ready", "Label": "Monitoring & alerting in place" }, { "Value": "cost_benefit", "Label": "Validated cost/benefit or ROI estimate" }, { "Value": "runbook", "Label": "Runbook for incidents & retraining defined" } ], "Required": false }, { "Key": "rollout_triggers", "FieldType": "textarea", "Label": "Roll‑out triggers (conditions to scale)", "HelpText": "Concrete conditions that must be met to expand the solution (e.g., 90 days of stable performance, operator acceptance survey >= 4/5).", "Required": false }, { "Key": "governance_and_stakeholders", "FieldType": "textarea", "Label": "Governance, reviewers, and stakeholders", "HelpText": "List decision owners, reviewers (quality, safety, IT, operations), and meeting cadence for reviews.", "Required": false }, { "Key": "compute_and_infra_needs", "FieldType": "textarea", "Label": "Compute, infra & integration needs", "HelpText": "Edge or cloud requirements, latency constraints, integration with MES/MES APIs, authentication, storage, and backup needs.", "Required": false }, { "Key": "ethical_privacy_considerations", "FieldType": "textarea", "Label": "Privacy, IP & ethical considerations", "HelpText": "Any PII, worker privacy, intellectual property, or ethical implications and how they are mitigated.", "Required": false }, { "Key": "estimated_costs", "FieldType": "text", "Label": "Estimated pilot cost (labor, compute, labeling)", "HelpText": "Rough cost estimate or budget line for the pilot.", "Required": false }, { "Key": "pilot_risk_level", "FieldType": "select", "Label": "Pilot risk level", "HelpText": "Quick risk assessment to prioritize governance attention.", "Options": [ { "Value": "low", "Label": "Low" }, { "Value": "medium", "Label": "Medium" }, { "Value": "high", "Label": "High" } ], "Required": true }, { "Key": "readiness_score", "FieldType": "scale", "Label": "Overall readiness (1 = not ready, 5 = production ready)", "HelpText": "Holistic judgement of pilot readiness after filling the workbook.", "Min": 1, "Max": 5, "Required": true }, { "Key": "acceptance_decision", "FieldType": "radio", "Label": "Recommended next decision after pilot review", "HelpText": "Select recommended action for the pilot after the formal review.", "Options": [ { "Value": "proceed", "Label": "Proceed to production" }, { "Value": "extend", "Label": "Extend pilot (collect more data)" }, { "Value": "rework", "Label": "Rework & retrain model" }, { "Value": "abort", "Label": "Abort pilot" } ], "Required": true }, { "Key": "notes_next_steps", "FieldType": "textarea", "Label": "Notes, open issues & next steps", "HelpText": "Record meetings, action items, owners, and due dates.", "Required": false } ] }