{
  "type": "record",
  "Aspect": {
    "name": "evalRunEvent",
    "schemaVersion": 3,
    "type": "timeseries"
  },
  "name": "EvalRunEvent",
  "namespace": "com.linkedin.pegasus2avro.eval",
  "fields": [
    {
      "Searchable": {
        "fieldName": "lastCompletedTime",
        "fieldType": "DATETIME"
      },
      "type": "long",
      "name": "timestampMillis",
      "doc": "When this eval run completed (epoch ms UTC)."
    },
    {
      "TimeseriesField": {},
      "java": {
        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
      },
      "type": "string",
      "name": "evalUrn",
      "doc": "The eval entity that was run.",
      "Urn": "Urn"
    },
    {
      "TimeseriesField": {},
      "type": {
        "type": "enum",
        "symbolDocs": {
          "COMPLETE": "The eval run completed (pass, fail, or error in result).",
          "QUEUED": "The eval has been accepted for execution and is waiting to be picked up\nby an executor. Emitted by the GMS-side execution request creator.",
          "RUNNING": "An executor has picked up the eval and started processing it. Emitted\nby the executor immediately before invoking the agent under test."
        },
        "name": "EvalRunStatus",
        "namespace": "com.linkedin.pegasus2avro.eval",
        "symbols": [
          "QUEUED",
          "RUNNING",
          "COMPLETE"
        ],
        "doc": "The lifecycle status of an eval run.\n\nEmitted as separate timeseries EvalRunEvent records:\nQUEUED -> RUNNING -> COMPLETE. Aggregations over results\n(pass/fail/error counts) should filter to COMPLETE-only events."
      },
      "name": "status",
      "doc": "Lifecycle status of this run event."
    },
    {
      "type": [
        "null",
        {
          "type": "record",
          "name": "EvalResult",
          "namespace": "com.linkedin.pegasus2avro.eval",
          "fields": [
            {
              "type": {
                "type": "enum",
                "symbolDocs": {
                  "ERROR": "An error occurred during evaluation (e.g. agent unreachable, judge failure).",
                  "FAIL": "One or more conditions failed.",
                  "PASS": "All conditions passed."
                },
                "name": "EvalResultType",
                "namespace": "com.linkedin.pegasus2avro.eval",
                "symbols": [
                  "PASS",
                  "FAIL",
                  "ERROR"
                ],
                "doc": "The verdict of an eval run."
              },
              "name": "type",
              "doc": "Overall verdict: PASS only if every condition passed."
            },
            {
              "type": [
                "null",
                "string"
              ],
              "name": "actualResponse",
              "default": null,
              "doc": "The actual response text produced by the agent."
            },
            {
              "type": [
                "null",
                {
                  "type": "array",
                  "items": {
                    "type": "record",
                    "name": "EvalConditionResult",
                    "namespace": "com.linkedin.pegasus2avro.eval",
                    "fields": [
                      {
                        "type": {
                          "type": "record",
                          "name": "EvalCondition",
                          "namespace": "com.linkedin.pegasus2avro.eval",
                          "fields": [
                            {
                              "type": {
                                "type": "enum",
                                "symbolDocs": {
                                  "LLM_JUDGE": "An LLM judge evaluates whether the agent's response satisfies\nwritten guidelines (e.g. \"must reference the revenue table\")."
                                },
                                "name": "EvalConditionType",
                                "namespace": "com.linkedin.pegasus2avro.eval",
                                "symbols": [
                                  "LLM_JUDGE"
                                ],
                                "doc": "The type of condition used to judge an eval response.\nAll conditions on an eval must pass for the eval to pass (logical AND).\n\nEach value maps to an optional sub-record on EvalCondition that holds\nthe type-specific configuration."
                              },
                              "name": "type",
                              "doc": "The type of this condition, which determines how it is judged."
                            },
                            {
                              "type": [
                                "null",
                                {
                                  "type": "record",
                                  "name": "EvalLlmJudgeCondition",
                                  "namespace": "com.linkedin.pegasus2avro.eval",
                                  "fields": [
                                    {
                                      "type": "string",
                                      "name": "guidelines",
                                      "doc": "Natural-language criteria the LLM judge uses to evaluate the\nagent's response (e.g. \"must reference the revenue table and\naggregate by quarter\")."
                                    }
                                  ],
                                  "doc": "Configuration for an LLM-judged condition. The judge evaluates the\nagent's response against these natural-language guidelines and\ndetermines pass/fail."
                                }
                              ],
                              "name": "llmJudge",
                              "default": null,
                              "doc": "LLM judge condition \u2014 populated when type is LLM_JUDGE."
                            }
                          ],
                          "doc": "A single condition that must be satisfied for an eval to pass.\nAll conditions on an eval are evaluated with AND semantics.\n\nThe type enum is the discriminator; exactly one of the per-type\nsub-records should be populated (matching the type value)."
                        },
                        "name": "condition",
                        "doc": "The condition that was evaluated (snapshot at run time)."
                      },
                      {
                        "type": "boolean",
                        "name": "passed",
                        "doc": "Whether this individual condition passed."
                      },
                      {
                        "type": [
                          "null",
                          "string"
                        ],
                        "name": "reasoning",
                        "default": null,
                        "doc": "Judge reasoning for this condition's verdict."
                      }
                    ],
                    "doc": "The result of evaluating a single condition within an eval run."
                  }
                }
              ],
              "name": "conditionResults",
              "default": null,
              "doc": "Per-condition results with individual pass/fail and judge reasoning."
            },
            {
              "type": [
                "null",
                "string"
              ],
              "name": "judgeModel",
              "default": null,
              "doc": "The LLM model used as the judge for this run.\nCaptured at execution time for reproducibility."
            },
            {
              "type": [
                "null",
                "string"
              ],
              "name": "judgeReasoning",
              "default": null,
              "doc": "Overall judge reasoning or summary."
            },
            {
              "type": [
                "null",
                "string"
              ],
              "name": "error",
              "default": null,
              "doc": "Error details if the result type is ERROR."
            }
          ],
          "doc": "The result of running an eval: overall verdict plus per-condition breakdown."
        }
      ],
      "name": "result",
      "default": null,
      "doc": "The evaluation result, present when status is COMPLETE."
    },
    {
      "type": [
        "null",
        {
          "type": "array",
          "items": "com.linkedin.pegasus2avro.eval.EvalCondition"
        }
      ],
      "name": "conditionsSnapshot",
      "default": null,
      "doc": "Snapshot of the conditions at run time.\nPrevents silent semantic drift when eval definitions are edited."
    },
    {
      "TimeseriesField": {},
      "type": [
        "null",
        "string"
      ],
      "name": "runId",
      "default": null,
      "doc": "UUID grouping all eval run events from a single task execution.\nEnables batch-level reporting (\"how did the suite perform last night?\")."
    },
    {
      "TimeseriesField": {},
      "type": [
        "null",
        {
          "type": "enum",
          "symbolDocs": {
            "ONLINE": "Triggered by a real production conversation (future).",
            "ON_DEMAND": "Triggered manually / on demand via the API.",
            "SCHEDULED": "Triggered by a scheduled Task (cron)."
          },
          "name": "EvalRunMode",
          "namespace": "com.linkedin.pegasus2avro.eval",
          "symbols": [
            "SCHEDULED",
            "ON_DEMAND",
            "ONLINE"
          ],
          "doc": "How an eval run was triggered."
        }
      ],
      "name": "runMode",
      "default": null,
      "doc": "How this run was triggered."
    },
    {
      "TimeseriesField": {},
      "java": {
        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
      },
      "type": [
        "null",
        "string"
      ],
      "name": "actionRequestUrn",
      "default": null,
      "doc": "The action request (e.g. Context Hub proposal) that triggered this eval run.\nWhen present, this run is a \"what-if\" evaluation against a proposed change,\nenabling delta comparison with the latest scheduled production baseline.",
      "Urn": "Urn"
    },
    {
      "java": {
        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
      },
      "type": [
        "null",
        "string"
      ],
      "name": "sourceRunUrn",
      "default": null,
      "doc": "For ONLINE runs: the production conversation/run that triggered this eval.",
      "Urn": "Urn"
    },
    {
      "java": {
        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
      },
      "type": [
        "null",
        "string"
      ],
      "name": "agentRunUrn",
      "default": null,
      "doc": "The agent run / conversation turn URN, for trace-level judging.",
      "Urn": "Urn"
    },
    {
      "type": [
        "null",
        "string"
      ],
      "name": "traceRef",
      "default": null,
      "doc": "Opaque reference to an external trace store (e.g. LangSmith trace ID)."
    },
    {
      "type": [
        "null",
        {
          "type": "map",
          "values": "string"
        }
      ],
      "name": "runtimeContext",
      "default": null,
      "doc": "Arbitrary key-value pairs for extensible runtime context."
    },
    {
      "type": [
        "null",
        "string"
      ],
      "name": "messageId",
      "default": null,
      "doc": "Optional custom message ID for deduplication."
    }
  ],
  "doc": "A timeseries event recording the result of running an individual eval.\n\nEach event is an immutable record: conditions and judge config are\nsnapshotted at execution time so historical results remain meaningful\neven if the eval definition is later modified."
}