{
  "type": "record",
  "Aspect": {
    "name": "evalInfo"
  },
  "name": "EvalInfo",
  "namespace": "com.linkedin.pegasus2avro.eval",
  "fields": [
    {
      "Relationship": {
        "entityTypes": [
          "aiAgent"
        ],
        "name": "EvalAgent"
      },
      "Searchable": {
        "fieldName": "agent",
        "fieldType": "URN"
      },
      "java": {
        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
      },
      "type": [
        "null",
        "string"
      ],
      "name": "agent",
      "default": null,
      "doc": "The agent this eval runs against. When absent, the eval targets the\ndefault Ask DataHub base agent. Used as the primary grouping key\n(\"all evals for agent X\").",
      "Urn": "Urn",
      "entityTypes": [
        "aiAgent"
      ]
    },
    {
      "Searchable": {
        "boostScore": 10.0,
        "enableAutocomplete": true,
        "fieldNameAliases": [
          "_entityName"
        ],
        "fieldType": "WORD_GRAM"
      },
      "type": "string",
      "name": "name",
      "doc": "Display name for this eval"
    },
    {
      "Searchable": {
        "fieldType": "TEXT",
        "hasValuesFieldName": "hasDescription"
      },
      "type": [
        "null",
        "string"
      ],
      "name": "description",
      "default": null,
      "doc": "Human-readable description of what this eval verifies"
    },
    {
      "type": "string",
      "name": "question",
      "doc": "The golden question / prompt sent to the agent under test"
    },
    {
      "type": [
        "null",
        "string"
      ],
      "name": "referenceOutput",
      "default": null,
      "doc": "Optional golden / expected answer for reference-based judging.\nWhen provided, the judge can compare the actual response against this\nfor correctness, not just style or completeness."
    },
    {
      "type": {
        "type": "array",
        "items": {
          "type": "record",
          "name": "EvalCondition",
          "namespace": "com.linkedin.pegasus2avro.eval",
          "fields": [
            {
              "type": {
                "type": "enum",
                "symbolDocs": {
                  "LLM_JUDGE": "An LLM judge evaluates whether the agent's response satisfies\nwritten guidelines (e.g. \"must reference the revenue table\")."
                },
                "name": "EvalConditionType",
                "namespace": "com.linkedin.pegasus2avro.eval",
                "symbols": [
                  "LLM_JUDGE"
                ],
                "doc": "The type of condition used to judge an eval response.\nAll conditions on an eval must pass for the eval to pass (logical AND).\n\nEach value maps to an optional sub-record on EvalCondition that holds\nthe type-specific configuration."
              },
              "name": "type",
              "doc": "The type of this condition, which determines how it is judged."
            },
            {
              "type": [
                "null",
                {
                  "type": "record",
                  "name": "EvalLlmJudgeCondition",
                  "namespace": "com.linkedin.pegasus2avro.eval",
                  "fields": [
                    {
                      "type": "string",
                      "name": "guidelines",
                      "doc": "Natural-language criteria the LLM judge uses to evaluate the\nagent's response (e.g. \"must reference the revenue table and\naggregate by quarter\")."
                    }
                  ],
                  "doc": "Configuration for an LLM-judged condition. The judge evaluates the\nagent's response against these natural-language guidelines and\ndetermines pass/fail."
                }
              ],
              "name": "llmJudge",
              "default": null,
              "doc": "LLM judge condition \u2014 populated when type is LLM_JUDGE."
            }
          ],
          "doc": "A single condition that must be satisfied for an eval to pass.\nAll conditions on an eval are evaluated with AND semantics.\n\nThe type enum is the discriminator; exactly one of the per-type\nsub-records should be populated (matching the type value)."
        }
      },
      "name": "conditions",
      "doc": "Conditions that must ALL pass for this eval to pass.\nEach condition has a type discriminator and a matching sub-record\nwith type-specific configuration."
    },
    {
      "Searchable": {
        "/type": {
          "fieldName": "executorType",
          "fieldType": "KEYWORD",
          "queryByDefault": false
        }
      },
      "type": {
        "type": "record",
        "name": "EvalExecutor",
        "namespace": "com.linkedin.pegasus2avro.eval",
        "fields": [
          {
            "type": {
              "type": "enum",
              "symbolDocs": {
                "EXTERNAL": "Run externally; results reported back to DataHub via reportEvalResult.",
                "NATIVE": "Run and judged by DataHub natively."
              },
              "name": "EvalExecutorType",
              "namespace": "com.linkedin.pegasus2avro.eval",
              "symbols": [
                "NATIVE",
                "EXTERNAL"
              ],
              "doc": "Who executes and judges this eval."
            },
            "name": "type",
            "doc": "Whether this eval is run natively by DataHub or externally."
          }
        ],
        "doc": "Describes who executes and judges this eval. The type field is required;\nadditional fields (e.g. endpoint, config) can be added as the executor\nmodel evolves."
      },
      "name": "executor",
      "doc": "Who executes and judges this eval. Contains a type (NATIVE or EXTERNAL)\nand can be extended with executor-specific config in the future."
    },
    {
      "Searchable": {
        "addToFilters": true,
        "fieldName": "origin",
        "fieldType": "KEYWORD"
      },
      "type": {
        "type": "enum",
        "symbolDocs": {
          "CONTEXT_HUB": "Created via the Context Hub experience for validating\ntext-to-SQL and document-grounded answers.",
          "CUSTOM_AGENT": "Created via the Custom Agent builder for validating\na user-built agent's behavior."
        },
        "name": "EvalOrigin",
        "namespace": "com.linkedin.pegasus2avro.eval",
        "symbols": [
          "CONTEXT_HUB",
          "CUSTOM_AGENT"
        ],
        "doc": "The product area that created this eval. Used for filtering and\ndetermining scheduling behavior (e.g. CONTEXT_HUB evals are\nauto-scheduled on a daily cron)."
      },
      "name": "origin",
      "doc": "Which product area created this eval. Determines scheduling behavior\n(CONTEXT_HUB evals are auto-scheduled daily) and UI filtering."
    },
    {
      "Searchable": {
        "/time": {
          "fieldName": "createdAt",
          "fieldType": "DATETIME",
          "queryByDefault": false
        }
      },
      "type": {
        "type": "record",
        "name": "AuditStamp",
        "namespace": "com.linkedin.pegasus2avro.common",
        "fields": [
          {
            "type": "long",
            "name": "time",
            "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
          },
          {
            "java": {
              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
            },
            "type": "string",
            "name": "actor",
            "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
            "Urn": "Urn"
          },
          {
            "java": {
              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
            },
            "type": [
              "null",
              "string"
            ],
            "name": "impersonator",
            "default": null,
            "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
            "Urn": "Urn"
          },
          {
            "type": [
              "null",
              "string"
            ],
            "name": "message",
            "default": null,
            "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
          }
        ],
        "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
      },
      "name": "created",
      "doc": "When this eval was created"
    },
    {
      "Searchable": {
        "/time": {
          "fieldName": "lastModifiedAt",
          "fieldType": "DATETIME",
          "queryByDefault": false
        }
      },
      "type": "com.linkedin.pegasus2avro.common.AuditStamp",
      "name": "lastModified",
      "doc": "When this eval was last modified"
    }
  ],
  "doc": "Core definition of an Eval \u2014 a golden question paired with conditions\nthat the agent's response must satisfy.\n\nEvals always run against a specific agent. When agent is absent, the\neval runs against the default Ask DataHub base agent. Grouping and\nlisting is by agent.\n\nAll conditions must pass for the eval to pass (logical AND)."
}