> ## Documentation Index
> Fetch the complete documentation index at: https://docs.egma.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Create and use a custom LLM grader

> Create a custom LLM grader owned by this project. Set its instructions, model, scope, and pass threshold. Omit settings to save the declared model defaults at creation.



## OpenAPI

````yaml openapi.json POST /v1/grader-library/custom
openapi: 3.1.0
info:
  title: Egma Platform API
  version: 1.0.0
  description: >-
    The customer-facing HTTP interface used by Egma's web app, CLI, and outside
    clients.
servers:
  - url: https://app.egma.ai
security: []
paths:
  /v1/grader-library/custom:
    post:
      tags:
        - Graders
      summary: Create and use a custom LLM grader
      description: >-
        Create a custom LLM grader owned by this project. Set its instructions,
        model, scope, and pass threshold. Omit settings to save the declared
        model defaults at creation.
      operationId: createCustomGrader
      parameters:
        - name: projectId
          in: query
          required: false
          schema:
            type: string
            minLength: 1
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                name:
                  type: string
                description:
                  anyOf:
                    - type: string
                    - type: 'null'
                gradingInstructions:
                  type: string
                  description: One rule to decide and the conversation evidence to inspect.
                passesWhen:
                  type: string
                  description: >-
                    The evidence that makes the rule pass. Include how to handle
                    a conversation where the checked action never occurs.
                failsWhen:
                  type: string
                  description: The evidence that makes the rule fail.
                settings:
                  type: object
                  additionalProperties: true
                  description: >-
                    A complete llm_provider and llm_model pair from Get grader
                    model choices. Omit settings to use the declared defaults.
                scope:
                  type: object
                  properties:
                    simulations:
                      type: array
                      items:
                        oneOf:
                          - type: object
                            properties:
                              kind:
                                type: string
                                enum:
                                  - all
                            required:
                              - kind
                            additionalProperties: false
                          - type: object
                            properties:
                              kind:
                                type: string
                                enum:
                                  - test_suite
                              id:
                                type: string
                                minLength: 1
                            required:
                              - kind
                              - id
                            additionalProperties: false
                          - type: object
                            properties:
                              kind:
                                type: string
                                enum:
                                  - test
                              id:
                                type: string
                                minLength: 1
                            required:
                              - kind
                              - id
                            additionalProperties: false
                    production:
                      anyOf:
                        - type: object
                          properties:
                            samplePercent:
                              type: number
                              minimum: 1
                              maximum: 100
                          required:
                            - samplePercent
                          additionalProperties: false
                        - type: 'null'
                  required:
                    - simulations
                    - production
                  additionalProperties: false
                  description: >-
                    The future simulations and/or production sample this project
                    should grade.
                passThreshold:
                  type: number
                  minimum: 0
                  maximum: 1
                  description: >-
                    The minimum score for this grader's individual result to
                    pass.
              required:
                - name
                - gradingInstructions
                - passesWhen
                - failsWhen
                - scope
                - passThreshold
              additionalProperties: false
              examples:
                - name: Appointment recap
                  gradingInstructions: >-
                    Decide whether the agent repeats the chosen appointment date
                    and time and asks the caller to confirm them. Use the
                    transcript.
                  passesWhen: >-
                    The agent repeats the chosen date and time and asks for
                    confirmation. If no appointment is chosen, the rule is met.
                  failsWhen: >-
                    An appointment is chosen and the agent ends the conversation
                    without repeating both its date and time and asking for
                    confirmation.
                  scope:
                    simulations:
                      - kind: all
                    production: null
                  passThreshold: 1
      responses:
        '201':
          description: The custom definition and its current-project policy.
          content:
            application/json:
              schema:
                type: object
                properties:
                  definition:
                    type: object
                    properties:
                      id:
                        type: string
                        minLength: 1
                      name:
                        type: string
                      description:
                        anyOf:
                          - type: string
                          - type: 'null'
                      owner:
                        type: string
                        enum:
                          - egma
                          - project
                      type:
                        type: string
                        enum:
                          - llm_as_judge
                          - code
                      scopeEditable:
                        type: boolean
                      currentDefinitionVersion:
                        type: integer
                        minimum: 1
                      definitionVersion:
                        type: integer
                        minimum: 1
                      modalities:
                        type: array
                        items:
                          type: string
                          enum:
                            - chat
                            - voice
                        minItems: 1
                        uniqueItems: true
                      gradingInstructions:
                        anyOf:
                          - type: string
                          - type: 'null'
                      requiredEvidence:
                        type: array
                        items:
                          type: string
                          enum:
                            - transcript
                            - ending_outcome
                            - tool_calls
                            - observed_metrics
                            - test_expected_behaviors
                            - turn_response_latency
                      settingDefinitions:
                        type: array
                        items:
                          type: object
                          properties:
                            key:
                              type: string
                            label:
                              type: string
                            valueType:
                              type: string
                              enum:
                                - integer
                                - number
                                - string
                            defaultValue:
                              oneOf:
                                - type: number
                                - type: string
                            unit:
                              anyOf:
                                - type: string
                                - type: 'null'
                            minimum:
                              anyOf:
                                - type: number
                                - type: 'null'
                            maximum:
                              anyOf:
                                - type: number
                                - type: 'null'
                          required:
                            - key
                            - label
                            - valueType
                            - defaultValue
                            - unit
                            - minimum
                            - maximum
                          additionalProperties: false
                      activeProjectGraderId:
                        anyOf:
                          - type: string
                            minLength: 1
                          - type: 'null'
                      createdAt:
                        type: string
                        format: date-time
                      updatedAt:
                        type: string
                        format: date-time
                    required:
                      - id
                      - name
                      - description
                      - owner
                      - type
                      - scopeEditable
                      - currentDefinitionVersion
                      - definitionVersion
                      - modalities
                      - gradingInstructions
                      - requiredEvidence
                      - settingDefinitions
                      - activeProjectGraderId
                      - createdAt
                      - updatedAt
                    additionalProperties: false
                  grader:
                    type: object
                    properties:
                      id:
                        type: string
                        minLength: 1
                      projectId:
                        type: string
                        minLength: 1
                      graderDefinitionId:
                        type: string
                        minLength: 1
                      name:
                        type: string
                      description:
                        anyOf:
                          - type: string
                          - type: 'null'
                      owner:
                        type: string
                        enum:
                          - egma
                          - project
                      type:
                        type: string
                        enum:
                          - llm_as_judge
                          - code
                      modalities:
                        type: array
                        items:
                          type: string
                          enum:
                            - chat
                            - voice
                        minItems: 1
                        uniqueItems: true
                      scopeEditable:
                        type: boolean
                      removable:
                        type: boolean
                      scope:
                        type: object
                        properties:
                          simulations:
                            type: array
                            items:
                              oneOf:
                                - type: object
                                  properties:
                                    kind:
                                      type: string
                                      enum:
                                        - all
                                  required:
                                    - kind
                                  additionalProperties: false
                                - type: object
                                  properties:
                                    kind:
                                      type: string
                                      enum:
                                        - test_suite
                                    id:
                                      type: string
                                      minLength: 1
                                  required:
                                    - kind
                                    - id
                                  additionalProperties: false
                                - type: object
                                  properties:
                                    kind:
                                      type: string
                                      enum:
                                        - test
                                    id:
                                      type: string
                                      minLength: 1
                                  required:
                                    - kind
                                    - id
                                  additionalProperties: false
                          production:
                            anyOf:
                              - type: object
                                properties:
                                  samplePercent:
                                    type: number
                                    minimum: 1
                                    maximum: 100
                                required:
                                  - samplePercent
                                additionalProperties: false
                              - type: 'null'
                        required:
                          - simulations
                          - production
                        additionalProperties: false
                      settings:
                        type: object
                        additionalProperties: true
                      passThreshold:
                        type: number
                        minimum: 0
                        maximum: 1
                      createdAt:
                        type: string
                        format: date-time
                      updatedAt:
                        type: string
                        format: date-time
                    required:
                      - id
                      - projectId
                      - graderDefinitionId
                      - name
                      - description
                      - owner
                      - type
                      - modalities
                      - scopeEditable
                      - removable
                      - scope
                      - settings
                      - passThreshold
                      - createdAt
                      - updatedAt
                    additionalProperties: false
                required:
                  - definition
                  - grader
                additionalProperties: false
        '400':
          description: The request was refused.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Refusal'
        '401':
          description: The request was refused.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Refusal'
        '403':
          description: The request was refused.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Refusal'
        '404':
          description: The request was refused.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Refusal'
        '409':
          description: The request was refused.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Refusal'
        '422':
          description: The request was refused.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Refusal'
        '429':
          description: The request rate limit was reached.
          headers:
            Retry-After:
              description: Seconds to wait before trying again.
              schema:
                type: integer
                minimum: 1
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Refusal'
      security:
        - bearerAuth: []
        - sessionCookie: []
components:
  schemas:
    Refusal:
      type: object
      properties:
        error:
          type: string
        message:
          type: string
        details:
          type: object
          additionalProperties: true
      required:
        - error
        - message
      additionalProperties: false
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
      description: An Egma API key.
    sessionCookie:
      type: apiKey
      in: cookie
      name: egma.session_token
      description: >-
        The browser session cookie issued by Egma. HTTPS deployments add the
        standard __Secure- prefix.

````