For AI agents: A markdown version of this page is available at https://dnd37xeb23o8n.cloudfront.net/stefon.simmons/docs-astro-web-10435/api/latest/agent-observability/create-or-update-a-custom-evaluator-configuration.md. A documentation index is available at /llms.txt.
homeHomedocsDocsapiAPI
Language
Datadog site

Create or update a custom evaluator configuration

WARNINGNote: This endpoint is in preview and is subject to change. If you have any feedback, contact Datadog support.
PUThttps://api.datadoghq.com/api/unstable/llm-obs/config/evaluators/custom/{eval_name}https://api.us3.datadoghq.com/api/unstable/llm-obs/config/evaluators/custom/{eval_name}https://api.us5.datadoghq.com/api/unstable/llm-obs/config/evaluators/custom/{eval_name}https://api.datadoghq.eu/api/unstable/llm-obs/config/evaluators/custom/{eval_name}https://api.ap1.datadoghq.com/api/unstable/llm-obs/config/evaluators/custom/{eval_name}https://api.ap2.datadoghq.com/api/unstable/llm-obs/config/evaluators/custom/{eval_name}https://api.uk1.datadoghq.com/api/unstable/llm-obs/config/evaluators/custom/{eval_name}https://api.ddog-gov.com/api/unstable/llm-obs/config/evaluators/custom/{eval_name}https://api.us2.ddog-gov.com/api/unstable/llm-obs/config/evaluators/custom/{eval_name}

Overview

Create or update a custom Agent Observability evaluator configuration by its name.

Arguments

Path Parameters

Name
Type
Description
eval_name [required]
string

The name of the custom Agent Observability evaluator configuration.

Request

Body Data (required)

Custom evaluator configuration payload.

Field
Type
Description
data [required]
object

Data object for creating or updating a custom Agent Observability evaluator configuration.

Response

OK

Code Example

# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="datadoghq.com"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF
# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="us3.datadoghq.com"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF
# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="us5.datadoghq.com"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF
# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="datadoghq.eu"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF
# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="ap1.datadoghq.com"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF
# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="ap2.datadoghq.com"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF
# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="uk1.datadoghq.com"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF
# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="ddog-gov.com"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF
# Path parameters
export eval_name="my-custom-evaluator"

# Set your Datadog site and credentials
export DD_SITE="us2.ddog-gov.com"
# Use a Personal Access Token or Service Access Token
export DD_BEARER_TOKEN="<PERSONAL_ACCESS_TOKEN OR SERVICE_ACCESS_TOKEN>"

curl -X PUT "https://api.${DD_SITE}/api/unstable/llm-obs/config/evaluators/custom/${eval_name}" \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${DD_BEARER_TOKEN}" \
-d @- << EOF
{
  "data": {
    "attributes": {
      "llm_judge_config": {
        "inference_params": {
          "max_tokens": 1024,
          "temperature": 0.7
        },
        "parsing_type": "structured_output"
      },
      "llm_provider": {
        "integration_provider": "openai",
        "model_name": "gpt-4o"
      },
      "target": {
        "application_name": "my-llm-app",
        "enabled": true,
        "sampling_percentage": 50
      }
    },
    "id": "my-custom-evaluator",
    "type": "evaluator_config"
  }
}
EOF