> ## Documentation Index
> Fetch the complete documentation index at: https://dokulabs.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.

# DataDog

> LLM Observability with DataDog and Doku

![](https://github.com/dokulabs/doku/blob/main/docs/images/datadog-dashboard-1.jpg?raw=true)

![](https://github.com/dokulabs/doku/blob/main/docs/images/datadog-dashboard-2.jpg?raw=true)

Doku can automatically send LLM usage data from Doku to DataDog using the [Submit Metrics API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics) and [Send Logs API](https://docs.datadoghq.com/api/latest/logs/#send-logs)

Make the following curl request to Doku to add a connection for DataDog. See more on the [Connections API](https://dokulabs.mintlify.site/0.1/api-reference/endpoint/connections/create.md)

```shell
curl --request POST \
  --url https://<YOUR_DOKU_INGESTER_URL>/api/connections \
  --header 'Authorization: <DOKU_API_KEY>' \
  --header 'Content-Type: application/json' \
  --data '{
  "Platform": "datadog",
  "MetricsUrl": "<YOUR_DD_METRICS_URL>",
  "LogsUrl": "<YOUR_DD_LOGS_URL>",
  "ApiKey": "<YOUR_DD_API_KEY>"
}'
```

## DataDog Resources

> **Dashboard**
>
> ```json
> {
> "title": "Doku - LLM Monitoring",
> "description": "Dashboard to monitor LLM Applications using Doku",
> "widgets": [
>   {
>     "id": 2592087041529600,
>     "definition": {
>       "type": "image",
>       "url": "https://github.com/dokulabs/.github/blob/main/profile/assets/favicon.png?raw=true",
>       "sizing": "contain",
>       "has_background": false,
>       "has_border": false,
>       "vertical_align": "center",
>       "horizontal_align": "center"
>     },
>     "layout": {
>       "x": 0,
>       "y": 0,
>       "width": 12,
>       "height": 1
>     }
>   },
>   {
>     "id": 1254363734970878,
>     "definition": {
>       "title": "Avg Request Duration",
>       "title_size": "16",
>       "title_align": "left",
>       "type": "query_value",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "avg:doku.llm.request.duration{host:doku,$applicationname,$environment}",
>               "aggregator": "avg"
>             }
>           ],
>           "conditional_formats": [
>             {
>               "comparator": "<=",
>               "value": 10,
>               "palette": "white_on_green"
>             },
>             {
>               "comparator": ">",
>               "value": 10,
>               "palette": "white_on_red"
>             }
>           ],
>           "formulas": [
>             {
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "second"
>                 }
>               },
>               "formula": "query1"
>             }
>           ]
>         }
>       ],
>       "autoscale": true,
>       "precision": 2,
>       "timeseries_background": {
>         "yaxis": {
>           "include_zero": false
>         },
>         "type": "area"
>       }
>     },
>     "layout": {
>       "x": 0,
>       "y": 1,
>       "width": 3,
>       "height": 2
>     }
>   },
>   {
>     "id": 5019934425891746,
>     "definition": {
>       "title": "Avg Total Tokens",
>       "title_size": "16",
>       "title_align": "left",
>       "type": "query_value",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "avg:doku.llm.total.tokens{host:doku,$applicationname,$environment}",
>               "aggregator": "avg"
>             }
>           ],
>           "conditional_formats": [
>             {
>               "comparator": "<=",
>               "value": 1000,
>               "palette": "white_on_green"
>             },
>             {
>               "comparator": ">",
>               "value": 1000,
>               "palette": "white_on_red"
>             }
>           ],
>           "formulas": [
>             {
>               "formula": "query1"
>             }
>           ]
>         }
>       ],
>       "autoscale": true,
>       "precision": 2,
>       "timeseries_background": {
>         "type": "area"
>       }
>     },
>     "layout": {
>       "x": 3,
>       "y": 1,
>       "width": 3,
>       "height": 2
>     }
>   },
>   {
>     "id": 1292553433133982,
>     "definition": {
>       "title": "Avg Cost / Request",
>       "title_size": "16",
>       "title_align": "left",
>       "type": "query_value",
>       "requests": [
>         {
>           "formulas": [
>             {
>               "formula": "query1",
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "dollar"
>                 }
>               }
>             }
>           ],
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "avg:doku.llm.usage.cost{host:doku,$applicationname,$environment}",
>               "aggregator": "max"
>             }
>           ],
>           "response_format": "scalar",
>           "conditional_formats": [
>             {
>               "comparator": ">",
>               "value": 0,
>               "palette": "custom_bg",
>               "custom_bg_color": "#5c7cff"
>             }
>           ]
>         }
>       ],
>       "autoscale": true,
>       "precision": 2,
>       "timeseries_background": {
>         "yaxis": {
>           "include_zero": true
>         },
>         "type": "area"
>       }
>     },
>     "layout": {
>       "x": 6,
>       "y": 1,
>       "width": 3,
>       "height": 2
>     }
>   },
>   {
>     "id": 5207596770618026,
>     "definition": {
>       "title": "Total Cost",
>       "title_size": "16",
>       "title_align": "left",
>       "type": "query_value",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "sum:doku.llm.usage.cost{host:doku,$applicationname,$environment}",
>               "aggregator": "avg"
>             }
>           ],
>           "formulas": [
>             {
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "dollar"
>                 }
>               },
>               "formula": "query1"
>             }
>           ]
>         }
>       ],
>       "autoscale": true,
>       "precision": 2,
>       "timeseries_background": {
>         "type": "bars"
>       }
>     },
>     "layout": {
>       "x": 9,
>       "y": 1,
>       "width": 3,
>       "height": 2
>     }
>   },
>   {
>     "id": 7163232112237060,
>     "definition": {
>       "title": "Requests",
>       "title_size": "16",
>       "title_align": "left",
>       "show_legend": false,
>       "legend_layout": "auto",
>       "legend_columns": [
>         "avg",
>         "min",
>         "max",
>         "value",
>         "sum"
>       ],
>       "type": "timeseries",
>       "requests": [
>         {
>           "formulas": [
>             {
>               "formula": "query1"
>             }
>           ],
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "count:doku.llm.request.duration{host:doku,$applicationname,$environment}"
>             }
>           ],
>           "response_format": "timeseries",
>           "style": {
>             "palette": "blue",
>             "line_type": "solid",
>             "line_width": "normal"
>           },
>           "display_type": "line"
>         }
>       ]
>     },
>     "layout": {
>       "x": 0,
>       "y": 3,
>       "width": 12,
>       "height": 3
>     }
>   },
>   {
>     "id": 6530509092714956,
>     "definition": {
>       "title": "Usage cost by Category",
>       "title_size": "16",
>       "title_align": "left",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "formulas": [
>             {
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "dollar"
>                 }
>               },
>               "formula": "query1",
>               "limit": {
>                 "count": 500,
>                 "order": "desc"
>               }
>             }
>           ],
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "sum:doku.llm.usage.cost{host:doku,$applicationname,$environment} by {generation}",
>               "aggregator": "avg"
>             }
>           ],
>           "style": {
>             "palette": "datadog16"
>           }
>         }
>       ],
>       "type": "sunburst",
>       "hide_total": true,
>       "legend": {
>         "type": "inline"
>       }
>     },
>     "layout": {
>       "x": 0,
>       "y": 6,
>       "width": 3,
>       "height": 3
>     }
>   },
>   {
>     "id": 8042116329904768,
>     "definition": {
>       "title": "Usage cost by Platform",
>       "title_size": "16",
>       "title_align": "left",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "formulas": [
>             {
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "dollar"
>                 }
>               },
>               "formula": "query1",
>               "limit": {
>                 "count": 500,
>                 "order": "desc"
>               }
>             }
>           ],
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "sum:doku.llm.usage.cost{host:doku,$applicationname,$environment} by {platform}",
>               "aggregator": "avg"
>             }
>           ],
>           "style": {
>             "palette": "datadog16"
>           }
>         }
>       ],
>       "type": "sunburst",
>       "hide_total": true,
>       "legend": {
>         "type": "inline"
>       }
>     },
>     "layout": {
>       "x": 3,
>       "y": 6,
>       "width": 3,
>       "height": 3
>     }
>   },
>   {
>     "id": 6701701162130436,
>     "definition": {
>       "title": "Usage cost by Application",
>       "title_size": "16",
>       "title_align": "left",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "formulas": [
>             {
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "dollar"
>                 }
>               },
>               "formula": "query1",
>               "limit": {
>                 "count": 500,
>                 "order": "desc"
>               }
>             }
>           ],
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "sum:doku.llm.usage.cost{host:doku,$applicationname,$environment} by {applicationname}",
>               "aggregator": "sum"
>             }
>           ],
>           "style": {
>             "palette": "datadog16"
>           }
>         }
>       ],
>       "type": "sunburst",
>       "hide_total": true,
>       "legend": {
>         "type": "inline"
>       }
>     },
>     "layout": {
>       "x": 6,
>       "y": 6,
>       "width": 3,
>       "height": 3
>     }
>   },
>   {
>     "id": 1437027828906050,
>     "definition": {
>       "title": "Usage cost by Environment",
>       "title_size": "16",
>       "title_align": "left",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "formulas": [
>             {
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "dollar"
>                 }
>               },
>               "formula": "query1",
>               "limit": {
>                 "count": 500,
>                 "order": "desc"
>               }
>             }
>           ],
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "sum:doku.llm.usage.cost{host:doku,$applicationname,$environment} by {environment}",
>               "aggregator": "avg"
>             }
>           ],
>           "style": {
>             "palette": "datadog16"
>           }
>         }
>       ],
>       "type": "sunburst",
>       "hide_total": true,
>       "legend": {
>         "type": "inline"
>       }
>     },
>     "layout": {
>       "x": 9,
>       "y": 6,
>       "width": 3,
>       "height": 3
>     }
>   },
>   {
>     "id": 2402383159288882,
>     "definition": {
>       "title": "Avg Prompt Tokens",
>       "title_size": "16",
>       "title_align": "left",
>       "type": "query_value",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "avg:doku.llm.prompt.tokens{host:doku,$applicationname,$environment}",
>               "aggregator": "avg"
>             }
>           ],
>           "conditional_formats": [
>             {
>               "comparator": "<=",
>               "value": 1000,
>               "palette": "white_on_green"
>             },
>             {
>               "comparator": ">",
>               "value": 1000,
>               "palette": "white_on_red"
>             }
>           ],
>           "formulas": [
>             {
>               "formula": "query1"
>             }
>           ]
>         }
>       ],
>       "autoscale": true,
>       "precision": 2,
>       "timeseries_background": {
>         "type": "area"
>       }
>     },
>     "layout": {
>       "x": 0,
>       "y": 9,
>       "width": 4,
>       "height": 1
>     }
>   },
>   {
>     "id": 1887750495626080,
>     "definition": {
>       "title": "Avg Token Usage",
>       "title_size": "16",
>       "title_align": "left",
>       "show_legend": true,
>       "legend_layout": "auto",
>       "legend_columns": [
>         "avg",
>         "min",
>         "max",
>         "value",
>         "sum"
>       ],
>       "type": "timeseries",
>       "requests": [
>         {
>           "formulas": [
>             {
>               "formula": "query1"
>             },
>             {
>               "formula": "query2"
>             }
>           ],
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "avg:doku.llm.prompt.tokens{host:doku,$applicationname,$environment}"
>             },
>             {
>               "data_source": "metrics",
>               "name": "query2",
>               "query": "avg:doku.llm.completion.tokens{host:doku,$applicationname,$environment}"
>             }
>           ],
>           "response_format": "timeseries",
>           "style": {
>             "palette": "dog_classic",
>             "line_type": "solid",
>             "line_width": "normal"
>           },
>           "display_type": "line"
>         }
>       ]
>     },
>     "layout": {
>       "x": 4,
>       "y": 9,
>       "width": 8,
>       "height": 2
>     }
>   },
>   {
>     "id": 8227251321537566,
>     "definition": {
>       "title": "Avg Completion Tokens",
>       "title_size": "16",
>       "title_align": "left",
>       "type": "query_value",
>       "requests": [
>         {
>           "response_format": "scalar",
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "avg:doku.llm.completion.tokens{host:doku,$applicationname,$environment}",
>               "aggregator": "avg"
>             }
>           ],
>           "conditional_formats": [
>             {
>               "comparator": "<=",
>               "value": 1000,
>               "palette": "white_on_green"
>             },
>             {
>               "comparator": ">",
>               "value": 1000,
>               "palette": "white_on_red"
>             }
>           ],
>           "formulas": [
>             {
>               "formula": "query1"
>             }
>           ]
>         }
>       ],
>       "autoscale": true,
>       "precision": 2,
>       "timeseries_background": {
>         "type": "area"
>       }
>     },
>     "layout": {
>       "x": 0,
>       "y": 10,
>       "width": 4,
>       "height": 1
>     }
>   },
>   {
>     "id": 3302397057575330,
>     "definition": {
>       "title": "Top Models by Usage",
>       "title_size": "16",
>       "title_align": "left",
>       "type": "toplist",
>       "requests": [
>         {
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "sum:doku.llm.usage.cost{host:doku,$applicationname,$environment} by {model}",
>               "aggregator": "avg"
>             }
>           ],
>           "formulas": [
>             {
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "dollar"
>                 }
>               },
>               "formula": "query1",
>               "limit": {
>                 "count": 5,
>                 "order": "desc"
>               }
>             }
>           ],
>           "response_format": "scalar"
>         }
>       ],
>       "style": {
>         "display": {
>           "type": "stacked",
>           "legend": "automatic"
>         }
>       }
>     },
>     "layout": {
>       "x": 0,
>       "y": 11,
>       "width": 4,
>       "height": 3
>     }
>   },
>   {
>     "id": 2328926103668336,
>     "definition": {
>       "title": "Avg Cost by Models",
>       "title_size": "16",
>       "title_align": "left",
>       "show_legend": true,
>       "legend_layout": "horizontal",
>       "legend_columns": [
>         "avg",
>         "min",
>         "max",
>         "value",
>         "sum"
>       ],
>       "type": "timeseries",
>       "requests": [
>         {
>           "formulas": [
>             {
>               "number_format": {
>                 "unit": {
>                   "type": "canonical_unit",
>                   "unit_name": "dollar"
>                 }
>               },
>               "formula": "query1"
>             }
>           ],
>           "queries": [
>             {
>               "data_source": "metrics",
>               "name": "query1",
>               "query": "avg:doku.llm.usage.cost{host:doku,$applicationname,$environment} by {model}"
>             }
>           ],
>           "response_format": "timeseries",
>           "style": {
>             "palette": "dog_classic",
>             "line_type": "solid",
>             "line_width": "normal"
>           },
>           "display_type": "line"
>         }
>       ]
>     },
>     "layout": {
>       "x": 4,
>       "y": 11,
>       "width": 8,
>       "height": 3
>     }
>   },
>   {
>     "id": 6390170780704056,
>     "definition": {
>       "title": "Prompt",
>       "title_size": "16",
>       "title_align": "left",
>       "requests": [
>         {
>           "response_format": "event_list",
>           "query": {
>             "data_source": "logs_stream",
>             "query_string": "host:doku type:prompt $applicationname $environment",
>             "indexes": [],
>             "storage": "hot"
>           },
>           "columns": [
>             {
>               "field": "status_line",
>               "width": "auto"
>             },
>             {
>               "field": "timestamp",
>               "width": "auto"
>             },
>             {
>               "field": "content",
>               "width": "compact"
>             }
>           ]
>         }
>       ],
>       "type": "list_stream"
>     },
>     "layout": {
>       "x": 0,
>       "y": 14,
>       "width": 6,
>       "height": 5
>     }
>   },
>   {
>     "id": 8605893614511422,
>     "definition": {
>       "title": "Response",
>       "title_size": "16",
>       "title_align": "left",
>       "requests": [
>         {
>           "response_format": "event_list",
>           "query": {
>             "data_source": "logs_stream",
>             "query_string": "host:doku $applicationname type:(response OR image) $environment",
>             "indexes": [],
>             "storage": "hot"
>           },
>           "columns": [
>             {
>               "field": "status_line",
>               "width": "auto"
>             },
>             {
>               "field": "timestamp",
>               "width": "auto"
>             },
>             {
>               "field": "content",
>               "width": "compact"
>             }
>           ]
>         }
>       ],
>       "type": "list_stream"
>     },
>     "layout": {
>       "x": 6,
>       "y": 14,
>       "width": 6,
>       "height": 5
>     }
>   }
> ],
> "template_variables": [
>   {
>     "name": "applicationname",
>     "prefix": "applicationname",
>     "available_values": [],
>     "default": "*"
>   },
>   {
>     "name": "environment",
>     "prefix": "environment",
>     "available_values": [],
>     "default": "*"
>   }
> ],
> "layout_type": "ordered",
> "notify_list": [],
> "reflow_type": "fixed"
> }
> ```

***

> **[Integrations](https://dokulabs.mintlify.site/0.1/integrations/introduction.md)**
>
> Integrate your LLM Provider with Doku

> **[API Reference](https://dokulabs.mintlify.site/0.1/api-reference/introduction.md)**
>
> OpenAPI specification for API Endpoints in Doku


This documentation is built and hosted on [Mintlify](https://mintlify.com), a developer documentation platform.