New Relic

LLM Observability with New Relic and Doku

Doku can automatically send LLM usage data from Doku to New Relic using the Report Metrics HTTP API and Send Logs API

Make the following curl request to Doku to add a connection for New Relic. See more on the Connections API

curl --request POST \
  --url https://<YOUR_DOKU_INGESTER_URL>/api/connections \
  --header 'Authorization: <DOKU_API_KEY>' \
  --header 'Content-Type: application/json' \
  --data '{
  "Platform": "newrelic",
  "MetricsUrl": "<YOUR_NEWRELIC_METRICS_URL>",
  "LogsUrl": "<YOUR_NEWRELIC_LOGS_URL>",
  "ApiKey": "<YOUR_NEWRELIC_API_KEY>"
}'

New Relic Resources

Dashboard

{
"name": "Doku - LLM Monitoring",
"description": null,
"permissions": "PUBLIC_READ_WRITE",
"pages": [
  {
    "name": "Doku - LLM Monitoring",
    "description": null,
    "widgets": [
      {
        "title": "",
        "layout": {
          "column": 1,
          "row": 1,
          "width": 1,
          "height": 1
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.markdown"
        },
        "rawConfiguration": {
          "text": "![doku-image](https://github.com/dokulabs/.github/blob/main/profile/assets/favicon.png?raw=true )"
        }
      },
      {
        "title": "",
        "layout": {
          "column": 2,
          "row": 1,
          "width": 2,
          "height": 1
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.billboard"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT COUNT(doku.LLM.Request.Duration) AS 'Total Requests' FROM Metric WHERE job.name='doku'"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Seconds",
        "layout": {
          "column": 4,
          "row": 1,
          "width": 2,
          "height": 1
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.billboard"
        },
        "rawConfiguration": {
          "dataFormatters": [],
          "facet": {
            "showOtherSeries": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT average(doku.LLM.Request.Duration) AS 'Avg Request Duration' FROM Metric"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "",
        "layout": {
          "column": 6,
          "row": 1,
          "width": 2,
          "height": 1
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.billboard"
        },
        "rawConfiguration": {
          "dataFormatters": [],
          "facet": {
            "showOtherSeries": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT AVERAGE(doku.LLM.Total.Tokens) AS 'Average Tokens / Request' FROM Metric"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "",
        "layout": {
          "column": 8,
          "row": 1,
          "width": 2,
          "height": 1
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.billboard"
        },
        "rawConfiguration": {
          "dataFormatters": [],
          "facet": {
            "showOtherSeries": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT AVERAGE(doku.LLM.Usage.Cost) AS 'Average Cost / Request' FROM Metric"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "",
        "layout": {
          "column": 10,
          "row": 1,
          "width": 3,
          "height": 1
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.billboard"
        },
        "rawConfiguration": {
          "dataFormatters": [],
          "facet": {
            "showOtherSeries": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT SUM(doku.LLM.Usage.Cost) AS 'Average Cost / Request' FROM Metric"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Total Request ",
        "layout": {
          "column": 1,
          "row": 2,
          "width": 12,
          "height": 2
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.area"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": false
          },
          "legend": {
            "enabled": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT count(doku.LLM.Request.Duration) FROM Metric TIMESERIES"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Usage Cost by Category",
        "layout": {
          "column": 1,
          "row": 4,
          "width": 6,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.pie"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": true
          },
          "legend": {
            "enabled": true
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT SUM(doku.LLM.Usage.Cost AS 'Generation')  FROM Metric FACET generation\n"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Usage cost by Platform",
        "layout": {
          "column": 7,
          "row": 4,
          "width": 6,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.pie"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": true
          },
          "legend": {
            "enabled": true
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT sum(doku.LLM.Usage.Cost AS 'Platform')  FROM Metric FACET platform\n"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Usage Cost by Environment",
        "layout": {
          "column": 1,
          "row": 7,
          "width": 6,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.pie"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": true
          },
          "legend": {
            "enabled": true
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT sum(doku.LLM.Usage.Cost AS 'Environment')  FROM Metric FACET environment\n"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Usage Cost by Application",
        "layout": {
          "column": 7,
          "row": 7,
          "width": 6,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.pie"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": true
          },
          "legend": {
            "enabled": true
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT SUM(doku.LLM.Usage.Cost AS 'Applications')  FROM Metric FACET applicationName\n"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Average Tokens ",
        "layout": {
          "column": 1,
          "row": 10,
          "width": 12,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.stacked-bar"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": false
          },
          "legend": {
            "enabled": true
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT average(doku.LLM.Prompt.Tokens), average(doku.LLM.Completion.Tokens) FROM Metric TIMESERIES"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          },
          "units": {
            "unit": "COUNT"
          }
        }
      },
      {
        "title": "",
        "layout": {
          "column": 1,
          "row": 13,
          "width": 6,
          "height": 1
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.billboard"
        },
        "rawConfiguration": {
          "dataFormatters": [],
          "facet": {
            "showOtherSeries": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT AVERAGE(doku.LLM.Prompt.Tokens) AS 'Average Prompt Tokens / Request' FROM Metric"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "",
        "layout": {
          "column": 7,
          "row": 13,
          "width": 6,
          "height": 1
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.billboard"
        },
        "rawConfiguration": {
          "dataFormatters": [],
          "facet": {
            "showOtherSeries": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT AVERAGE(doku.LLM.Completion.Tokens) AS 'Average Completion Tokens / Request' FROM Metric"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Top Models by Usage",
        "layout": {
          "column": 1,
          "row": 14,
          "width": 4,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.bar"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": false
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT COUNT(doku.LLM.Request.Duration AS 'Generation')  FROM Metric FACET model\n"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          }
        }
      },
      {
        "title": "Average Cost / Model",
        "layout": {
          "column": 5,
          "row": 14,
          "width": 8,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "viz.area"
        },
        "rawConfiguration": {
          "facet": {
            "showOtherSeries": false
          },
          "legend": {
            "enabled": true
          },
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT AVERAGE(doku.LLM.Usage.Cost AS 'Generation')  FROM Metric FACET model TIMESERIES\n"
            }
          ],
          "platformOptions": {
            "ignoreTimeRange": false
          },
          "units": {
            "unit": "APDEX"
          }
        }
      },
      {
        "title": "Prompts",
        "layout": {
          "column": 1,
          "row": 17,
          "width": 6,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "logger.log-table-widget"
        },
        "rawConfiguration": {
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT timestamp, message FROM Log WHERE job.name='doku' AND type='prompt'"
            }
          ]
        }
      },
      {
        "title": "Response's",
        "layout": {
          "column": 7,
          "row": 17,
          "width": 6,
          "height": 3
        },
        "linkedEntityGuids": null,
        "visualization": {
          "id": "logger.log-table-widget"
        },
        "rawConfiguration": {
          "nrqlQueries": [
            {
              "accountIds": [
                4060909
              ],
              "query": "SELECT timestamp, message FROM Log WHERE job.name='doku' AND type='response' OR type='image'"
            }
          ]
        }
      }
    ]
  }
],
"variables": []
}

---

Integrations

Integrate your LLM Provider with Doku

API Reference

OpenAPI specification for API Endpoints in Doku