mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-03 04:40:06 +06:00
9a94dfe123
* feat: add `benchmarks_entrypoint.py` Adding `benchmarks_entrypoint.py` file, which will be run from the benchmarks CI. This python script will list all python files from the `benchmark/` folder and run the included `run_benchmark` function, allowing people to add new benchmarks scripts. * feat: add `MetricsRecorder` * feat: update dashboard * fix: add missing arguments to `MetricsRecorder` * feat: update dash & add datasource + `default.yml` * fix: move responsibility to create `MetricsRecorder` in bench script * fix: update incorrect datasource UID * fix: incorrect variable values * debug: benchmark entrypoint script * refactor: update log level * fix: update broken import * feat: add debug log in `MetricsRecorder` * debug: set log level to debug * fix: set connection `autocommit` to `True`
2376 lines
64 KiB
JSON
2376 lines
64 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 0,
|
|
"id": 1,
|
|
"links": [
|
|
{
|
|
"asDropdown": false,
|
|
"icon": "external link",
|
|
"includeVars": false,
|
|
"keepTime": false,
|
|
"tags": [],
|
|
"targetBlank": false,
|
|
"title": "Go to data",
|
|
"tooltip": "Go to data",
|
|
"type": "link",
|
|
"url": "http://transformers-benchmarks.hf.co/d/fdz33iyzln9c0a/transformers-benchmarks?orgId=1&from=${StartTime}&to=${EndTime}"
|
|
}
|
|
],
|
|
"liveNow": true,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"custom": {
|
|
"align": "left",
|
|
"cellOptions": {
|
|
"type": "auto"
|
|
},
|
|
"inspect": false
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "gpu_name"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 202
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "left"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 407
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "commit_message"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 524
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "commit_id"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 353
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "model_id"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 216
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 5,
|
|
"options": {
|
|
"cellHeight": "sm",
|
|
"footer": {
|
|
"countRows": false,
|
|
"fields": "",
|
|
"reducer": [
|
|
"sum"
|
|
],
|
|
"show": false
|
|
},
|
|
"showHeader": true,
|
|
"sortBy": []
|
|
},
|
|
"pluginVersion": "11.2.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT commit_id, commit_message, metadata->>'gpu_name' as gpu_name, metadata->>'model_id' as model_id, created_at AS date FROM benchmarks WHERE branch = '${branch}' AND metadata->>'gpu_name' = '${gpu_name}' ORDER BY benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "commit_id",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_name",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baaa8aaa-89ab-4cde-b012-31922f96de3f",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "benchmarks"
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "table"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"id": 13,
|
|
"panels": [],
|
|
"title": "Eager Forward Pass",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"id": 7,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"pluginVersion": "11.2.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'first_eager_forward_pass_time_secs' AS double precision) AS first_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "First eager forward pass",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"id": 9,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'second_eager_forward_pass_time_secs' AS double precision) AS second_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Second eager forward pass",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 18
|
|
},
|
|
"id": 16,
|
|
"panels": [],
|
|
"title": "Time to next token",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 19
|
|
},
|
|
"id": 17,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'time_to_first_token_secs' AS double precision) AS time_to_first_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Time to first token",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 19
|
|
},
|
|
"id": 18,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'time_to_second_token_secs' AS double precision) AS time_to_second_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Time to second token",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 30
|
|
},
|
|
"id": 19,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'time_to_third_token_secs' AS double precision) AS time_to_third_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Time to third token",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 30
|
|
},
|
|
"id": 20,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'time_to_next_token_mean_secs' AS double precision) AS time_to_next_token_mean_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Time to subsequent next tokens mean",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"id": 14,
|
|
"panels": [],
|
|
"title": "Compiled Generate",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 42
|
|
},
|
|
"id": 8,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'first_compile_generate_time_secs' AS double precision) AS first_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "First compile generate",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 42
|
|
},
|
|
"id": 10,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'second_compile_generate_time_secs' AS double precision) AS second_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Second compile generate",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 53
|
|
},
|
|
"id": 11,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'third_compile_generate_time_secs' AS double precision) AS third_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Third compile generate",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 53
|
|
},
|
|
"id": 12,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'fourth_compile_generate_time_secs' AS double precision) AS fourth_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = '${branch}' AND b.metadata->>'gpu_name' = '${gpu_name}' ORDER BY b.benchmark_id DESC LIMIT ${last_n_commits};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Fourth compile generate",
|
|
"transformations": [
|
|
{
|
|
"id": "sortBy",
|
|
"options": {
|
|
"fields": {},
|
|
"sort": [
|
|
{
|
|
"field": "time"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 64
|
|
},
|
|
"id": 15,
|
|
"panels": [
|
|
{
|
|
"datasource": {},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": 60000,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 65
|
|
},
|
|
"id": 1,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT\n d.cpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}';",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "cpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "\"time\"",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baa888b8-89ab-4cde-b012-31922f8671e9",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "measurements"
|
|
}
|
|
],
|
|
"title": "CPU Utilization",
|
|
"transparent": true,
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": 60000,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 65
|
|
},
|
|
"id": 4,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT\n b.commit_id,\n d.gpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}';",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "cpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "\"time\"",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baa888b8-89ab-4cde-b012-31922f8671e9",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "measurements"
|
|
}
|
|
],
|
|
"title": "GPU Utilization",
|
|
"transparent": true,
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": 60000,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "decmbytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 74
|
|
},
|
|
"id": 2,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT d.mem_megabytes, d.time FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}';",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "cpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "\"time\"",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baa888b8-89ab-4cde-b012-31922f8671e9",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "measurements"
|
|
}
|
|
],
|
|
"title": "Memory usage",
|
|
"transparent": true,
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": 60000,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "decmbytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 74
|
|
},
|
|
"id": 3,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT\n d.gpu_mem_megabytes,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = '${branch}';",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "cpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "\"time\"",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baa888b8-89ab-4cde-b012-31922f8671e9",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "measurements"
|
|
}
|
|
],
|
|
"title": "GPU memory usage",
|
|
"transparent": true,
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Usage metrics",
|
|
"type": "row"
|
|
}
|
|
],
|
|
"schemaVersion": 39,
|
|
"tags": [],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "main",
|
|
"value": "main"
|
|
},
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"definition": "SELECT DISTINCT branch FROM benchmarks;",
|
|
"description": "",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "branch",
|
|
"multi": false,
|
|
"name": "branch",
|
|
"options": [],
|
|
"query": "SELECT DISTINCT branch FROM benchmarks;",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "1729701492845",
|
|
"value": "1729701492845"
|
|
},
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"definition": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = '${branch}' ORDER BY benchmark_id ASC LIMIT 1;",
|
|
"description": "",
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "StartTime",
|
|
"options": [],
|
|
"query": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = '${branch}' ORDER BY benchmark_id ASC LIMIT 1;",
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "1730393397577",
|
|
"value": "1730393397577"
|
|
},
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"definition": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}' ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;",
|
|
"description": "",
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "EndTime",
|
|
"options": [],
|
|
"query": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = '${branch}' ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "NVIDIA A10G",
|
|
"value": "NVIDIA A10G"
|
|
},
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "be28nkzirtb0gd"
|
|
},
|
|
"definition": "SELECT DISTINCT metadata->>'gpu_name' FROM benchmarks;",
|
|
"description": "",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "GPU",
|
|
"multi": false,
|
|
"name": "gpu_name",
|
|
"options": [],
|
|
"query": "SELECT DISTINCT metadata->>'gpu_name' FROM benchmarks;",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": true,
|
|
"text": "10",
|
|
"value": "10"
|
|
},
|
|
"description": "The number of commits to display, going from most recent to the nth commit.",
|
|
"hide": 0,
|
|
"label": "Last # of commits",
|
|
"name": "last_n_commits",
|
|
"options": [
|
|
{
|
|
"selected": true,
|
|
"text": "10",
|
|
"value": "10"
|
|
}
|
|
],
|
|
"query": "10",
|
|
"skipUrlSync": false,
|
|
"type": "textbox"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"hidden": false
|
|
},
|
|
"timezone": "browser",
|
|
"title": "Transformers benchmarks",
|
|
"uid": "fdz33iyzln9c0a",
|
|
"version": 10,
|
|
"weekStart": ""
|
|
}
|