mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-03 12:50:06 +06:00
144852fb6b
* refactor: benchmarks Based on a discussion with @LysandreJik & @ArthurZucker, the goal of this PR is to improve transformers' benchmark system. This is a WIP, for the moment the infrastructure required to make things work is not ready. Will update the PR description when it is the case. * feat: add db init in benchmarks CI * fix: pg_config is missing in runner * fix: add psql to the runner * fix: connect info from env vars + PR comments * refactor: set database as env var * fix: invalid working directory * fix: `commit_msg` -> `commit_message` * fix: git marking checked out repo as unsafe * feat: add logging * fix: invalid device * feat: update grafana dashboard for prod grafana * feat: add `commit_id` to header table * feat: commit latest version of dashboard * feat: move measurements into json field * feat: remove drop table migration queries * fix: `torch.arrange` -> `torch.arange` * fix: add missing `s` to `cache_position` positional argument * fix: change model * revert: `cache_positions` -> `cache_position` * fix: set device for `StaticCache` * fix: set `StaticCache` dtype * feat: limit max cache len * fix script * raise error on failure! * not try catch * try to skip generate compilation * update * update docker image! * update * update again!@ * update * updates * ??? * ?? * use `torch.cuda.synchronize()` * fix json * nits * fix * fixed! * f**k * feat: add TTNT panels * feat: add try except --------- Co-authored-by: Arthur Zucker <arthur.zucker@gmail.com>
2212 lines
57 KiB
JSON
2212 lines
57 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 0,
|
|
"id": 1,
|
|
"links": [
|
|
{
|
|
"asDropdown": false,
|
|
"icon": "external link",
|
|
"includeVars": false,
|
|
"keepTime": false,
|
|
"tags": [],
|
|
"targetBlank": false,
|
|
"title": "Go to data",
|
|
"tooltip": "Go to data",
|
|
"type": "link",
|
|
"url": "http://transformers-benchmarks.huggingface.co/d/fdz33iyzln9c0a/transformers-benchmarks?orgId=1&from=${StartTime}&to=${EndTime}"
|
|
}
|
|
],
|
|
"liveNow": true,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"custom": {
|
|
"align": "left",
|
|
"cellOptions": {
|
|
"type": "auto"
|
|
},
|
|
"inspect": false
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "gpu_name"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 364
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "left"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 407
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "commit_message"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 708
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "commit_id"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 388
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 5,
|
|
"options": {
|
|
"cellHeight": "sm",
|
|
"footer": {
|
|
"countRows": false,
|
|
"fields": "",
|
|
"reducer": [
|
|
"sum"
|
|
],
|
|
"show": false
|
|
},
|
|
"showHeader": true,
|
|
"sortBy": []
|
|
},
|
|
"pluginVersion": "11.2.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT commit_id as commit_id, commit_message, gpu_name FROM benchmarks WHERE branch = ${branch};",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "commit_id",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_name",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baaa8aaa-89ab-4cde-b012-31922f96de3f",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "benchmarks"
|
|
}
|
|
],
|
|
"transparent": true,
|
|
"type": "table"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"id": 13,
|
|
"panels": [],
|
|
"title": "Eager Forward Pass",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"id": 7,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.3,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"pluginVersion": "11.2.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'first_eager_forward_pass_time_secs' AS double precision) AS first_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "First eager forward pass",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"id": 9,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'second_eager_forward_pass_time_secs' AS double precision) AS second_eager_forward_pass_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Second eager forward pass",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 18
|
|
},
|
|
"id": 16,
|
|
"panels": [],
|
|
"title": "Time to next token",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 19
|
|
},
|
|
"id": 17,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'time_to_first_token_secs' AS double precision) AS time_to_first_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Time to first token",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 19
|
|
},
|
|
"id": 18,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'time_to_second_token_secs' AS double precision) AS time_to_second_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Time to second token",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 30
|
|
},
|
|
"id": 19,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'time_to_third_token_secs' AS double precision) AS time_to_third_token_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Time to third token",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 30
|
|
},
|
|
"id": 20,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'time_to_next_token_mean_secs' AS double precision) AS time_to_next_token_mean_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Time to subsequent next tokens mean",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"id": 14,
|
|
"panels": [],
|
|
"title": "Compiled Generate",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 42
|
|
},
|
|
"id": 8,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "always",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'first_compile_generate_time_secs' AS double precision) AS first_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}'",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "First compile generate",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 42
|
|
},
|
|
"id": 10,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'second_compile_generate_time_secs' AS double precision) AS second_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}';",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Second compile generate",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "scheme",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 53
|
|
},
|
|
"id": 11,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'third_compile_generate_time_secs' AS double precision) AS third_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}';",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Third compile generate",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-YlBl"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineWidth": 0,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 11,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 53
|
|
},
|
|
"id": 12,
|
|
"options": {
|
|
"barRadius": 0.05,
|
|
"barWidth": 0.8,
|
|
"fullHighlight": false,
|
|
"groupWidth": 0.7,
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"orientation": "auto",
|
|
"showValue": "auto",
|
|
"stacking": "none",
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
},
|
|
"xTickLabelRotation": 0,
|
|
"xTickLabelSpacing": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT CAST(m.measurements->'fourth_compile_generate_time_secs' AS double precision) AS fourth_compile_generate_time_secs, left(b.commit_id, 7), m.time FROM benchmarks as b JOIN model_measurements AS m ON b.benchmark_id = m.benchmark_id WHERE b.branch = ${branch} AND gpu_name = '${gpu_name}';",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50
|
|
}
|
|
}
|
|
],
|
|
"title": "Fourth compile generate",
|
|
"transparent": true,
|
|
"type": "barchart"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 64
|
|
},
|
|
"id": 15,
|
|
"panels": [],
|
|
"title": "Usage metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": 60000,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 65
|
|
},
|
|
"id": 1,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT\n d.cpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = ${branch}",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "cpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "\"time\"",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baa888b8-89ab-4cde-b012-31922f8671e9",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "measurements"
|
|
}
|
|
],
|
|
"title": "CPU Utilization",
|
|
"transparent": true,
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": 60000,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 65
|
|
},
|
|
"id": 4,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT\n b.commit_id,\n d.gpu_util,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = ${branch}",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "cpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "\"time\"",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baa888b8-89ab-4cde-b012-31922f8671e9",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "measurements"
|
|
}
|
|
],
|
|
"title": "GPU Utilization",
|
|
"transparent": true,
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": 60000,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "decmbytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 74
|
|
},
|
|
"id": 2,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT d.mem_megabytes, d.time FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = ${branch}",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "cpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "\"time\"",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baa888b8-89ab-4cde-b012-31922f8671e9",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "measurements"
|
|
}
|
|
],
|
|
"title": "Memory usage",
|
|
"transparent": true,
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": true,
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": 60000,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "decmbytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 74
|
|
},
|
|
"id": 3,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "bdz2yss7sxo1sc"
|
|
},
|
|
"editorMode": "code",
|
|
"format": "table",
|
|
"rawQuery": true,
|
|
"rawSql": "SELECT\n d.gpu_mem_megabytes,\n d.time\nFROM\n benchmarks AS b\n JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id\nWHERE\n branch = ${branch}",
|
|
"refId": "A",
|
|
"sql": {
|
|
"columns": [
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "cpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_util",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "gpu_mem_megabytes",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
},
|
|
{
|
|
"parameters": [
|
|
{
|
|
"name": "\"time\"",
|
|
"type": "functionParameter"
|
|
}
|
|
],
|
|
"type": "function"
|
|
}
|
|
],
|
|
"groupBy": [
|
|
{
|
|
"property": {
|
|
"type": "string"
|
|
},
|
|
"type": "groupBy"
|
|
}
|
|
],
|
|
"limit": 50,
|
|
"whereJsonTree": {
|
|
"children1": [
|
|
{
|
|
"id": "baa888b8-89ab-4cde-b012-31922f8671e9",
|
|
"properties": {
|
|
"field": "commit_id",
|
|
"fieldSrc": "field",
|
|
"operator": "equal",
|
|
"value": [
|
|
"${commit}"
|
|
],
|
|
"valueError": [
|
|
null
|
|
],
|
|
"valueSrc": [
|
|
"value"
|
|
],
|
|
"valueType": [
|
|
"text"
|
|
]
|
|
},
|
|
"type": "rule"
|
|
}
|
|
],
|
|
"id": "bab88a98-0123-4456-b89a-b1922f7d4f11",
|
|
"type": "group"
|
|
},
|
|
"whereString": "commit_id = '${commit}'"
|
|
},
|
|
"table": "measurements"
|
|
}
|
|
],
|
|
"title": "GPU memory usage",
|
|
"transparent": true,
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"schemaVersion": 39,
|
|
"tags": [],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": true,
|
|
"text": [
|
|
"refactor/benchmarks"
|
|
],
|
|
"value": [
|
|
"refactor/benchmarks"
|
|
]
|
|
},
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"definition": "SELECT DISTINCT branch FROM benchmarks;",
|
|
"description": "",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "branch",
|
|
"multi": false,
|
|
"name": "branch",
|
|
"options": [],
|
|
"query": "SELECT DISTINCT branch FROM benchmarks;",
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "1728570853117",
|
|
"value": "1728570853117"
|
|
},
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"definition": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = ${branch} ORDER BY benchmark_id ASC LIMIT 1;",
|
|
"description": "",
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "StartTime",
|
|
"options": [],
|
|
"query": "SELECT created_at - INTERVAL '5 secs' FROM benchmarks WHERE branch = ${branch} ORDER BY benchmark_id ASC LIMIT 1;",
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "1728657828802",
|
|
"value": "1728657828802"
|
|
},
|
|
"datasource": {
|
|
"type": "grafana-postgresql-datasource",
|
|
"uid": "de0dbhs18ho1sc"
|
|
},
|
|
"definition": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = ${branch} ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;",
|
|
"description": "",
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "EndTime",
|
|
"options": [],
|
|
"query": "SELECT time + INTERVAL '5 secs' FROM benchmarks AS b JOIN device_measurements AS d ON b.benchmark_id = d.benchmark_id WHERE branch = ${branch} ORDER BY b.benchmark_id DESC, d.measurement_id DESC LIMIT 1;",
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "NVIDIA A10G",
|
|
"value": "NVIDIA A10G"
|
|
},
|
|
"definition": "SELECT DISTINCT gpu_name FROM benchmarks;",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "GPU",
|
|
"multi": false,
|
|
"name": "gpu_name",
|
|
"options": [],
|
|
"query": "SELECT DISTINCT gpu_name FROM benchmarks;",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "2024-10-11T13:10:01.641Z",
|
|
"to": "2024-10-11T13:25:21.783Z"
|
|
},
|
|
"timepicker": {
|
|
"hidden": false
|
|
},
|
|
"timezone": "browser",
|
|
"title": "Transformers benchmarks",
|
|
"uid": "fdz33iyzln9c0a",
|
|
"version": 9,
|
|
"weekStart": ""
|
|
}
|