{ "annotations": { "list": [] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "links": [], "schemaVersion": 39, "tags": [ "hpc", "overview", "slurm", "gpu" ], "templating": { "list": [] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "HPC Overview", "uid": "hpc-overview", "version": 1, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "title": "Node: node01 \u00b7 System Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 4, "x": 0, "y": 1 }, "id": 101, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "Disk Usage (/)", "type": "gauge", "targets": [ { "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\",instance=\"node01:9100\"} / node_filesystem_size_bytes{mountpoint=\"/\",instance=\"node01:9100\"}) * 100", "legendFormat": "Disk" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 3, "x": 4, "y": 1 }, "id": 102, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Disk Used (/)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/\",instance=\"node01:9100\"} - node_filesystem_avail_bytes{mountpoint=\"/\",instance=\"node01:9100\"}", "legendFormat": "Used" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 3, "w": 3, "x": 4, "y": 4 }, "id": 103, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Disk Total (/)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/\",instance=\"node01:9100\"}", "legendFormat": "Total" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 4, "x": 7, "y": 1 }, "id": 104, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "Disk Usage (/scratch)", "type": "gauge", "targets": [ { "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/scratch\",instance=\"node01:9100\"} / node_filesystem_size_bytes{mountpoint=\"/scratch\",instance=\"node01:9100\"}) * 100", "legendFormat": "Disk /scratch" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 3, "x": 11, "y": 1 }, "id": 105, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Disk Used (/scratch)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/scratch\",instance=\"node01:9100\"} - node_filesystem_avail_bytes{mountpoint=\"/scratch\",instance=\"node01:9100\"}", "legendFormat": "Used" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 3, "w": 3, "x": 11, "y": 4 }, "id": 106, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Disk Total (/scratch)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/scratch\",instance=\"node01:9100\"}", "legendFormat": "Total" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 3, "x": 17, "y": 1 }, "id": 107, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "RAM Usage", "type": "gauge", "targets": [ { "expr": "(1 - node_memory_MemAvailable_bytes{instance=\"node01:9100\"} / node_memory_MemTotal_bytes{instance=\"node01:9100\"}) * 100", "legendFormat": "RAM" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 3, "x": 14, "y": 1 }, "id": 108, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "CPU Usage", "type": "gauge", "targets": [ { "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\",instance=\"node01:9100\"}[5m])) * 100)", "legendFormat": "CPU" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 4 }, "id": 109, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Total RAM", "type": "stat", "targets": [ { "expr": "node_memory_MemTotal_bytes{instance=\"node01:9100\"}", "legendFormat": "Total" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 1 }, "id": 110, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Total CPU Cores", "type": "stat", "targets": [ { "expr": "count(node_cpu_seconds_total{mode=\"idle\",instance=\"node01:9100\"})", "legendFormat": "Cores" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "percent", "min": 0 } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "id": 111, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "CPU & RAM Usage Over Time", "type": "timeseries", "targets": [ { "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\",instance=\"node01:9100\"}[5m])) * 100)", "legendFormat": "CPU %" }, { "expr": "(1 - node_memory_MemAvailable_bytes{instance=\"node01:9100\"} / node_memory_MemTotal_bytes{instance=\"node01:9100\"}) * 100", "legendFormat": "RAM %" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "Bps" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "id": 112, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "Network Traffic", "type": "timeseries", "targets": [ { "expr": "rate(node_network_receive_bytes_total{device!~\"lo|docker.*|br.*|veth.*\",instance=\"node01:9100\"}[5m])", "legendFormat": "{{ device }} RX" }, { "expr": "-rate(node_network_transmit_bytes_total{device!~\"lo|docker.*|br.*|veth.*\",instance=\"node01:9100\"}[5m])", "legendFormat": "{{ device }} TX" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 120, "title": "Node: node01 \u00b7 GPU", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 16 }, "id": 121, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "GPU Utilization", "type": "gauge", "targets": [ { "expr": "nvidia_smi_utilization_gpu_ratio{instance=\"node01:9835\"} * 100", "legendFormat": "GPU {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 16 }, "id": 122, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "GPU Memory Usage", "type": "gauge", "targets": [ { "expr": "(nvidia_smi_memory_used_bytes{instance=\"node01:9835\"} / nvidia_smi_memory_total_bytes{instance=\"node01:9835\"}) * 100", "legendFormat": "VRAM {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "celsius", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 85 } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 18, "y": 16 }, "id": 123, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "GPU Temperature", "type": "stat", "targets": [ { "expr": "nvidia_smi_temperature_gpu{instance=\"node01:9835\"}", "legendFormat": "GPU {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "watt", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 250 }, { "color": "red", "value": 350 } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 18, "y": 19 }, "id": 124, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "GPU Power Draw", "type": "stat", "targets": [ { "expr": "nvidia_smi_power_draw_watts{instance=\"node01:9835\"}", "legendFormat": "GPU {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 16 }, "id": 125, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "GPU Memory Used", "type": "stat", "targets": [ { "expr": "nvidia_smi_memory_used_bytes{instance=\"node01:9835\"}", "legendFormat": "{{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 19 }, "id": 126, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "GPU Memory Total", "type": "stat", "targets": [ { "expr": "nvidia_smi_memory_total_bytes{instance=\"node01:9835\"}", "legendFormat": "{{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "percent", "min": 0 } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, "id": 127, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "GPU Utilization Over Time", "type": "timeseries", "targets": [ { "expr": "nvidia_smi_utilization_gpu_ratio{instance=\"node01:9835\"} * 100", "legendFormat": "GPU {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "bytes" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, "id": 128, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "GPU Memory Over Time", "type": "timeseries", "targets": [ { "expr": "nvidia_smi_memory_used_bytes{instance=\"node01:9835\"}", "legendFormat": "Used {{ uuid }}" }, { "expr": "nvidia_smi_memory_total_bytes{instance=\"node01:9835\"}", "legendFormat": "Total {{ uuid }}" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, "id": 300, "title": "Node: node02 \u00b7 System Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 4, "x": 0, "y": 31 }, "id": 301, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "Disk Usage (/)", "type": "gauge", "targets": [ { "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\",instance=\"node02:9100\"} / node_filesystem_size_bytes{mountpoint=\"/\",instance=\"node02:9100\"}) * 100", "legendFormat": "Disk" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 3, "x": 4, "y": 31 }, "id": 302, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Disk Used (/)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/\",instance=\"node02:9100\"} - node_filesystem_avail_bytes{mountpoint=\"/\",instance=\"node02:9100\"}", "legendFormat": "Used" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 3, "w": 3, "x": 4, "y": 34 }, "id": 303, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Disk Total (/)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/\",instance=\"node02:9100\"}", "legendFormat": "Total" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 4, "x": 7, "y": 31 }, "id": 304, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "Disk Usage (/scratch)", "type": "gauge", "targets": [ { "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/scratch\",instance=\"node02:9100\"} / node_filesystem_size_bytes{mountpoint=\"/scratch\",instance=\"node02:9100\"}) * 100", "legendFormat": "Disk /scratch" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 3, "x": 11, "y": 31 }, "id": 305, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Disk Used (/scratch)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/scratch\",instance=\"node02:9100\"} - node_filesystem_avail_bytes{mountpoint=\"/scratch\",instance=\"node02:9100\"}", "legendFormat": "Used" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 3, "w": 3, "x": 11, "y": 34 }, "id": 306, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Disk Total (/scratch)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/scratch\",instance=\"node02:9100\"}", "legendFormat": "Total" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 3, "x": 17, "y": 31 }, "id": 307, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "RAM Usage", "type": "gauge", "targets": [ { "expr": "(1 - node_memory_MemAvailable_bytes{instance=\"node02:9100\"} / node_memory_MemTotal_bytes{instance=\"node02:9100\"}) * 100", "legendFormat": "RAM" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 3, "x": 14, "y": 31 }, "id": 308, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "CPU Usage", "type": "gauge", "targets": [ { "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\",instance=\"node02:9100\"}[5m])) * 100)", "legendFormat": "CPU" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 34 }, "id": 309, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Total RAM", "type": "stat", "targets": [ { "expr": "node_memory_MemTotal_bytes{instance=\"node02:9100\"}", "legendFormat": "Total" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 31 }, "id": 310, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Total CPU Cores", "type": "stat", "targets": [ { "expr": "count(node_cpu_seconds_total{mode=\"idle\",instance=\"node02:9100\"})", "legendFormat": "Cores" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "percent", "min": 0 } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, "id": 311, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "CPU & RAM Usage Over Time", "type": "timeseries", "targets": [ { "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\",instance=\"node02:9100\"}[5m])) * 100)", "legendFormat": "CPU %" }, { "expr": "(1 - node_memory_MemAvailable_bytes{instance=\"node02:9100\"} / node_memory_MemTotal_bytes{instance=\"node02:9100\"}) * 100", "legendFormat": "RAM %" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "Bps" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, "id": 312, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "Network Traffic", "type": "timeseries", "targets": [ { "expr": "rate(node_network_receive_bytes_total{device!~\"lo|docker.*|br.*|veth.*\",instance=\"node02:9100\"}[5m])", "legendFormat": "{{ device }} RX" }, { "expr": "-rate(node_network_transmit_bytes_total{device!~\"lo|docker.*|br.*|veth.*\",instance=\"node02:9100\"}[5m])", "legendFormat": "{{ device }} TX" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, "id": 320, "title": "Node: node02 \u00b7 GPU", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 46 }, "id": 321, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "GPU Utilization", "type": "gauge", "targets": [ { "expr": "nvidia_smi_utilization_gpu_ratio{instance=\"node02:9835\"} * 100", "legendFormat": "GPU {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 46 }, "id": 322, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "GPU Memory Usage", "type": "gauge", "targets": [ { "expr": "(nvidia_smi_memory_used_bytes{instance=\"node02:9835\"} / nvidia_smi_memory_total_bytes{instance=\"node02:9835\"}) * 100", "legendFormat": "VRAM {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "celsius", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 85 } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 18, "y": 46 }, "id": 323, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "GPU Temperature", "type": "stat", "targets": [ { "expr": "nvidia_smi_temperature_gpu{instance=\"node02:9835\"}", "legendFormat": "GPU {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "watt", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 250 }, { "color": "red", "value": 350 } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 18, "y": 49 }, "id": 324, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "GPU Power Draw", "type": "stat", "targets": [ { "expr": "nvidia_smi_power_draw_watts{instance=\"node02:9835\"}", "legendFormat": "GPU {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 46 }, "id": 325, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "GPU Memory Used", "type": "stat", "targets": [ { "expr": "nvidia_smi_memory_used_bytes{instance=\"node02:9835\"}", "legendFormat": "{{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 49 }, "id": 326, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "GPU Memory Total", "type": "stat", "targets": [ { "expr": "nvidia_smi_memory_total_bytes{instance=\"node02:9835\"}", "legendFormat": "{{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "percent", "min": 0 } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, "id": 327, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "GPU Utilization Over Time", "type": "timeseries", "targets": [ { "expr": "nvidia_smi_utilization_gpu_ratio{instance=\"node02:9835\"} * 100", "legendFormat": "GPU {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "bytes" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, "id": 328, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "GPU Memory Over Time", "type": "timeseries", "targets": [ { "expr": "nvidia_smi_memory_used_bytes{instance=\"node02:9835\"}", "legendFormat": "Used {{ uuid }}" }, { "expr": "nvidia_smi_memory_total_bytes{instance=\"node02:9835\"}", "legendFormat": "Total {{ uuid }}" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 60 }, "id": 400, "options": { "content": "", "mode": "markdown" }, "title": "NAS Storage \u00b7 /home (shared)", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "red", "value": 85 } ] }, "unit": "percent", "min": 0, "max": 100 } }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 61 }, "id": 401, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "title": "NAS Disk Usage (/home)", "type": "gauge", "targets": [ { "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/home\",instance=\"node01:9100\"} / node_filesystem_size_bytes{mountpoint=\"/home\",instance=\"node01:9100\"}) * 100", "legendFormat": "NAS /home" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 61 }, "id": 402, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "NAS Used (/home)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/home\",instance=\"node01:9100\"} - node_filesystem_avail_bytes{mountpoint=\"/home\",instance=\"node01:9100\"}", "legendFormat": "Used" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 64 }, "id": 403, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "NAS Total (/home)", "type": "stat", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/home\",instance=\"node01:9100\"}", "legendFormat": "Total" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 1, "fillOpacity": 10 } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 61 }, "id": 404, "options": { "tooltip": { "mode": "single" }, "legend": { "displayMode": "list", "placement": "bottom" } }, "title": "NAS Usage Over Time (/home)", "type": "timeseries", "targets": [ { "expr": "node_filesystem_size_bytes{mountpoint=\"/home\",instance=\"node01:9100\"} - node_filesystem_avail_bytes{mountpoint=\"/home\",instance=\"node01:9100\"}", "legendFormat": "Used" }, { "expr": "node_filesystem_size_bytes{mountpoint=\"/home\",instance=\"node01:9100\"}", "legendFormat": "Total" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, "id": 500, "title": "SLURM Jobs", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 20 } ] } } }, "gridPos": { "h": 6, "w": 4, "x": 0, "y": 69 }, "id": 501, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Jobs Running", "type": "stat", "targets": [ { "expr": "sum(slurm_queue_running)", "legendFormat": "Running" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 20 } ] } } }, "gridPos": { "h": 6, "w": 4, "x": 4, "y": 69 }, "id": 502, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Jobs Pending", "type": "stat", "targets": [ { "expr": "sum(slurm_queue_pending)", "legendFormat": "Pending" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 6, "w": 4, "x": 8, "y": 69 }, "id": 503, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "CPUs Allocated", "type": "stat", "targets": [ { "expr": "sum(slurm_cpus_alloc)", "legendFormat": "Allocated" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 6, "w": 4, "x": 12, "y": 69 }, "id": 504, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "CPUs Idle", "type": "stat", "targets": [ { "expr": "sum(slurm_cpus_idle)", "legendFormat": "Idle" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 6, "w": 4, "x": 16, "y": 69 }, "id": 505, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "CPUs Total", "type": "stat", "targets": [ { "expr": "sum(slurm_cpus_total)", "legendFormat": "Total" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } }, "gridPos": { "h": 6, "w": 4, "x": 20, "y": 69 }, "id": 506, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "textMode": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "title": "Nodes Idle", "type": "stat", "targets": [ { "expr": "sum(slurm_nodes_idle)", "legendFormat": "Idle" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "short", "min": 0 } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 75 }, "id": 507, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "SLURM Jobs Over Time", "type": "timeseries", "targets": [ { "expr": "sum(slurm_queue_running)", "legendFormat": "Running" }, { "expr": "sum(slurm_queue_pending)", "legendFormat": "Pending" } ] }, { "datasource": { "type": "prometheus", "uid": "afn5lu3umf75sf" }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 1, "fillOpacity": 20, "spanNulls": false, "showPoints": "never" }, "unit": "short", "min": 0 } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 75 }, "id": 508, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, "title": "SLURM CPU Allocation Over Time", "type": "timeseries", "targets": [ { "expr": "sum(slurm_cpus_alloc)", "legendFormat": "Allocated" }, { "expr": "sum(slurm_cpus_idle)", "legendFormat": "Idle" } ] } ] }