System Metrics

.gitignore

@@ -55,3 +55,5 @@ uv.lock
 
 # evalml
 .evalml_snakemake_cmd.txt
+rulegraph.svg
+dag.svg

pyproject.toml

@@ -57,7 +57,8 @@ markers = [
 packages = [
   "src/evalml",
   "src/verification",
-  "src/data_input"
+  "src/data_input",
+  "src/diagnostics"
 ]
 
 [tool.uv.sources]

resources/report/dashboard/script.js

@@ -185,3 +185,131 @@ function updateChart() {
 
 // Initial chart
 updateChart()
+
+
+// ---- System metrics tab ----
+
+const sysDataEl = document.getElementById("sysmetrics-data");
+const sysData = sysDataEl ? JSON.parse(sysDataEl.textContent) : [];
+
+if (sysData.length > 0) {
+  choicesInstances["sys-model-type-select"] = new Choices("#sys-model-type-select", {
+    searchEnabled: false,
+    removeItemButton: true,
+    shouldSort: false,
+    itemSelectText: "",
+    placeholder: false,
+  });
+  document.getElementById("sys-model-type-select").addEventListener("change", updateSysChart);
+
+  choicesInstances["sys-source-select"] = new Choices("#sys-source-select", {
+    searchEnabled: false,
+    removeItemButton: true,
+    shouldSort: false,
+    itemSelectText: "",
+    placeholder: false,
+  });
+  document.getElementById("sys-source-select").addEventListener("change", updateSysChart);
+
+  // Populate metric filter from the data, then initialise Choices on it
+  const sysMetricEl = document.getElementById("sys-metric-select");
+  [...new Set(sysData.map(d => d.metric))].sort().forEach(m => {
+    const opt = document.createElement("option");
+    opt.value = m;
+    opt.textContent = m;
+    opt.selected = true;
+    sysMetricEl.appendChild(opt);
+  });
+  choicesInstances["sys-metric-select"] = new Choices("#sys-metric-select", {
+    searchEnabled: false,
+    removeItemButton: true,
+    shouldSort: false,
+    itemSelectText: "",
+    placeholder: false,
+  });
+  document.getElementById("sys-metric-select").addEventListener("change", updateSysChart);
+
+  const sysSpec = {
+    "data": { "values": sysData },
+    "facet": { "field": "metric", "type": "nominal", "title": null },
+    "columns": 4,
+    "resolve": { "scale": { "x": "shared", "y": "independent" } },
+    "spec": {
+      "params": [
+        {
+          "name": "xZoom",
+          "select": {
+            "type": "interval",
+            "encodings": ["x"],
+            "zoom": "wheel![!event.shiftKey]"
+          },
+          "bind": "scales"
+        }
+      ],
+      "width": 300,
+      "height": 200,
+      "mark": { "type": "line", "point": { "filled": true, "size": 50 } },
+      "encoding": {
+        "x": {
+          "field": "init_time",
+          "type": "temporal",
+          "title": null,
+          "axis": { "labelAngle": -30, "format": "%b %d" }
+        },
+        "y": {
+          "field": "value",
+          "type": "quantitative",
+          "title": null,
+          "scale": { "zero": false }
+        },
+        "color": {
+          "field": "source",
+          "type": "nominal",
+          "legend": { "orient": "top", "title": "Source" }
+        },
+        "shape": {
+          "field": "model_type",
+          "type": "nominal",
+          "legend": { "orient": "top", "title": "Model type" }
+        },
+        "strokeDash": {
+          "field": "model_type",
+          "type": "nominal",
+          "legend": { "orient": "top", "title": "Model type" }
+        },
+        "tooltip": [
+          { "field": "source", "type": "nominal", "title": "Source" },
+          { "field": "model_type", "type": "nominal", "title": "Model type" },
+          { "field": "init_time", "type": "temporal", "title": "Init time", "format": "%Y-%m-%d %H:%M" },
+          { "field": "metric", "type": "nominal", "title": "Metric" },
+          { "field": "value", "type": "quantitative", "title": "Value", "format": ".3f" },
+          { "field": "n_gpu", "type": "quantitative", "title": "GPUs" },
+          { "field": "job_id", "type": "nominal", "title": "Job ID" },
+        ],
+      },
+    },
+  };
+
+  function updateSysChart() {
+    const selectedModelTypes = getSelectedValues("sys-model-type-select");
+    const selectedSources = getSelectedValues("sys-source-select");
+    const selectedMetrics = getSelectedValues("sys-metric-select");
+    const newSpec = JSON.parse(JSON.stringify(sysSpec));
+    const filters = [];
+    if (selectedModelTypes.length > 0) {
+      filters.push({ field: "model_type", oneOf: selectedModelTypes });
+    }
+    if (selectedSources.length > 0) {
+      filters.push({ field: "source", oneOf: selectedSources });
+    }
+    if (selectedMetrics.length > 0) {
+      filters.push({ field: "metric", oneOf: selectedMetrics });
+    }
+    if (filters.length > 0) {
+      newSpec.transform = [{ filter: { and: filters } }];
+    }
+    vegaEmbed("#sys-vis", newSpec, { actions: false });
+  }
+
+  updateSysChart();
+}

resources/report/dashboard/template.html.jinja2

@@ -95,6 +95,9 @@
     <div class="header">
         <div class="tab-buttons">
             <button class="tab-link active" data-tab="tab_scores">Standard verification</button>
+            {% if sysmetrics_sources %}
+            <button class="tab-link" data-tab="tab_sysmetrics">System metrics</button>
+            {% endif %}
             <button class="tab-link" data-tab="tab_config">Config</button>
         </div>
     </div>
@@ -155,6 +158,35 @@
     </div>
 
 
+    <!-- System metrics tab -->
+    {% if sysmetrics_sources %}
+    <div id="tab_sysmetrics" class="tab-content">
+        <div class="controls">
+            <div class="control-group">
+                <label>Model type(s)</label>
+                <select id="sys-model-type-select" multiple>
+                    {% for model_type in sysmetrics_model_types %}
+                    <option value="{{model_type}}" selected>{{model_type}}</option>
+                    {% endfor %}
+                </select>
+            </div>
+            <div class="control-group">
+                <label>Source(s)</label>
+                <select id="sys-source-select" multiple>
+                    {% for source in sysmetrics_sources %}
+                    <option value="{{source}}" selected>{{source}}</option>
+                    {% endfor %}
+                </select>
+            </div>
+            <div class="control-group">
+                <label>Metric(s)</label>
+                <select id="sys-metric-select" multiple></select>
+            </div>
+        </div>
+        <div id="sys-vis"></div>
+    </div>
+    {% endif %}
+
     <!-- Config file tab -->
     <div id="tab_config" class="tab-content">
         <pre style="background:#f7f7f7; border:1px solid #ddd; padding:1em; overflow-x:auto; max-height:60vh;">
@@ -163,6 +195,10 @@
     </div>
 
 
+    <script id="sysmetrics-data" type="application/json">
+        {{ sysmetrics_data | safe | indent(8, false) }}
+    </script>
+
     <script id="verif-data" type="application/json">
         {{ verif_data | safe | indent(8, false)}}
     </script>