feat(openmetrics): add 9 missing host RRD metrics

Author: mpiton

CHANGELOG.unreleased.md

@@ -9,6 +9,8 @@
 
 ### Enhancements
 
+- [OpenMetrics] Add 9 missing host RRD metrics: `hostload`, `memory_reclaimed`, `memory_reclaimed_max`, `running_vcpus`, `pif_aggr_rx`, `pif_aggr_tx`, `iops_total`, `io_throughput_total`, `latency` per SR (PR [#XXXX](https://github.com/vatesfr/xen-orchestra/pull/XXXX))
+
 > Users must be able to say: "Nice enhancement, I'm eager to test it"
 
 ### Bug fixes
@@ -36,5 +38,7 @@
 - @xen-orchestra/immutable-backups patch
 - @xen-orchestra/web patch
 - @xen-orchestra/web-core patch
+- xo-server minor
+- xo-server-openmetrics minor
 
 <!--packages-end-->

docs/docs/advanced.md

@@ -437,25 +437,34 @@ Infrastructure metrics are prefixed with `xcp_` and XO management plane metrics
 
 #### Host Metrics
 
-| Metric                                  | Type    | Description                                                            |
-| --------------------------------------- | ------- | ---------------------------------------------------------------------- |
-| `xcp_host_load_average`                 | gauge   | Host load average                                                      |
-| `xcp_host_memory_free_bytes`            | gauge   | Free memory in bytes                                                   |
-| `xcp_host_memory_total_bytes`           | gauge   | Total memory in bytes                                                  |
-| `xcp_host_cpu_average`                  | gauge   | Average CPU usage (0-1)                                                |
-| `xcp_host_cpu_core_usage`               | gauge   | Per-core CPU usage                                                     |
-| `xcp_host_network_receive_bytes_total`  | counter | Network bytes received per interface                                   |
-| `xcp_host_network_transmit_bytes_total` | counter | Network bytes transmitted per interface                                |
-| `xcp_host_disk_iops_read`               | gauge   | Disk read IOPS per SR                                                  |
-| `xcp_host_disk_iops_write`              | gauge   | Disk write IOPS per SR                                                 |
-| `xcp_host_disk_throughput_read_bytes`   | gauge   | Disk read throughput (bytes/s)                                         |
-| `xcp_host_disk_throughput_write_bytes`  | gauge   | Disk write throughput (bytes/s)                                        |
-| `xcp_host_disk_read_latency_seconds`    | gauge   | Disk read latency                                                      |
-| `xcp_host_disk_write_latency_seconds`   | gauge   | Disk write latency                                                     |
-| `xcp_host_disk_iowait`                  | gauge   | Disk IO wait ratio                                                     |
-| `xcp_host_power_consumption_watts`      | gauge   | Power consumption in watts (DCMI)                                      |
-| `xcp_host_uptime_seconds`               | gauge   | Host uptime in seconds since boot                                      |
-| `xcp_host_status`                       | gauge   | Host status (1 = current state, `power_state` and `enabled` in labels) |
+| Metric                                       | Type    | Description                                                            |
+| -------------------------------------------- | ------- | ---------------------------------------------------------------------- |
+| `xcp_host_cpu_average`                       | gauge   | Average CPU usage (0-1)                                                |
+| `xcp_host_cpu_core_usage`                    | gauge   | Per-core CPU usage                                                     |
+| `xcp_host_disk_iops_read`                    | gauge   | Disk read IOPS per SR                                                  |
+| `xcp_host_disk_iops_write`                   | gauge   | Disk write IOPS per SR                                                 |
+| `xcp_host_disk_iops_total`                   | gauge   | Total IOPS (read + write) per SR                                       |
+| `xcp_host_disk_iowait`                       | gauge   | Disk IO wait ratio                                                     |
+| `xcp_host_disk_latency_seconds`              | gauge   | Combined I/O latency per SR in seconds                                 |
+| `xcp_host_disk_read_latency_seconds`         | gauge   | Disk read latency                                                      |
+| `xcp_host_disk_write_latency_seconds`        | gauge   | Disk write latency                                                     |
+| `xcp_host_disk_throughput_read_bytes`        | gauge   | Disk read throughput (bytes/s)                                         |
+| `xcp_host_disk_throughput_write_bytes`       | gauge   | Disk write throughput (bytes/s)                                        |
+| `xcp_host_disk_throughput_total_bytes`       | gauge   | Total I/O throughput per SR (bytes/s)                                  |
+| `xcp_host_load`                              | gauge   | Normalized host load                                                   |
+| `xcp_host_load_average`                      | gauge   | Host load average                                                      |
+| `xcp_host_memory_free_bytes`                 | gauge   | Free memory in bytes                                                   |
+| `xcp_host_memory_reclaimed_bytes`            | gauge   | Reclaimed host memory in bytes                                         |
+| `xcp_host_memory_reclaimed_max_bytes`        | gauge   | Maximum reclaimable host memory in bytes                               |
+| `xcp_host_memory_total_bytes`                | gauge   | Total memory in bytes                                                  |
+| `xcp_host_network_aggregated_receive_bytes`  | gauge   | Aggregated received bytes per second                                   |
+| `xcp_host_network_aggregated_transmit_bytes` | gauge   | Aggregated transmitted bytes per second                                |
+| `xcp_host_network_receive_bytes_total`       | counter | Network bytes received per interface                                   |
+| `xcp_host_network_transmit_bytes_total`      | counter | Network bytes transmitted per interface                                |
+| `xcp_host_power_consumption_watts`           | gauge   | Power consumption in watts (DCMI)                                      |
+| `xcp_host_running_vcpus`                     | gauge   | Total number of running vCPUs                                          |
+| `xcp_host_status`                            | gauge   | Host status (1 = current state, `power_state` and `enabled` in labels) |
+| `xcp_host_uptime_seconds`                    | gauge   | Host uptime in seconds since boot                                      |
 
 #### VM Metrics
 
@@ -578,7 +587,7 @@ rate(xcp_host_network_receive_bytes_total[5m]) / 1024 / 1024
 xcp_vm_disk_read_latency_seconds > 0.01
 
 # Total IOPS per Storage Repository
-sum by (sr_name) (xcp_host_disk_iops_read + xcp_host_disk_iops_write)
+sum by (sr_name) (xcp_host_disk_iops_total)
 
 # SR usage percentage
 (xcp_sr_physical_usage_bytes / xcp_sr_physical_size_bytes) * 100

packages/xo-server-openmetrics/src/openmetric-formatter.mts

@@ -115,6 +115,20 @@ export const HOST_METRICS: MetricDefinition[] = [
     extractLabels: matches => ({ core: matches[1]! }),
   },
 
+  // Aggregated network metrics (PIF)
+  {
+    test: 'pif_aggr_rx',
+    openMetricName: 'host_network_aggregated_receive_bytes',
+    type: 'gauge',
+    help: 'Aggregated received bytes per second',
+  },
+  {
+    test: 'pif_aggr_tx',
+    openMetricName: 'host_network_aggregated_transmit_bytes',
+    type: 'gauge',
+    help: 'Aggregated transmitted bytes per second',
+  },
+
   // Network metrics (PIF)
   {
     test: /^pif_(.+)_rx$/,
@@ -199,6 +213,67 @@ export const HOST_METRICS: MetricDefinition[] = [
     type: 'gauge',
     help: 'Host power consumption in watts (DCMI)',
   },
+
+  // Normalized host load
+  {
+    test: 'hostload',
+    openMetricName: 'host_load',
+    type: 'gauge',
+    help: 'Normalized host load',
+  },
+
+  // Reclaimed memory metrics
+  {
+    test: 'memory_reclaimed',
+    openMetricName: 'host_memory_reclaimed_bytes',
+    type: 'gauge',
+    help: 'Reclaimed host memory in bytes',
+    transformValue: v => v * 1024, // KiB to bytes
+  },
+  {
+    test: 'memory_reclaimed_max',
+    openMetricName: 'host_memory_reclaimed_max_bytes',
+    type: 'gauge',
+    help: 'Maximum reclaimable host memory in bytes',
+    transformValue: v => v * 1024, // KiB to bytes
+  },
+
+  // Running vCPUs
+  {
+    test: 'running_vcpus',
+    openMetricName: 'host_running_vcpus',
+    type: 'gauge',
+    help: 'Total number of running vCPUs',
+  },
+
+  // Total disk IOPS per SR
+  {
+    test: /^iops_total_(.+)$/,
+    openMetricName: 'host_disk_iops_total',
+    type: 'gauge',
+    help: 'Total IOPS (read + write) per SR',
+    extractLabels: matches => ({ sr: matches[1]! }),
+  },
+
+  // Total disk throughput per SR
+  {
+    test: /^io_throughput_total_(.+)$/,
+    openMetricName: 'host_disk_throughput_total_bytes',
+    type: 'gauge',
+    help: 'Total I/O throughput per SR in bytes per second',
+    transformValue: v => v * Math.pow(2, 20), // MiB to bytes
+    extractLabels: matches => ({ sr: matches[1]! }),
+  },
+
+  // Combined disk latency per SR
+  {
+    test: /^latency_(.+)$/,
+    openMetricName: 'host_disk_latency_seconds',
+    type: 'gauge',
+    help: 'Combined I/O latency per SR in seconds',
+    transformValue: v => v / 1e6, // µs to seconds
+    extractLabels: matches => ({ sr: matches[1]! }),
+  },
 ]
 
 /**

packages/xo-server-openmetrics/src/openmetric-formatter.test.mts

@@ -188,6 +188,118 @@ describe('HOST_METRICS DCMI', () => {
   })
 })
 
+describe('HOST_METRICS new metrics', () => {
+  it('should include hostload metric', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_load')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.equal(metric.test, 'hostload')
+    assert.equal(metric.transformValue, undefined)
+  })
+
+  it('should include memory_reclaimed metric with KiB to bytes transformation', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_memory_reclaimed_bytes')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.equal(metric.test, 'memory_reclaimed')
+    assert.ok(metric.transformValue)
+    // 512 KiB * 1024 = 524288 bytes
+    assert.equal(metric.transformValue!(512), 512 * 1024)
+  })
+
+  it('should include memory_reclaimed_max metric with KiB to bytes transformation', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_memory_reclaimed_max_bytes')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.equal(metric.test, 'memory_reclaimed_max')
+    assert.ok(metric.transformValue)
+    assert.equal(metric.transformValue!(1024), 1024 * 1024)
+  })
+
+  it('should include running_vcpus metric', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_running_vcpus')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.equal(metric.test, 'running_vcpus')
+    assert.equal(metric.transformValue, undefined)
+  })
+
+  it('should include pif_aggr_rx metric', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_network_aggregated_receive_bytes')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.equal(metric.test, 'pif_aggr_rx')
+    assert.equal(metric.transformValue, undefined)
+  })
+
+  it('should include pif_aggr_tx metric', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_network_aggregated_transmit_bytes')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.equal(metric.test, 'pif_aggr_tx')
+    assert.equal(metric.transformValue, undefined)
+  })
+
+  it('should match pif_aggr_rx before the generic PIF regex', () => {
+    const result = findMetricDefinition('pif_aggr_rx', 'host')
+    assert.ok(result)
+    assert.equal(result.definition.openMetricName, 'host_network_aggregated_receive_bytes')
+  })
+
+  it('should include iops_total per SR with label extraction', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_disk_iops_total')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.ok(metric.extractLabels)
+
+    const regex = metric.test as RegExp
+    const match = regex.exec('iops_total_abc12345')
+    assert.ok(match)
+    assert.deepEqual(metric.extractLabels!(match), { sr: 'abc12345' })
+  })
+
+  it('should include io_throughput_total per SR with MiB to bytes transformation', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_disk_throughput_total_bytes')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.ok(metric.transformValue)
+    assert.ok(metric.extractLabels)
+
+    // 2 MiB/s = 2 * 2^20 bytes/s
+    assert.equal(metric.transformValue!(2), 2 * 2 ** 20)
+
+    const regex = metric.test as RegExp
+    const match = regex.exec('io_throughput_total_def-456-789')
+    assert.ok(match)
+    assert.deepEqual(metric.extractLabels!(match), { sr: 'def-456-789' })
+  })
+
+  it('should include latency per SR with µs to seconds transformation', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_disk_latency_seconds')
+    assert.ok(metric)
+    assert.equal(metric.type, 'gauge')
+    assert.ok(metric.transformValue)
+    assert.ok(metric.extractLabels)
+
+    // 500 µs / 1e6 = 0.0005 seconds
+    assert.equal(metric.transformValue!(500), 0.0005)
+
+    const regex = metric.test as RegExp
+    const match = regex.exec('latency_abc-def-123')
+    assert.ok(match)
+    assert.deepEqual(metric.extractLabels!(match), { sr: 'abc-def-123' })
+  })
+
+  it('should not match read_latency or write_latency with latency_<sr> regex', () => {
+    const metric = HOST_METRICS.find(m => m.openMetricName === 'host_disk_latency_seconds')
+    assert.ok(metric)
+    const regex = metric.test as RegExp
+    // ^latency_ anchor prevents matching read_latency_ and write_latency_
+    assert.equal(regex.exec('read_latency_abc12345'), null)
+    assert.equal(regex.exec('write_latency_abc12345'), null)
+  })
+})
+
 // ============================================================================
 // findMetricDefinition Tests
 // ============================================================================

packages/xo-server/src/xapi-stats.mjs

@@ -156,6 +156,10 @@ const STATS = {
         test: /^iops_write_(\w+)$/,
         getPath: matches => ['iops', 'w', matches[1]],
       },
+      total: {
+        test: /^iops_total_(\w+)$/,
+        getPath: matches => ['iops', 'total', matches[1]],
+      },
     },
     ioThroughput: {
       r: {
@@ -168,6 +172,11 @@ const STATS = {
         getPath: matches => ['ioThroughput', 'w', matches[1]],
         transformValue: value => value * 2 ** 20,
       },
+      total: {
+        test: /^io_throughput_total_(\w+)$/,
+        getPath: matches => ['ioThroughput', 'total', matches[1]],
+        transformValue: value => value * 2 ** 20, // MiB/s to bytes/s
+      },
     },
     latency: {
       r: {
@@ -180,11 +189,40 @@ const STATS = {
         getPath: matches => ['latency', 'w', matches[1]],
         transformValue: value => value / 1e3,
       },
+      combined: {
+        test: /^latency_(\w+)$/,
+        getPath: matches => ['latency', 'combined', matches[1]],
+        transformValue: value => value / 1e3,
+      },
     },
     iowait: {
       test: /^iowait_(\w+)$/,
       getPath: matches => ['iowait', matches[1]],
     },
+    hostload: {
+      test: 'hostload',
+    },
+    memoryReclaimed: {
+      test: 'memory_reclaimed',
+      transformValue: value => value * 1024, // KiB to bytes
+    },
+    memoryReclaimedMax: {
+      test: 'memory_reclaimed_max',
+      transformValue: value => value * 1024, // KiB to bytes
+    },
+    runningVcpus: {
+      test: 'running_vcpus',
+    },
+    pifsAggr: {
+      rx: {
+        test: 'pif_aggr_rx',
+        getPath: () => ['pifsAggr', 'rx'],
+      },
+      tx: {
+        test: 'pif_aggr_tx',
+        getPath: () => ['pifsAggr', 'tx'],
+      },
+    },
   },
   vm: {
     memoryFree: {
@@ -279,12 +317,12 @@ const STATS = {
       r: {
         test: /^vbd_xvd(.)_read_latency$/,
         getPath: matches => ['vbdLatency', 'r', matches[1]],
-        transformValue: value => value / 1000,
+        transformValue: value => value / 1e3, // µs to ms
       },
       w: {
         test: /^vbd_xvd(.)_write_latency$/,
         getPath: matches => ['vbdLatency', 'w', matches[1]],
-        transformValue: value => value / 1000,
+        transformValue: value => value / 1e3, // µs to ms
       },
     },
     ioThroughput: {
@@ -307,7 +345,7 @@ const STATS = {
     vbdAvgLatency: {
       test: /^vbd_xvd(.)_latency$/,
       getPath: matches => ['vbdAvgLatency', matches[1]],
-      transformValue: value => value / 1e3,
+      transformValue: value => value / 1e3, // µs to ms (for xo-web formatTime)
     },
     vbdIowait: {
       test: /^vbd_xvd(.)_iowait$/,