|
| 1 | +# -------------------------------------------------------------------------- # |
| 2 | +# Copyright 2002-2026, OpenNebula Project, OpenNebula Systems # |
| 3 | +# # |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); you may # |
| 5 | +# not use this file except in compliance with the License. You may obtain # |
| 6 | +# a copy of the License at # |
| 7 | +# # |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 # |
| 9 | +# # |
| 10 | +# Unless required by applicable law or agreed to in writing, software # |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, # |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # |
| 13 | +# See the License for the specific language governing permissions and # |
| 14 | +# limitations under the License. # |
| 15 | +#--------------------------------------------------------------------------- # |
| 16 | + |
| 17 | +require 'socket' |
| 18 | +require 'timeout' |
| 19 | +require 'xmlrpc/client' |
| 20 | +require 'rexml/document' |
| 21 | +require 'opennebula' |
| 22 | + |
| 23 | +# Metrics exposed: |
| 24 | +# - opennebula_ha_is_leader{one_server_fqdn="..."} 0|1 |
| 25 | +# - opennebula_ha_error_members{one_server_fqdn="..."} N |
| 26 | +# - opennebula_ha_members{one_server_fqdn="..."} N |
| 27 | +class OpenNebulaHACollector |
| 28 | + |
| 29 | + LABELS = [:one_server_fqdn].freeze |
| 30 | + |
| 31 | + STATE_FOLLOWER = 2 |
| 32 | + STATE_LEADER = 3 |
| 33 | + |
| 34 | + METRICS = { |
| 35 | + 'ha_is_leader' => { |
| 36 | + :type => :gauge, |
| 37 | + :docstr => 'This server is RAFT leader (1) or not (0)', |
| 38 | + :labels => LABELS |
| 39 | + }, |
| 40 | + 'ha_error_members' => { |
| 41 | + :type => :gauge, |
| 42 | + :docstr => 'Number of HA members with failed or unexpected RAFT state', |
| 43 | + :labels => LABELS |
| 44 | + }, |
| 45 | + 'ha_members' => { |
| 46 | + :type => :gauge, |
| 47 | + :docstr => 'Total number of HA members reported by zone pool', |
| 48 | + :labels => LABELS |
| 49 | + } |
| 50 | + }.freeze |
| 51 | + |
| 52 | + def initialize(registry, _client, namespace) |
| 53 | + @metrics = build_metrics(registry, namespace) |
| 54 | + @auth = load_auth |
| 55 | + @xmlrpc_endpoint = load_xmlrpc_endpoint |
| 56 | + @timeout = load_timeout |
| 57 | + @fqdn = resolve_fqdn |
| 58 | + @one_client = build_one_client |
| 59 | + end |
| 60 | + |
| 61 | + def collect |
| 62 | + labels = metric_labels |
| 63 | + statuses = cluster_statuses |
| 64 | + return if statuses.empty? |
| 65 | + |
| 66 | + write_metrics(statuses, labels) |
| 67 | + rescue StandardError => e |
| 68 | + warn("[OpenNebulaHACollector] #{e.class}: #{e.message}") |
| 69 | + nil |
| 70 | + end |
| 71 | + |
| 72 | + private |
| 73 | + |
| 74 | + def build_metrics(registry, namespace) |
| 75 | + metrics = {} |
| 76 | + |
| 77 | + METRICS.each do |name, conf| |
| 78 | + metrics[name] = registry.public_send( |
| 79 | + conf[:type], |
| 80 | + "#{namespace}_#{name}".to_sym, |
| 81 | + :docstring => conf[:docstr], |
| 82 | + :labels => conf[:labels] |
| 83 | + ) |
| 84 | + end |
| 85 | + |
| 86 | + metrics |
| 87 | + end |
| 88 | + |
| 89 | + def load_auth |
| 90 | + path = ENV.fetch('ONE_AUTH', File.expand_path('~/.one/one_auth')) |
| 91 | + File.read(path).strip |
| 92 | + end |
| 93 | + |
| 94 | + def load_xmlrpc_endpoint |
| 95 | + ENV.fetch('ONE_XMLRPC', 'http://localhost:2633/RPC2') |
| 96 | + end |
| 97 | + |
| 98 | + def load_timeout |
| 99 | + (ENV['ONEZONE_TIMEOUT'] || '5').to_i |
| 100 | + end |
| 101 | + |
| 102 | + def build_one_client |
| 103 | + OpenNebula::Client.new(@auth, @xmlrpc_endpoint) |
| 104 | + end |
| 105 | + |
| 106 | + def metric_labels |
| 107 | + { :one_server_fqdn => @fqdn } |
| 108 | + end |
| 109 | + |
| 110 | + def cluster_statuses |
| 111 | + servers = fetch_zone_servers |
| 112 | + return [] if servers.empty? |
| 113 | + |
| 114 | + servers.map {|srv| fetch_server_status(srv) } |
| 115 | + end |
| 116 | + |
| 117 | + def write_metrics(statuses, labels) |
| 118 | + @metrics['ha_members'].set(member_count(statuses), :labels => labels) |
| 119 | + @metrics['ha_error_members'].set(error_member_count(statuses), :labels => labels) |
| 120 | + @metrics['ha_is_leader'].set(local_leader_value, :labels => labels) |
| 121 | + end |
| 122 | + |
| 123 | + def member_count(statuses) |
| 124 | + statuses.size |
| 125 | + end |
| 126 | + |
| 127 | + def error_member_count(statuses) |
| 128 | + statuses.count {|status| problem_state?(status) } |
| 129 | + end |
| 130 | + |
| 131 | + def local_leader_value |
| 132 | + local_status = fetch_local_status |
| 133 | + local_status[:state_num] == STATE_LEADER ? 1 : 0 |
| 134 | + end |
| 135 | + |
| 136 | + def fetch_zone_servers |
| 137 | + pool = OpenNebula::ZonePool.new(@one_client) |
| 138 | + rc = pool.info |
| 139 | + |
| 140 | + raise rc.message if OpenNebula.is_error?(rc) |
| 141 | + |
| 142 | + extract_servers(pool) |
| 143 | + end |
| 144 | + |
| 145 | + def extract_servers(pool) |
| 146 | + servers = [] |
| 147 | + |
| 148 | + pool.each do |zone| |
| 149 | + zone.each('SERVER_POOL/SERVER') do |srv| |
| 150 | + servers << zone_server_hash(srv) |
| 151 | + end |
| 152 | + end |
| 153 | + |
| 154 | + servers |
| 155 | + end |
| 156 | + |
| 157 | + def zone_server_hash(server) |
| 158 | + { |
| 159 | + :id => server['ID'].to_i, |
| 160 | + :name => server['NAME'].to_s, |
| 161 | + :endpoint => server['ENDPOINT'].to_s |
| 162 | + } |
| 163 | + end |
| 164 | + |
| 165 | + def fetch_local_status |
| 166 | + fetch_raft(@xmlrpc_endpoint) |
| 167 | + rescue StandardError |
| 168 | + error_status |
| 169 | + end |
| 170 | + |
| 171 | + def fetch_server_status(server) |
| 172 | + status = fetch_raft(server[:endpoint]) |
| 173 | + merge_server_status(status, server) |
| 174 | + rescue StandardError |
| 175 | + merge_server_status(error_status, server) |
| 176 | + end |
| 177 | + |
| 178 | + def merge_server_status(status, server) |
| 179 | + status.merge( |
| 180 | + :id => server[:id], |
| 181 | + :name => server[:name] |
| 182 | + ) |
| 183 | + end |
| 184 | + |
| 185 | + def error_status |
| 186 | + { |
| 187 | + :state_num => nil, |
| 188 | + :rpc_error => true |
| 189 | + } |
| 190 | + end |
| 191 | + |
| 192 | + def fetch_raft(endpoint) |
| 193 | + xml = rpc_call(endpoint, 'one.zone.raftstatus') |
| 194 | + parse_raft(xml) |
| 195 | + end |
| 196 | + |
| 197 | + def rpc_call(endpoint, method) |
| 198 | + Timeout.timeout(@timeout) do |
| 199 | + client = XMLRPC::Client.new2(endpoint) |
| 200 | + ok, payload, error_code = client.call(method, @auth) |
| 201 | + |
| 202 | + raise "#{method} failed: #{error_code}" unless ok |
| 203 | + |
| 204 | + payload |
| 205 | + end |
| 206 | + end |
| 207 | + |
| 208 | + def parse_raft(xml) |
| 209 | + doc = REXML::Document.new(xml) |
| 210 | + raft = REXML::XPath.first(doc, '//RAFT') |
| 211 | + |
| 212 | + raise 'RAFT not found' unless raft |
| 213 | + |
| 214 | + { |
| 215 | + :state_num => text(raft, 'STATE')&.to_i, |
| 216 | + :rpc_error => false |
| 217 | + } |
| 218 | + end |
| 219 | + |
| 220 | + def text(node, name) |
| 221 | + el = REXML::XPath.first(node, name) |
| 222 | + el&.text |
| 223 | + end |
| 224 | + |
| 225 | + def problem_state?(status) |
| 226 | + return true if status[:rpc_error] |
| 227 | + |
| 228 | + state = status[:state_num] |
| 229 | + state != STATE_LEADER && state != STATE_FOLLOWER |
| 230 | + end |
| 231 | + |
| 232 | + def resolve_fqdn |
| 233 | + host = Socket.gethostname |
| 234 | + Addrinfo.getaddrinfo(host, nil).first.getnameinfo.first |
| 235 | + rescue StandardError |
| 236 | + Socket.gethostname |
| 237 | + end |
| 238 | + |
| 239 | +end |
0 commit comments